Merge branch 'gh-4142--nginx-ssl-aggressive': extends filter.d/nginx-http-auth.conf - modes fallback and aggressive match more SSL failures by SSL_do_handshake or SSL_read (gh-4142, gh-2881)

Improve systemd service

.codespellrc

@@ -1,6 +1,6 @@
 [codespell]
 # THANKS - names
-skip = .git,*.pdf,*.svg,venv,.codespellrc,THANKS,*test*.log,logs
+skip = .git,*.pdf,*.svg,venv,.codespellrc,.typos.toml,THANKS,*test*.log,logs
 check-hidden = true
 # Ignore all acronyms etc as plenty e.g. in fail2ban/server/strptime.py
 # Try to identify incomplete words which are part of a regex, hence having [] at the beginning
@@ -9,4 +9,4 @@ check-hidden = true
 ignore-regex = (\b([A-Z][A-Z][A-Z]+|gir\.st)\b)|\[[a-zA-Z]+\][a-z]+\b|[a-z]+://\S+|.*codespell-ignore.*
 # some oddly named variables, some names, etc
 # wee -- comes in regex etc for weeks
-ignore-words-list = theis,timere,alls,wee,wight,ans,re-use
+ignore-words-list = assertIn,theis,timere,alls,wee,wight,ans,re-use,pre-emptive

.github/FUNDING.yml

@@ -1,4 +1,5 @@
 # These are supported funding model platforms
 
 github: [sebres]
-custom: [paypal.me/sebres]
+custom: [https://paypal.me/sebres]
+liberapay: sebres

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,12 +1,10 @@
 Before submitting your PR, please review the following checklist:
 
-- [ ] **CHOOSE CORRECT BRANCH**: if filing a bugfix/enhancement
-      against certain release version, choose `0.9`, `0.10` or `0.11` branch,
-      for dev-edition use `master` branch
 - [ ] **CONSIDER adding a unit test** if your PR resolves an issue
-- [ ] **LIST ISSUES** this PR resolves
+- [ ] **LIST ISSUES** this PR resolves or describe the approach in detail
 - [ ] **MAKE SURE** this PR doesn't break existing tests
-- [ ] **KEEP PR small** so it could be easily reviewed.
+- [ ] **KEEP PR small** so it could be easily reviewed
 - [ ] **AVOID** making unnecessary stylistic changes in unrelated code
 - [ ] **ACCOMPANY** each new `failregex` for filter `X` with sample log lines
-      within `fail2ban/tests/files/logs/X` file
+      (and `# failJSON`) within `fail2ban/tests/files/logs/X` file
+- [ ] **PROVIDE ChangeLog** entry describing the pull request

.github/workflows/main.yml

@@ -19,10 +19,10 @@ jobs:
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13.0-alpha.2', pypy3.10]
+        python-version: [3.8, 3.9, '3.10', '3.11', '3.12', '3.13', '3.14', '3.15.0-alpha.5', pypy3.11]
       fail-fast: false
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -30,7 +30,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -79,7 +79,7 @@ jobs:
           cd "$GITHUB_WORKSPACE"
           _debug() { echo -n "$1 "; err=$("${@:2}" 2>&1) && echo 'OK' || echo -e "FAIL\n$err"; }
           # (debug) output current preferred encoding:
-          _debug 'Encodings:' python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))'
+          echo 'Encodings:' $(python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))')
           # (debug) backend availabilities:
           echo 'Backends:'
           _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd'

.github/workflows/publish.yml

@@ -0,0 +1,32 @@
+name: Upload Package to PyPI
+
+on:
+  workflow_dispatch:
+  release:
+    types: [created]
+
+jobs:
+  pypi-publish:
+    name: Publish release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/fail2ban
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip || echo "can't upgrade pip"
+          pip install setuptools wheel || echo "can't install/update setuptools or wheel"
+      - name: Build package
+        run: |
+          # python -m build ...
+          python setup.py sdist bdist_wheel
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.typos.toml

@@ -0,0 +1,35 @@
+[files]
+extend-exclude = [
+    ".git/",
+    ".codespellrc",
+    "fail2ban/tests/files/logs/",
+]
+ignore-hidden = false
+
+[default]
+extend-ignore-re = [
+    "Christoph Theis",
+    "\\[[0-9a-f]{7,8}\\]",
+    "hash_[0-9a-f]{38}",
+    "\t[0-9.-]+[ A-Z]+",
+    "Erreur d'authentification",
+    "WebEMailExtrac",
+    "ssh2: RSA 14:ba:xx",
+    "\\[Cc\\]lient",
+    "\\[Gg\\]ot",
+    "\\[nN\\]ot",
+    "\\[Uu\\]nknown",
+    "\\[uU\\]ser",
+    "\\[Uu\\]\\(\\?:ser",
+]
+
+[default.extend-words]
+"alls" = "alls"
+"helo" = "helo"
+
+[default.extend-identifiers]
+"failManager2nd" = "failManager2nd"
+"log2nd" = "log2nd"
+"routeros" = "routeros"
+"REFERERS" = "REFERERS"
+"tre_search" = "tre_search"

ChangeLog

@@ -7,6 +7,124 @@
 Fail2Ban: Changelog
 ===================
 
+ver. 1.1.1-dev-1 (20??/??/??) - development nightly edition
+-----------
+
+### Compatibility
+* `action.d/iptables.conf` rewritten due to support of multiple chains (gh-3909), therefore user-level derivations 
+  (action including iptables-based action) may become incompatible, e. g. some tags if used need to be replaced,
+  e. g. `<chain>` with `$chain` or `<_ipt_for_proto-iter>` with `<_ipt-iter>`;
+* `filter.d/exim.conf` - several rules of mode `normal` moved to new mode `more`, because of too risky handling (see [gh-3940](https://github.com/fail2ban/fail2ban/pull/3940)),
+  to use it as before set `mode = more` for exim jail, but be aware of the consequences.
+
+### Fixes
+* fixes `systemd` bug with missing journal descriptor after rotation by reopening of journal if it is recognized as not alive (gh-3929)
+* improve threaded clean-up of all filters, new thread functions `afterStop` (to force clean-up after stop) and `done`, invoking `afterStop` once
+* ensure journal-reader is always closed (additional prevention against leaks and "too many open files"), thereby avoid sporadic segfault
+  in systemd module (see https://github.com/systemd/python-systemd/issues/143)
+* fixes `systemd` causing "too many open files" error for a lot of journal files and large amount of systemd jails 
+  (see new parameter `rotated` below, gh-3391);
+* passing of arguments from jails to action or filter will affect conditional section too (gh-4069), 
+  e. g. setting `blocktype="DROP"` via jail for action would now apply for IPv4 and IPv6 chains,
+  to submit different `blocktype` for IPv4 and IPv6 from jail, one can pass them like in this example:
+  `banaction = iptables-ipset[blocktype="...", blocktype?family=inet6="..."]`
+* `jail.conf`:
+  - default banactions need to be specified in `paths-*.conf` (maintainer level) now
+  - since stock fail2ban includes `paths-debian.conf` by default, banactions are `nftables`
+    (can be overwritten in `jail.local` by user)
+* `paths-common.conf`:
+  - changed default `mysql_log` path (default `logpath` of `mysqld-auth` jail without maintainer overrides, gh-3932)
+* `paths-debian.conf`:
+  - default banactions are `nftables`
+  - sshd backend switched to `systemd` (gh-3292)
+  - postfix backend switched to `systemd` (gh-3527)
+* `action.d/firewallcmd-ipset.conf`:
+  - rename `ipsettype` to `ipsetbackend` (gh-2620), parameter `ipsettype` will be used now to the real set type (gh-3760)
+* `action.d/nftables.conf`:
+  - action fixed for SELinux without execmem permission, rewrite capturing with `grep -P` using `grep -E` or `sed`
+    (PCRE-JIT by `grep -P` may cause SELinux denial for execmem, see gh-4137)
+* `action.d/xarf-login-attack.conf` - ignore errors or warnings in output of `dig` provided as comment (gh-4068)
+* `filter.d/apache-badbots.conf`, `filter.d/apache-fakegooglebot.conf`:
+  - regexs rewritten more strict (removed catch-alls, etc);
+  - regexs fixed to match lines with vhost in accesslog (gh-1594)
+* `filter.d/apache-noscript.conf` - consider new log-format with "AH02811: stderr from /..." (gh-3900)
+* `filter.d/apache-overflows.conf` - consider AH10244: invalid URI path (gh-3778, gh-3900)
+* `filter.d/asterisk.conf` - fixed RE for "no matching endpoint" with retry info (like `after X tries in Y ms`) at end, 
+  loosening of end anchor (ignore any simple text tokens at end if no single quote found), gh-4037
+* `filter.d/exim.conf`:
+  - several rules of mode `normal` moved to new mode `more`, because of too risky handling (gh-3940),
+    thereby mode `aggressive` is not affected, because it fully includes mode `more` now;
+  - mode `aggressive` extended to catch dropped by ACL failures, e.g. "ACL: Country is banned"
+* `filter.d/freeswitch.conf` - bypass some new info in prefix before [WARNING] (changed default `_pref_line`),
+  FreeSWITCH log line prefix has changed in newer versions (gh-3143)
+* `filter.d/lighttpd-auth.conf` - fixed regex (if failures generated by systemd-journal), bypass several prefixes now (gh-3955)
+* `filter.d/postfix.conf`:
+  - consider CONNECT and other rejected commands as a valid `_pref` (gh-3800)
+  - default `_daemon` in prefix-line is loosened - can match everything starting with word postfix, like `postfix-example.com/smtpd` (gh-3297)
+  - add optional `NOQUEUE:` prefix to ddos regex (gh-4072)
+  - internal parameter `_pref` is renamed to `_cmd`, `_pref` matches now optional prefix like `NOQUEUE: ` etc
+  - modes `ddos` and `aggressive` extended to match `rate limit exceeded` for connection or message delivery request rates (gh-3265, gh-4073)
+* `filter.d/dropbear.conf`:
+  - recognizes extra pid/timestamp if logged into stdout/journal, added `journalmatch` (gh-3597)
+  - failregex extended to match different format of "Exit before auth" message (gh-3791)
+* `filter.d/recidive.conf` - restore possibility to set jail name in the filter, _jailname is positive now (gh-3769)
+* `filter.d/roundcube-auth.conf` - improved RE better matching log format of roundcube version 1.4+ (gh-3816)
+* `filter.d/sendmail-reject.conf`: (gh-4020)
+  - support `<F-MLFID>` for BSD-style logfiles
+  - add match for `User unknown` to default
+  - the relay field may not always have a hostname before the ip address
+  - mode `aggressive` enables match for `lost input channel` and `Cannot resolve PTR record`
+* `filter.d/sshd.conf`:
+  - adapted to conform possible new daemon name sshd-session, since OpenSSH 9.8 
+    several log messages will be tagged with as originating from a process named "sshd-session" rather than "sshd" (gh-3782)
+  - `ddos` and `aggressive` modes: regex extended for timeout before authentication (optional connection from part, gh-3907)
+* `filter.d/vsftpd.conf` - fixed regex (if failures generated by systemd-journal, gh-3954)
+* `filter.d/froxlor-auth.conf` - updated the regex to the new logging situation for froxlor and changed logpath in jail.conf (gh-4075).	
+
+### New Features and Enhancements
+* backend `systemd` extended with new parameter `rotated` (default `false`, as prevention against "too many open files"),
+  that allows to monitor only actual journals and ignore now a lot of rotated files by default; so can drastically reduce
+  amount of used file descriptors, normally to 1 or 2 descriptors per jail (gh-3391)
+* new jail option `skip_if_nologs` to ignore jail if no `logpath` matches found, fail2ban continue to start with warnings/errors,
+  thus other jails become running (gh-2756)
+* implements automatic switch `backend = auto` to backend `systemd`, when the following is true (RFE gh-3768):
+  - no files matching `logpath` found for this jail;
+  - no `systemd_if_nologs = false` is specified for the jail (`true` by default);
+  - option `journalmatch` is set for the jail or its filter (otherwise it'd be too heavy to allow all auto-jails,
+    even if they have never been foreseen for journal monitoring);
+  (option `skip_if_nologs` will be ignored if we could switch backend to `systemd`)
+* configuration `ignoreip` and fail2ban-client commands `addignoreip`/`delignoreip` extended with `file:...` syntax 
+  to ignore IPs from file-ip-set (containing IP, subnet, dns/fqdn or raw strings); the file would be read lazy on demand,
+  by first ban (and automatically reloaded by update after small latency to avoid expensive stats check on every compare);
+  the entries inside the file can be separated by comma, space or new line with optional comments (text following chars
+  `#` or `;` after space or newline would be ignored up to next newline)
+* `action.d/apprise.conf` - updated to support tagging and other command line args (gh-4141)
+* `action.d/*-ipset.conf`:
+  - parameter `ipsettype` to set type of ipset, e. g. hash:ip, hash:net, etc (gh-3760)
+* `action.d/iptables.conf` - action and few derivatives of it extended to handle multiple chains,
+  e. g. would also accept `chain = INPUT,FORWARD` (gh-3909)
+* `action.d/nftables.conf` (gh-3291):
+  - new parameter `addr_options` for addr-set (default `flags interval\;`, allows to store CIDR or address ranges);
+    can be set to empty value to create simple addresses set (restore previous behavior).
+* `action.d/firewallcmd-rich-*.conf` - fixed incorrect quoting, disabling port variable expansion
+  by substitution of rich rule (gh-3815)
+* `filter.d/dovecot.conf`:
+  - add support for latest Dovecot 2.4 release (gh-4016)
+  - mode `aggressive` covered new variant for `no auth attempts in X secs` with `Login aborted` and `(no_auth_attempts)`
+  - mode `aggressive` extended to match `disconnected during TLS handshake` with `no application protocol` and `no shared cipher`
+* `filter.d/nginx-http-auth.conf`:
+  - extended with `prefregex` to capture content of error only (bypass common prefix and suffix, like server, request, host, referrer);
+  - extended to match PAM authentication failures (gh-4071)
+  - modes `fallback` and `aggressive` extended to match more SSL failures by SSL_do_handshake or SSL_read (gh-4142, gh-2881)
+* `filter.d/nginx-limit-req.conf` - extended to ban hosts failed by limit connection in ngx_http_limit_conn_module (gh-3674, gh-4047)
+* `filter.d/proxmox.conf` - add support to Proxmox Web GUI (gh-2966)
+* `filter.d/openvpn.conf` - new filter and jail for openvpn recognizing failed TLS handshakes (gh-2702)
+* `filter.d/sendmail-reject.conf` - also recognize "Domain of sender address ... does not resolve" (gh-4035)
+* `filter.d/vaultwarden.conf` - new filter and jail for Vaultwarden (gh-3979)
+* `filter.d/xrdp.conf` - new filter for XRDP, an open source RDP server (gh-3254)
+* `fail2ban-regex` extended with new option `-i` or `--invert` to output not-matched lines by `-o` or `--out` (gh-4001)
+
+
 ver. 1.1.0 (2024/04/25) - object-found--norad-59479-cospar-2024-069a--altitude-36267km
 -----------
 
@@ -2602,7 +2720,7 @@ ver. 0.3.1 (2005/03/31) - beta
 
 ver. 0.3.0 (2005/02/24) - beta
 ----------
-- Re-writting of parts of the code in order to handle several log files with
+- Re-writing of parts of the code in order to handle several log files with
   different rules
 - Removed `sshd.py` because it is no more needed
 - Fixed a bug when exiting with IP in the ban list

FILTERS

@@ -232,16 +232,24 @@ the <> at the start so regex should be similar to '^<> error <HOST> is evil$' us
 
 The following general rules apply to regular expressions:
 
-* ensure regexes start with a ^ and are as restrictive as possible. E.g. do not
-  use .* if \d+ is sufficient;
+* ensure regexes are anchored (e. g. start with a ^) and are as restrictive
+  as possible. E.g. do not use catch-alls .+ or .* if \d+ or [^"]* is sufficient.
+  Basically avoid the catch-alls where it is possible, especially non-greedy 
+  catch-alls on RE with many branches or ambiguous matches;
 * use functionality of Python regexes defined in the standard Python re library
-  http://docs.python.org/2/library/re.html;
-* make regular expressions readable (as much as possible). E.g.
-  (?:...) represents a non-capturing regex but (...) is more readable, thus
-  preferred.
+  https://docs.python.org/library/re.html;
+* try to write regular expressions as efficient as possible. E.g. do not write
+  several REs for almost the same messages, just with A or B or C, if they can
+  be matched by single RE using | operator like ...(?:A|B|C)... and order them
+  by their frequency, so A before B and C, if A is more frequent or will match
+  faster;
+* make regular expressions readable (as much as possible), but only if it is
+  justified. E.g. (?:...) represents a non-capturing regex and (...) is more
+  readable, but capturing groups make the RE a bit slower, thus (?:...) may be
+  more preferable.
 
 If you have only a basic knowledge of regular repressions we advise to read
-http://docs.python.org/2/library/re.html first.  It doesn't take long and would
+https://docs.python.org/library/re.html first.  It doesn't take long and would
 remind you e.g. which characters you need to escape and which you don't.
 
 Developing/testing a regex
@@ -265,10 +273,6 @@ Take note of  -l heavydebug  / -l debug  and -v as they might be very useful.
      parts are constrained and different formats depending on configuration or
      less common usages.
 
-.. TIP::
-     For looking through source code - http://sourcecodebrowser.com/ . It has
-     call graphs and can browse different versions.
-
 .. TIP::
      Some applications log spaces at the end. If you are not sure add \s*$ as
      the end part of the regex.
@@ -309,6 +313,8 @@ So more specifically in the [filter] section in jail.conf:
 Submit github pull request (See "Pull Requests" above) for
 github.com/fail2ban/fail2ban containing your great work.
 
+You may also consider https://github.com/fail2ban/fail2ban/wiki/Best-practice
+
 Filter Security
 ===============
 

README.md

@@ -18,7 +18,7 @@ attempts, it cannot eliminate the risk presented by weak authentication.
 Set up services to use only two factor, or public/private authentication
 mechanisms if you really want to protect services.
      
-<img src="http://www.worldipv6launch.org/wp-content/themes/ipv6/downloads/World_IPv6_launch_logo.svg" height="52pt"/> | Since v0.10 fail2ban supports the matching of IPv6 addresses.
+<img src="http://www.worldipv6launch.org/wp-content/themes/ipv6/downloads/World_IPv6_launch_logo.svg" style="height:52pt;"/> | Since v0.10 fail2ban supports the matching of IPv6 addresses.
 ------|------
 
 This README is a quick introduction to Fail2Ban. More documentation, FAQ, and HOWTOs
@@ -29,13 +29,13 @@ and the website: https://www.fail2ban.org
 Installation:
 -------------
 
-Fail2Ban is likely already packaged for your Linux distribution and [can installed with a simple command](https://github.com/fail2ban/fail2ban/wiki/How-to-install-fail2ban-packages).
+Fail2Ban is likely already packaged for your Linux distribution and [can be installed with a simple command](https://github.com/fail2ban/fail2ban/wiki/How-to-install-fail2ban-packages).
 
 If your distribution is not listed, you can install from GitHub:
 
 Required:
 - [Python >= 3.5](https://www.python.org) or [PyPy3](https://pypy.org)
-- python-setuptools, python-distutils (or python3-setuptools) for installation from source
+- python-setuptools (or python3-setuptools) for installation from source
 
 Optional:
 - [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify), may require:
@@ -52,7 +52,7 @@ To install:
     cd fail2ban-master
     sudo python setup.py install
    
-Alternatively, you can clone the source from GitHub to a directory of Your choice, and do the install from there. Pick the correct branch, for example, master or 0.11
+Alternatively, you can clone the source from GitHub to a directory of your choice, and do the install from there. Pick the correct branch, for example, master or 0.11
 
     git clone https://github.com/fail2ban/fail2ban.git
     cd fail2ban

config/action.d/abuseipdb.conf

@@ -80,7 +80,7 @@ actioncheck =
 #          use my (Shaun's) helper PHP script by commenting out the first #actionban
 #          line below, uncommenting the second one, and pointing the URL at
 #          wherever you install the helper script. For the PHP helper script, see
-#          <https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban>
+#          <https://github.com/parseword/fail2ban-abuseipdb/>
 #
 # Tags:    See jail.conf(5) man page
 # Values:  CMD

config/action.d/apprise.conf

@@ -2,7 +2,61 @@
 #
 # Author: Chris Caron <lead2gold@gmail.com>
 #
+# ban & send a notification to one or more of the 120+ services supported by
+# Apprise.
+#  - See https://appriseit.com/services/ for details on what is supported.
+#  - See https://appriseit.com/getting-started/configuration/ for information
+#    on how to prepare an Apprise configuration file.
 #
+#    This plugin requires that Apprise is installed on your system:
+#
+#       pip install apprise
+#
+#    Breakdown:
+#       config   provide a path to an Apprise Config file
+#                The default is /etc/fail2ban/apprise.conf if not provided.
+#                Both YAML and TEXT formats are supported.
+#                You can even point your configuration to an Apprise API
+#                endpoint.
+#
+#       args     Provide additional arguments to support the Apprise CLI.
+#                See https://appriseit.com/cli/usage/ for additional options.
+#                the --tag (-g) is incredibly useful for integrating with
+#                fail2ban as you can exclusively have it target specific
+#                notifications this way.
+#
+#    Config Example #1: Simple
+#      1. Create a /etc/fail2ban/apprise.conf
+#         ```
+#         # /etc/fail2ban/apprise.conf
+#         fail2ban=mailto://user:pass@example.com
+#         ```
+#      2  In jail:
+#         ```
+#         action = %(action_)s
+#                  apprise[args='--tag fail2ban']
+#         ```
+#
+#    Config Example #2: YAML an Custom path
+#      1. Create a /etc/fail2ban/apprise.conf
+#         ```
+#         # /etc/fail2ban/apprise.yaml
+#         urls:
+#           - mailto://user:pass@example.com:
+#              tags: f2b
+#         ```
+#      2. In jail:
+#         ```
+#         action = %(action_)s
+#                  apprise[config='/etc/fail2ban/apprise.yaml',args='--tag f2b']
+#         ```
+#
+#    Config Example #3: Apprise API
+#      1. In jail:
+#         ```
+#         action = %(action_)s
+#                  apprise[config='http://apprise.example.ca/get/mykey',args='-g f2b']
+#         ```
 
 [Definition]
 
@@ -10,7 +64,7 @@
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
 #
-actionstart = printf %%b "The jail <name> as been started successfully." | <apprise> -t "[Fail2Ban] <name>: started on `uname -n`"
+actionstart = printf %%b "The jail <name> has been started successfully." | <apprise> -t "[Fail2Ban] <name>: started on `uname -n`"
 
 # Option:  actionstop
 # Notes.:  command executed once at the end of Fail2Ban
@@ -45,5 +99,9 @@ actionunban =
 # Define location of the default apprise configuration file to use
 #
 config = /etc/fail2ban/apprise.conf
+
+# Support passing in arguments for example: "-g fail2ban"
 #
-apprise = apprise -c "<config>"
+args =
+#
+apprise = apprise -c "<config>" <args>

config/action.d/blocklist_de.conf

@@ -30,6 +30,9 @@
 
 [Definition]
 
+# bypass reporting of restored (already reported) tickets:
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD

config/action.d/cloudflare-token.conf

@@ -50,7 +50,7 @@ actionban = curl -s -X POST "<_cf_api_url>" \
 #          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
-actionunban = id=$(curl -s -X GET "<_cf_api_url>" \
+actionunban = id=$(curl -s -G -X GET "<_cf_api_url>" \
               --data-urlencode "mode=<cfmode>" --data-urlencode "notes=<notes>" --data-urlencode "configuration.target=<cftarget>" --data-urlencode "configuration.value=<ip>" \
               <_cf_api_prms> \
                   | awk -F"[,:}]" '{for(i=1;i<=NF;i++){if($i~/'id'\042/){print $(i+1)}}}' \

config/action.d/complain.conf

@@ -16,7 +16,7 @@
 #
 # Please do not use this action unless you are certain that fail2ban
 # does not result in "false positives" for your deployment.  False
-# positive reports could serve a mis-favor to the original cause by
+# positive reports could serve a misfavor to the original cause by
 # flooding corresponding contact addresses, and complicating the work
 # of administration personnel responsible for handling (verified) legit
 # complains.

config/action.d/csf.conf

@@ -0,0 +1,26 @@
+# Fail2Ban configuration file
+# http://configserver.com/cp/csf.html
+#
+# Note: CSF doesn't play nicely with other actions. It has been observed to
+# remove bans created by other iptables based actions. If you are going to use
+# this action, use it for all of your jails.
+#
+# DON'T MIX CSF and other IPTABLES based actions
+
+[Definition]
+
+actionstart = 
+actionstop = 
+actioncheck = 
+actionban = csf --deny <ip> "banned by Fail2Ban <name>"
+actionunban = csf --denyrm <ip>
+
+[Init]
+
+# Name used in CSF configuration
+#
+name = default
+
+# DEV NOTES:
+#
+# based on apf.conf by Mark McKinstry

config/action.d/firewallcmd-ipset.conf

@@ -18,36 +18,36 @@ before = firewallcmd-common.conf
 
 [Definition]
 
-actionstart = <ipstype_<ipsettype>/actionstart>
+actionstart = <ipsbackend_<ipsetbackend>/actionstart>
               firewall-cmd --direct --add-rule <family> filter <chain> 0 <actiontype> -m set --match-set <ipmset> src -j <blocktype>
 
-actionflush = <ipstype_<ipsettype>/actionflush>
+actionflush = <ipsbackend_<ipsetbackend>/actionflush>
 
 actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 <actiontype> -m set --match-set <ipmset> src -j <blocktype>
              <actionflush>
-             <ipstype_<ipsettype>/actionstop>
+             <ipsbackend_<ipsetbackend>/actionstop>
 
-actionban = <ipstype_<ipsettype>/actionban>
+actionban = <ipsbackend_<ipsetbackend>/actionban>
 
 # actionprolong = %(actionban)s
 
-actionunban = <ipstype_<ipsettype>/actionunban>
+actionunban = <ipsbackend_<ipsetbackend>/actionunban>
 
-[ipstype_ipset]
+[ipsbackend_ipset]
 
-actionstart = ipset -exist create <ipmset> hash:ip timeout <default-ipsettime> maxelem <maxelem> <familyopt>
+actionstart = ipset -exist create <ipmset> <ipsettype> timeout <default-ipsettime> maxelem <maxelem> <familyopt>
 
 actionflush = ipset flush <ipmset>
 
-actionstop = ipset destroy <ipmset>
+actionstop = ipset destroy <ipmset> 2>/dev/null || { sleep 1; ipset destroy <ipmset>; }
 
 actionban = ipset -exist add <ipmset> <ip> timeout <ipsettime>
 
 actionunban = ipset -exist del <ipmset> <ip>
 
-[ipstype_firewalld]
+[ipsbackend_firewalld]
 
-actionstart = firewall-cmd --direct --new-ipset=<ipmset> --type=hash:ip --option=timeout=<default-ipsettime> --option=maxelem=<maxelem> <firewalld_familyopt>
+actionstart = firewall-cmd --direct --new-ipset=<ipmset> --type=<ipsettype> --option=timeout=<default-ipsettime> --option=maxelem=<maxelem> <firewalld_familyopt>
 
 # TODO: there doesn't seem to be an explicit way to invoke the ipset flush function using firewall-cmd
 actionflush = 
@@ -60,6 +60,11 @@ actionunban = firewall-cmd --ipset=<ipmset> --remove-entry=<ip>
 
 [Init]
 
+# Option: ipsettype
+# Notes:  specifies type of set, see `man --pager='less -p "^SET TYPES"' ipset` for details
+# Values: hash:ip, hash:net, etc... Default: hash:ip
+ipsettype = hash:ip
+
 # Option:  chain
 # Notes    specifies the iptables chain to which the fail2ban rules should be
 #          added
@@ -87,11 +92,11 @@ maxelem = 65536
 # banaction = %(known/banaction)s[ipsettime='<timeout-bantime>']
 timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)
 
-# Option: ipsettype
-# Notes.: defines type of ipset used for match-set (firewalld or ipset)
+# Option: ipsetbackend
+# Notes.: defines the backend of ipset used for match-set (firewalld or ipset)
 # Values: firewalld or ipset
 # Default: ipset
-ipsettype = ipset
+ipsetbackend = ipset
 
 # Option: actiontype
 # Notes.: defines additions to the blocking rule

config/action.d/firewallcmd-rich-rules.conf

@@ -35,7 +35,7 @@ actioncheck =
 #
 # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp 
 
-fwcmd_rich_rule = rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' %(rich-suffix)s
+fwcmd_rich_rule = rule family=\"<family>\" source address=\"<ip>\" port port=\"$p\" protocol=\"<protocol>\" %(rich-suffix)s
 
 actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done
 	   

config/action.d/iptables-ipset.conf

@@ -24,7 +24,7 @@ before = iptables.conf
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
-actionstart = ipset -exist create <ipmset> hash:ip timeout <default-ipsettime> maxelem <maxelem> <familyopt>
+actionstart = ipset -exist create <ipmset> <ipsettype> timeout <default-ipsettime> maxelem <maxelem> <familyopt>
               <_ipt_add_rules>
 
 # Option:  actionflush
@@ -39,7 +39,7 @@ actionflush = ipset flush <ipmset>
 #
 actionstop = <_ipt_del_rules>
              <actionflush>
-             ipset destroy <ipmset>
+             ipset destroy <ipmset> 2>/dev/null || { sleep 1; ipset destroy <ipmset>; }
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
@@ -66,6 +66,11 @@ rule-jump = -m set --match-set <ipmset> src -j <blocktype>
 
 [Init]
 
+# Option: ipsettype
+# Notes:  specifies type of set, see `man --pager='less -p "^SET TYPES"' ipset` for details
+# Values: hash:ip, hash:net, etc... Default: hash:ip
+ipsettype = hash:ip
+
 # Option: default-ipsettime
 # Notes:  specifies default timeout in seconds (handled default ipset timeout only)
 # Values:  [ NUM ]  Default: 0 (no timeout, managed by fail2ban by unban)

config/action.d/iptables-xt_recent-echo.conf

@@ -12,8 +12,9 @@ before = iptables.conf
 [Definition]
 
 _ipt_chain_rule = -m recent --update --seconds 3600 --name <iptname> -j <blocktype>
-_ipt_for_proto-iter =
-_ipt_for_proto-done =
+_ipt_check_rule = <iptables> -C <chain> %(_ipt_chain_rule)s
+_ipt-iter =
+_ipt-done =
 
 # Option:  actionstart
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
@@ -60,7 +61,7 @@ actionstop = echo / > /proc/net/xt_recent/<iptname>
 # Notes.:  command executed as invariant check (error by ban)
 # Values:  CMD
 #
-actioncheck = { <iptables> -C <chain> %(_ipt_chain_rule)s; } && test -e /proc/net/xt_recent/<iptname>
+actioncheck = { %(_ipt_check_rule)s >/dev/null 2>&1; } && test -e /proc/net/xt_recent/<iptname>
 
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the

config/action.d/iptables.conf

@@ -64,23 +64,23 @@ rule-jump = -j <_ipt_rule_target>
 
 # Several capabilities used internally:
 
-_ipt_for_proto-iter = for proto in $(echo '<protocol>' | sed 's/,/ /g'); do
-_ipt_for_proto-done = done
+_ipt-iter = for chain in $(echo '<chain>' | sed 's/,/ /g'); do for proto in $(echo '<protocol>' | sed 's/,/ /g'); do
+_ipt-done = done; done
 
-_ipt_add_rules = <_ipt_for_proto-iter>
-              { %(_ipt_check_rule)s >/dev/null 2>&1; } || { <iptables> -I <chain> %(_ipt_chain_rule)s; }
-              <_ipt_for_proto-done>
+_ipt_add_rules = <_ipt-iter>
+              { %(_ipt_check_rule)s >/dev/null 2>&1; } || { <iptables> -I $chain %(_ipt_chain_rule)s; }
+              <_ipt-done>
 
-_ipt_del_rules = <_ipt_for_proto-iter>
-              <iptables> -D <chain> %(_ipt_chain_rule)s
-              <_ipt_for_proto-done>
+_ipt_del_rules = <_ipt-iter>
+              <iptables> -D $chain %(_ipt_chain_rule)s
+              <_ipt-done>
 
-_ipt_check_rules = <_ipt_for_proto-iter>
+_ipt_check_rules = <_ipt-iter>
               %(_ipt_check_rule)s
-              <_ipt_for_proto-done>
+              <_ipt-done>
 
 _ipt_chain_rule = <pre-rule><ipt_<type>/_chain_rule>
-_ipt_check_rule = <iptables> -C <chain> %(_ipt_chain_rule)s
+_ipt_check_rule = <iptables> -C $chain %(_ipt_chain_rule)s
 _ipt_rule_target = f2b-<name>
 
 [ipt_oneport]
@@ -99,8 +99,9 @@ _chain_rule = -p $proto <rule-jump>
 [Init]
 
 # Option:  chain
-# Notes    specifies the iptables chain to which the Fail2Ban rules should be
-#          added
+# Notes    specifies the iptables chains to which the Fail2Ban rules should be
+#          added. May be a single chain (e.g. INPUT) or a comma separated list
+#          (e.g. INPUT, FORWARD)
 # Values:  STRING  Default: INPUT
 chain = INPUT
 
@@ -135,7 +136,7 @@ returntype = RETURN
 
 # Option:  lockingopt
 # Notes.:  Option was introduced to iptables to prevent multiple instances from
-#          running concurrently and causing irratic behavior.  -w was introduced
+#          running concurrently and causing erratic behavior.  -w was introduced
 #          in iptables 1.4.20, so might be absent on older systems
 #          See https://github.com/fail2ban/fail2ban/issues/1122
 # Values:  STRING

config/action.d/ipthreat.conf

@@ -15,7 +15,7 @@
 # Reporting an IP is a serious action. Make sure that it is legit.
 # Consider using this action only for:
 #   * IP that has been banned more than once
-#   * High max retry to avoid user mis-typing password
+#   * High max retry to avoid user mistyping password
 #   * Filters that are unlikely to be human error
 #
 # Example:

config/action.d/nftables.conf

@@ -53,9 +53,9 @@ _nft_for_proto-multiport-iter = for proto in $(echo '<protocol>' | sed 's/,/ /g'
 _nft_for_proto-multiport-done = done
 
 _nft_list = <nftables> -a list chain <table_family> <table> <chain>
-_nft_get_handle_id = grep -oP '@<addr_set>\s+.*\s+\Khandle\s+(\d+)$'
+_nft_get_handle_id = sed -nE 's/.*@<addr_set>\s+.*\s+\#\s*(handle\s+[0-9]+)$/\1/p'
 
-_nft_add_set = <nftables> add set <table_family> <table> <addr_set> \{ type <addr_type>\; \}
+_nft_add_set = <nftables> add set <table_family> <table> <addr_set> \{ type <addr_type>\;<addr_options> \}
               <_nft_for_proto-<type>-iter>
               <nftables> add rule <table_family> <table> <chain> %(rule_stat)s
               <_nft_for_proto-<type>-done>
@@ -67,7 +67,7 @@ _nft_del_set = { %(_nft_list)s | %(_nft_get_handle_id)s; } | while read -r hdl;
 # Notes.:  command executed after the stop in order to delete table (it checks that no sets are available):
 # Values:  CMD
 #
-_nft_shutdown_table = { <nftables> list table <table_family> <table> | grep -qP '^\s+set\s+'; } || {
+_nft_shutdown_table = { <nftables> list table <table_family> <table> | grep -qE '^\s+set\s+'; } || {
                         <nftables> delete table <table_family> <table>
                       }
 
@@ -97,7 +97,7 @@ actionstop = %(_nft_del_set)s
              <_nft_shutdown_table>
 
 # Option:  actioncheck
-# Notes.:  command executed once before each actionban command
+# Notes.:  command executed once in error case by other command (during the check/restore sane environment process)
 # Values:  CMD
 #
 actioncheck = <nftables> list chain <table_family> <table> <chain> | grep -q '@<addr_set>[ \t]'
@@ -197,6 +197,11 @@ addr_set = addr-set-<name>
 # Values: [ ip | ip6 ]
 addr_family = ip
 
+# Option: addr_options
+# Notes: Additional options for the addr-set, by default allows to store CIDR or address ranges.
+#        Can be set to empty value to create simple addresses set.
+addr_options = <sp>flags interval\;
+
 [Init?family=inet6]
 addr_family = ip6
 addr_type = ipv6_addr

config/action.d/shorewall-ipset-proto6.conf

@@ -51,7 +51,7 @@
 # Values:  CMD
 #
 actionstart = if ! ipset -quiet -name list f2b-<name> >/dev/null;
-              then ipset -quiet -exist create f2b-<name> hash:ip timeout <default-ipsettime> maxelem <maxelem>;
+              then ipset -quiet -exist create f2b-<name> <ipsettype> timeout <default-ipsettime> maxelem <maxelem>;
               fi
 
 # Option:  actionstop
@@ -94,6 +94,11 @@ timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0)
 
 [Init]
 
+# Option: ipsettype
+# Notes:  specifies type of set, see `man --pager='less -p "^SET TYPES"' ipset` for details
+# Values: hash:ip, hash:net, etc... Default: hash:ip
+ipsettype = hash:ip
+
 # Option: maxelem
 # Notes:  maximal number of elements which can be stored in the ipset
 #         You may want to increase this for long-duration/high-volume jails

config/action.d/smtp.py

@@ -19,7 +19,8 @@
 
 import socket
 import smtplib
-from email.mime.text import MIMEText
+import email.policy
+from email.message import EmailMessage
 from email.utils import formatdate, formataddr
 
 from fail2ban.server.actions import ActionBase, CallingMap
@@ -151,7 +152,8 @@ class SMTPAction(ActionBase):
 			See Python `smtplib` for full list of other possible
 			exceptions.
 		"""
-		msg = MIMEText(text)
+		msg = EmailMessage(policy=email.policy.SMTP)
+		msg.set_content(text)
 		msg['Subject'] = subject
 		msg['From'] = formataddr((self.fromname, self.fromaddr))
 		msg['To'] = self.toaddr

config/action.d/ufw.conf

@@ -44,7 +44,7 @@ _kill_conntrack = conntrack -D -s "<ip>"
 
 # Option: kill
 # Notes.: can be used to specify custom killing feature, by default depending on option kill-mode
-# Examples: banaction = ufw[kill='ss -K "( sport = :http || sport = :https )" dst "[<ip>]"']
+# Examples: banaction = ufw[kill='ss -K "dst = [<ip>] && ( sport = :http || sport = :https )"']
 #           banaction = ufw[kill='cutter "<ip>"']
 kill = <_kill_<kill-mode>>
 

config/action.d/xarf-login-attack.conf

@@ -44,7 +44,13 @@ actioncheck =
 actionban = oifs=${IFS};
             RESOLVER_ADDR="%(addr_resolver)s"
             if [ "<debug>" -gt 0 ]; then echo "try to resolve $RESOLVER_ADDR"; fi
-            ADDRESSES=$(dig +short -t txt -q $RESOLVER_ADDR | tr -d '"')
+            ADDRESSES=$(dig +short -t txt -q $RESOLVER_ADDR | grep -v ';;' | tr -d '"')
+            if [ "<debug>" -gt 0 ]; then echo "returned address $ADDRESSES"; fi
+            if [ -z "$ADDRESSES" ]; then
+               echo "address for $RESOLVER_ADDR cannot be found or timeout from dig";
+               if [ "<debug>" -gt 0 ]; then exit 1; fi
+               exit 0
+            fi
             IFS=,; ADDRESSES=$(echo $ADDRESSES)
             IFS=${oifs}
             IP=<ip>
@@ -55,13 +61,11 @@ actionban = oifs=${IFS};
             TLP=<tlp>
             PORT=<port>
             DATE=`LC_ALL=C date --date=@<time> +"%%a, %%d %%h %%Y %%T %%z"`
-            if [ ! -z "$ADDRESSES" ]; then
-                oifs=${IFS}; IFS=,; ADDRESSES=$(echo $ADDRESSES)
-                IFS=${oifs}
-                (printf -- %%b "<header>\n<message>\n<report>\n\n";
-                 date '+Note: Local timezone is %%z (%%Z)';
-                 printf -- %%b "\n<ipmatches>\n\n<footer>") | <mailcmd> <mailargs> $ADDRESSES
-            fi
+            oifs=${IFS}; IFS=,; ADDRESSES=$(echo $ADDRESSES)
+            IFS=${oifs}
+            (printf -- %%b "<header>\n<message>\n<report>\n\n";
+             date '+Note: Local timezone is %%z (%%Z)';
+             printf -- %%b "\n<ipmatches>\n\n<footer>") | <mailcmd> <mailargs> $ADDRESSES
 
 actionunban =
 

config/filter.d/apache-badbots.conf

@@ -10,7 +10,8 @@
 badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider|(?:Mozilla/\d+\.\d+ )?Jorgee
 badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 \+http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots, \+http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00
 
-failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$
+vhostpref = (?:[\w]+[\w\-\.]*\s+)?
+failregex = ^\s*%(vhostpref)s<ADDR> [^"]*"[A-Z]{3,10} [^"]+" \d+ \d+ "[^"]+" "[^"]*(?:%(badbots)s|%(badbotscustom)s)"$
 
 ignoreregex =
 

config/filter.d/apache-fakegooglebot.conf

@@ -2,7 +2,8 @@
 
 [Definition]
 
-failregex = ^\s*<HOST> \S+ \S+(?: \S+)?\s+\S+ "[A-Z]+ /\S* [^"]*" \d+ \d+ \"[^"]*\" "[^"]*\bGooglebot/[^"]*"
+vhostpref = (?:[\w]+[\w\-\.]*\s+)?
+failregex = ^\s*%(vhostpref)s<ADDR> [^"]*"[A-Z]{3,10} /\S* [^"]*" \d+ \d+ "[^"]*" "[^"]*\bGooglebot/[^"]*"
 
 ignoreregex =
 

config/filter.d/apache-noscript.conf

@@ -19,11 +19,10 @@ before = apache-common.conf
 
 script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl|\bcgi-bin/)
 
-prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)|2811): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)|2811): )?(?=(?:[Ff]ile|[Ss]cript|[Gg]ot error|stderr from) )<F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^(?:does not exist|not found or unable to stat): <script>\b
-            ^'<script>\S*' not found or unable to stat
-            ^error '[Pp]rimary script unknown(?:\\n)?'
+failregex = ^(?:(?:[Ff]ile does not exist|[Ss]cript not found or unable to stat): <script>\b|[Gg]ot error '[Pp]rimary script unknown\b)
+            ^(?:stderr from |script (?P<_q>'))<script>\S*(?(_q)'|) (?:script )?(?:does not exist|not found or unable to stat)
 
 ignoreregex = 
 

config/filter.d/apache-overflows.conf

@@ -8,7 +8,7 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s (?:(?:AH001[23][456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)
+failregex = ^%(_apache_error_client)s (?:(?:AH(?:001[23][456]|10244): )?[Ii]nvalid (method|URI)\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)
 
 ignoreregex =
 

config/filter.d/asterisk.conf

@@ -21,13 +21,13 @@ log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])?:? [^:]+
 prefregex = ^%(__prefix_line)s%(log_prefix)s <F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^Registration from '[^']*' failed for '<HOST>(:\d+)?' - (?:Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$
-            ^Call from '[^']*' \((?:(?:TCP|UDP):)?<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
+            ^Call (?:from '[^']*' )?\((?:(?:TCP|UDP):)?<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context
             ^(?:Host )?<HOST> (?:failed (?:to authenticate\b|MD5 authentication\b)|tried to authenticate with nonexistent user\b)
             ^No registration for peer '[^']*' \(from <HOST>\)$
             ^hacking attempt detected '<HOST>'$
             ^SecurityEvent="(?:FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)"(?:(?:,(?!RemoteAddress=)\w+="[^"]*")*|.*?),RemoteAddress="IPV[46]/[^/"]+/<HOST>/\d+"(?:,(?!RemoteAddress=)\w+="[^"]*")*$
             ^"Rejecting unknown SIP connection from <HOST>(?::\d+)?"$
-            ^Request (?:'[^']*' )?from '(?:[^']*|.*?)' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$
+            ^Request (?:'[^']*' )?from '(?:[^']*|.*?)' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\b[^']*$
 
 # FreePBX (todo: make optional in v.0.10):
 #            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )[^:]+: Friendly Scanner from <HOST>$

config/filter.d/dovecot.conf

@@ -16,12 +16,12 @@ _bypass_reject_reason = (?:: (?:\w+\([^\):]*\) \w+|[^\(]+))*
 prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap|managesieve|submission)-login: )?(?:Info: )?%(_auth_worker_info)s<F-CONTENT>.+</F-CONTENT>$
 
 failregex = ^authentication failure; logname=<F-ALT_USER1>\S*</F-ALT_USER1> uid=\S* euid=\S* tty=dovecot ruser=<F-USER>\S*</F-USER> rhost=<HOST>(?:\s+user=<F-ALT_USER>\S*</F-ALT_USER>)?\s*$
-            ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)%(_bypass_reject_reason)s \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+            ^(?:Login aborted|Aborted login|Disconnected|Remote closed connection|Client has quit the connection)%(_bypass_reject_reason)s \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\)[^:]*:(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
             ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \([Pp]assword mismatch\?\)|Permission denied)\s*$
             ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:[Uu]nknown user|[Ii]nvalid credentials|[Pp]assword mismatch)
             <mdre-<mode>>
 
-mdre-aggressive = ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)%(_bypass_reject_reason)s \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+mdre-aggressive = ^(?:Login aborted|Aborted login|Disconnected|Remote closed connection|Client has quit the connection)%(_bypass_reject_reason)s \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,|disconnected during TLS handshake)(?: (?:in|waited) \d+ secs)?\)[^:]*:(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
 
 mdre-normal = 
 
@@ -43,6 +43,7 @@ datepattern = {^LN-BEG}TAI64N
 # DEV Notes:
 # * the first regex is essentially a copy of pam-generic.conf
 # * Probably doesn't do dovecot sql/ldap backends properly (resolved in edit 21/03/2016)
+# * Dovecot version 2.4 changed event log structure, line prior needed to maintain 2.3 support
 #
 # Author: Martin Waschbuesch
 #         Daniel Black (rewrote with begin and end anchors)

config/filter.d/dropbear.conf

@@ -23,14 +23,17 @@ before = common.conf
 
 _daemon = dropbear
 
-prefregex = ^%(__prefix_line)s<F-CONTENT>(?:[Ll]ogin|[Bb]ad|[Ee]xit).+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s(?:\[\d+\] \w{2,3} [\d:\s]+)?<F-CONTENT>(?:[Ll]ogin|[Bb]ad|[Ee]xit).+</F-CONTENT>$
 
-failregex = ^[Ll]ogin attempt for nonexistent user ('.*' )?from <HOST>:\d+$
-            ^[Bb]ad (PAM )?password attempt for .+ from <HOST>(:\d+)?$
-            ^[Ee]xit before auth \(user '.+', \d+ fails\): Max auth tries reached - user '.+' from <HOST>:\d+\s*$
+failregex = ^[Ll]ogin attempt for nonexistent user (?:'<F-USER>.*</F-USER>' )?from <HOST>:\d+$
+            ^[Bb]ad (?:PAM )?password attempt for '<F-USER>.+</F-USER>' from <HOST>(?::\d+)?$
+            ^[Ee]xit before auth from \<?<ADDR>:\d+\>?: (?:\([^\)]*\): )?Max auth tries reached - user '<F-USER>.+</F-USER>'\s*$
+            ^[Ee]xit before auth \(user '.+', \d+ fails\): Max auth tries reached - user '<F-USER>.+</F-USER>' from <HOST>:\d+\s*$
 
 ignoreregex = 
 
+journalmatch = _SYSTEMD_UNIT=dropbear.service + _COMM=dropbear
+
 # DEV Notes:
 #
 # The first two regexs here match the unmodified dropbear messages. It isn't

config/filter.d/exim.conf

@@ -15,18 +15,21 @@ before = exim-common.conf
 
 prefregex = ^%(__prefix_line)s<F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^%(host_info)s sender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
-            ^\s?\w+ authenticator failed for%(host_info)s: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$
-            ^%(host_info)s rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
+failregex = ^\s?\w+ authenticator failed for%(host_info)s: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$
+            <mdre-<mode>>
+
+mdre-more = ^%(host_info)s sender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
+            ^%(host_info)s rejected RCPT (?:<F-RCPT>[^@]+@\S+</F-RCPT>:)?
             ^\s?SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+")%(host_info)s (?:next )?input=".*"\s*$
             ^\s?SMTP call from%(host_info)s dropped: too many (?:(?:nonmail|unrecognized) commands|syntax or protocol errors)
             ^\s?SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?"%(host_info)s [A-Z]+ (?:command used when not advertised|authentication mechanism not supported)\s*$
             ^\s?no MAIL in SMTP connection from%(host_info)s
             ^\s?(?:[\w\-]+ )?SMTP connection from%(host_info)s closed by DROP in ACL\s*$
-            <mdre-<mode>>
 
-mdre-aggressive = ^\s?no host name found for IP address <ADDR>$
+mdre-aggressive = %(mdre-more)s
+                  ^\s?no host name found for IP address <ADDR>$
                   ^\s?no IP address found for host \S+ \(during SMTP connection from%(host_info)s\)$
+                  ^%(host_info)s dropped by '[^']+' ACL:
 
 mdre-normal = 
 

config/filter.d/freeswitch.conf

@@ -29,9 +29,9 @@ _daemon = freeswitch
 mode = extra
               
 # Prefix contains common prefix line (server, daemon, etc.) and 2 datetimes if used systemd backend
-_pref_line = ^%(__prefix_line)s(?:(?:\d+-)?\d+-\d+ \d+:\d+:\d+\.\d+)?
+_pref_line = ^%(__prefix_line)s[^\[]*
 
-prefregex = ^%(_pref_line)s \[WARN(?:ING)?\](?: \[SOFIA\])? \[?sofia_reg\.c:\d+\]? <F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(_pref_line)s\s*\[WARN(?:ING)?\](?: \[SOFIA\])? \[?sofia_reg\.c:\d+\]? <F-CONTENT>.+</F-CONTENT>$
 
 cmnfailre = ^Can't find user \[[^@]+@[^\]]+\] from <HOST>$
 

config/filter.d/froxlor-auth.conf

@@ -1,11 +1,15 @@
 # Fail2Ban configuration file to block repeated failed login attempts to Frolor installation(s)
 # 
 # Froxlor needs to log to Syslog User (e.g. /var/log/user.log) with one of the following messages
-# <syslog prefix> Froxlor: [Login Action <HOST>] Unknown user '<USER>' tried to login.
-# <syslog prefix> Froxlor: [Login Action <HOST>] User '<USER>' tried to login with wrong password.
+# - for type=2
+#     <syslog prefix> froxlor[1-6]: froxlor.WARNING: Unknown user tried to login. {"source":"login","action":"50","user":"<ADDR>"} []
+#     <syslog prefix> froxlor[1-6]: froxlor.WARNING: User tried to login with wrong password. {"source":"login","action":"50","user":"<ADDR>"} []
+# - for type=1:
+#     <syslog prefix> Froxlor: [Login Action <ADDR>] Unknown user '<USER>' tried to login.
+#     <syslog prefix> Froxlor: [Login Action <ADDR>] User '<USER>' tried to login with wrong password.
 #
 # Author: Joern Muehlencord
-#
+# Modified: Para-do-x™️ - Andreas Duennwald
 
 [INCLUDES]
 
@@ -13,28 +17,18 @@
 # common.local
 before = common.conf
 
+[DEFAULT]
+_daemon = [Ff]roxlor
+
+_re = (?:Unknown )?[uU]ser(?: '<F-USER>(?:\S*|[^']*)</F-USER>')? tried to login(?: with wrong password)?\.
+
+[type1]
+failregex = ^%(__prefix_line)s\[Login Action <ADDR>\] %(_re)s$
+
+[type2]
+failregex = ^%(__prefix_line)sfroxlor\.WARNING: %(_re)s \{(?:"[^"]+":"[^"]*",\s*){,5}"user":"<ADDR>"\} \[\]$
 
 [Definition]
-
-_daemon = Froxlor
-
-# Option:  failregex
-# Notes.:  regex to match the password failures messages in the logfile. The
-#          host must be matched by a group named "host". The tag "<HOST>" can
-#          be used for standard IP/hostname matching and is only an alias for
-#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
-# Values:  TEXT
-#
-
-prefregex = ^%(__prefix_line)s\[Login Action <HOST>\] <F-CONTENT>.+</F-CONTENT>$
-
-failregex = ^Unknown user \S* tried to login.$
-            ^User \S* tried to login with wrong password.$
-
-
-# Option:  ignoreregex
-# Notes.:  regex to ignore. If this regex matches, the line is ignored.
-# Values:  TEXT
-#
+type = 2
+failregex = <type<type>/failregex>
 ignoreregex =
-

config/filter.d/lighttpd-auth.conf

@@ -3,8 +3,8 @@
 
 [Definition]
 
-failregex = ^\s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+<F-USER>(?:\S+|.*?)</F-USER>\s*|digest: auth failed(?: for\s+<F-ALT_USER>(?:\S+|.*?)</F-ALT_USER>\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: <HOST>\s*$
+failregex = ^[^\)]*\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+<F-USER>(?:\S+|.*?)</F-USER>\s*|digest: auth failed(?: for\s+<F-ALT_USER>(?:\S+|.*?)</F-ALT_USER>\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: <HOST>\s*$
 
-ignoreregex = 
+ignoreregex =
 
-# Author: Francois Boulogne <fboulogne@april.org>
+# Authors: Francois Boulogne <fboulogne@april.org>, Lucian Maly <lmaly@redhat.com>

config/filter.d/mysqld-auth.conf

@@ -1,9 +1,10 @@
 # Fail2Ban filter for unsuccessful MySQL authentication attempts
 #
 #
-# To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld]:
-# log-error=/var/log/mysqld.log
-# log-warnings = 2
+# To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld],
+# `log_error_verbosity` system variable set to 3 (`log-warnings = 2` for older versions),
+# and check whether `log_error` (or `log-error`) system variable would match the `logpath` of fail2ban
+# (see https://dev.mysql.com/doc/refman/en/communication-errors.html)
 #
 # If using mysql syslog [mysql_safe] has syslog in /etc/my.cnf
 

config/filter.d/nginx-http-auth.conf

@@ -8,16 +8,18 @@ before = nginx-error-common.conf
 
 mode = normal
 
-__err_type = <_ertp-<mode>>
+__err_type = (?:error|crit)
 
-_ertp-auth = error
-mdre-auth = ^%(__prefix_line)suser "(?:[^"]+|.*?)":? (?:password mismatch|was not found in "[^\"]*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(?:, referrer: "\S+")?\s*$
-_ertp-fallback = crit
-mdre-fallback = ^%(__prefix_line)sSSL_do_handshake\(\) failed \(SSL: error:\S+(?: \S+){1,3} too (?:long|short)\)[^,]*, client: <HOST>
+__suffix_line = , client: <ADDR>(?:, (?:server|request|host|referrer): (?:"[^"]*"|\S*)){0,4}
+
+prefregex = ^%(__prefix_line)s<F-CONTENT>.*</F-CONTENT>%(__suffix_line)s\s*$
+
+mdre-auth = ^user "<F-USER>(?:[^"]+|.*?)</F-USER>":? (?:password mismatch|was not found in "[^\"]*")$
+            ^(?:PAM: )?user '<F-USER>(?:[^']+|.*?)</F-USER>' - not authenticated: Authentication failure$
+mdre-fallback = ^SSL_(?:do_handshake|read)\(\) failed \(SSL: error:\S+(?: \S+){1,3}[^\)]*\)[^,]*
 
-_ertp-normal = %(_ertp-auth)s
 mdre-normal = %(mdre-auth)s
-_ertp-aggressive = (?:%(_ertp-auth)s|%(_ertp-fallback)s)
+
 mdre-aggressive = %(mdre-auth)s
                   %(mdre-fallback)s
 

config/filter.d/nginx-limit-req.conf

@@ -46,7 +46,7 @@ __err_type = [a-z]+
 # failregex = ^%(__prefix_line)slimiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$
 
 # Shortly, much faster and stable version of regexp:
-failregex = ^%(__prefix_line)slimiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: <HOST>,
+failregex = ^%(__prefix_line)s(?:limiting|delaying) (?:request|connection)s?(?:, excess: [\d\.]+,?)? by zone "(?:%(ngx_limit_req_zones)s)", client: <ADDR>,
 
 ignoreregex = 
 

config/filter.d/openvpn.conf

@@ -0,0 +1,13 @@
+# Fail2Ban filter for openvpn server
+# Detecting wrong TLS handshakes
+# typically logged in /var/log/syslog
+# Author: Philipp Burndorfer
+
+[INCLUDES]
+before = common.conf
+
+[Definition]
+_daemon = ovpn-server\d*
+
+failregex = ^%(__prefix_line)s<HOST>:\d{4,5} (?:TLS Auth Error:|VERIFY ERROR:|TLS Error: TLS handshake failed\b|SIGUSR1\[soft,connection-reset\] received\b)
+            ^%(__prefix_line)sTLS Error: cannot locate HMAC in incoming packet from \[AF_INET\]\s*<HOST>:\d{4,5}

config/filter.d/postfix.conf

@@ -10,50 +10,63 @@ before = common.conf
 
 [Definition]
 
-_daemon = postfix(-\w+)?/[^/\[:\s]+(?:/smtp[ds])?
+_daemon = postfix\b([^\[\s]+)?
+# optional port:
 _port = (?::\d+)?
-_pref = [A-Z]{4}
+# optional prefix like `NOQUEUE: ` or `00ADB3C0899: ` etc...
+_pref = (?:\w+: )?
+# SMTP commands like RCPT etc
+_cmd = [A-Z]{4,}
 
-prefregex = ^%(__prefix_line)s<mdpr-<mode>> <F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s%(_pref)s<mdpr-<mode>> <F-CONTENT>.+</F-CONTENT>$
 
 # Extended RE for normal mode to match reject by unknown users or undeliverable address, can be set to empty to avoid this:
 exre-user = |[Uu](?:ser unknown|ndeliverable address)  ; pragma: codespell-ignore
 
-mdpr-normal = (?:\w+: (?:milter-)?reject:|(?:improper command pipelining|too many errors) after \S+)
-mdre-normal=^%(_pref)s from [^[]*\[<HOST>\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|Access denied|(?:Client host|Command|Data command) rejected|Relay access denied|Malformed DNS server reply|(?:Host|Domain) not found|need fully-qualified hostname|match%(exre-user)s)\b
+mdpr-normal = (?:(?:milter-)?reject:|(?:improper command pipelining|too many errors) after \S+)
+mdre-normal=^%(_cmd)s from [^[]*\[<HOST>\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|Access denied|(?:Client host|Command|Data command) rejected|Relay access denied|Malformed DNS server reply|(?:Host|Domain) not found|need fully-qualified hostname|match%(exre-user)s)\b
             ^from [^[]*\[<HOST>\]%(_port)s:?
+mdad-normal =
 
 mdpr-auth = warning:
 mdre-auth = ^[^[]*\[<HOST>\]%(_port)s: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed:(?! Connection lost to authentication server| Invalid authentication mechanism)
 mdre-auth2= ^[^[]*\[<HOST>\]%(_port)s: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed:(?! Connection lost to authentication server)
 # todo: check/remove "Invalid authentication mechanism" from ignore list, if gh-1243 will get finished (see gh-1297).
+mdad-auth =
+mdad-auth2 =
 
 # Mode "rbl" currently included in mode "normal", but if needed for jail "postfix-rbl" only:
 mdpr-rbl = %(mdpr-normal)s
-mdre-rbl  = ^%(_pref)s from [^[]*\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
+mdre-rbl  = ^%(_cmd)s from [^[]*\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b
+mdad-rbl =
 
 # Mode "rbl" currently included in mode "normal" (within 1st rule)
 mdpr-more = %(mdpr-normal)s
 mdre-more = %(mdre-normal)s
+mdad-more =
 
 # Includes some of the log messages described in
 # <http://www.postfix.org/POSTSCREEN_README.html>.
-mdpr-ddos = (?:lost connection after (?!(?:DATA|AUTH)\b)[A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))|(?:PREGREET \d+|HANGUP) after \S+|COMMAND (?:TIME|COUNT|LENGTH) LIMIT)
+mdpr-ddos = (?:lost connection after (?!(?:DATA|AUTH)\b)[A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))|(?:PREGREET \d+|HANGUP) after \S+|COMMAND (?:TIME|COUNT|LENGTH) LIMIT|warning:)
 mdre-ddos = ^from [^[]*\[<HOST>\]%(_port)s:?
+mdad-ddos = ^(?:Message delivery request|Connection) rate limit exceeded: \d+ from [^[]*\[<ADDR>\]
 
 mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s)
 mdre-extra = %(mdre-auth)s
             %(mdre-normal)s
+mdad-extra =
 
 mdpr-aggressive = (?:%(mdpr-auth)s|%(mdpr-normal)s|%(mdpr-ddos)s)
 mdre-aggressive = %(mdre-auth2)s
                   %(mdre-normal)s
+mdad-aggressive = %(mdad-ddos)s
 
 mdpr-errors = too many errors after \S+
 mdre-errors = ^from [^[]*\[<HOST>\]%(_port)s$
-
+mdad-errors =
 
 failregex = <mdre-<mode>>
+            <mdad-<mode>>
 
 # Parameter "mode": more (default combines normal and rbl), auth, normal, rbl, ddos, extra or aggressive (combines all)
 # Usage example (for jail.local):

config/filter.d/proxmox.conf

@@ -0,0 +1,20 @@
+# Fail2Ban filter for Proxmox Web GUI
+#
+# Jail example:
+#    [proxmox]
+#    enabled = true
+#    port = https,http,8006
+#    filter = proxmox
+#    logpath = /var/log/daemon.log
+#    maxretry = 3
+#    # 1 hour
+#    bantime = 3600
+
+[Definition]
+
+_daemon = pvedaemon
+
+failregex = ^\s*\S+ %(_daemon)s\[\d+\]: authentication failure; rhost=<ADDR> user=<F-USER>\S+</F-USER>
+
+ignoreregex =
+

config/filter.d/recidive.conf

@@ -24,14 +24,15 @@ before = common.conf
 _daemon = (?:fail2ban(?:-server|\.actions)\s*)
 
 # The name of the jail that this filter is used for. In jail.conf, name the jail using
-# this filter 'recidive', or supply another name with `filter = recidive[_jailname="jail"]`
-_jailname = recidive
+# this filter 'recidive', or supply another name with `filter = recidive[_jailname="jail"]`,
+# default all jails excepting recidive
+_jailname = (?!recidive\])[^\]]*
 
-failregex = ^%(__prefix_line)s(?:\s*fail2ban\.actions\s*%(__pid_re)s?:\s+)?NOTICE\s+\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+<HOST>\s*$
+failregex = ^%(__prefix_line)s(?:\s*fail2ban\.actions\s*%(__pid_re)s?:\s+)?NOTICE\s+\[<_jailname>\]\s+Ban\s+<HOST>
 
 [lt_short]
 _daemon = (?:fail2ban(?:-server|\.actions)?\s*)
-failregex = ^%(__prefix_line)s(?:\s*fail2ban(?:\.actions)?\s*%(__pid_re)s?:\s+)?(?:NOTICE\s+)?\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+<HOST>\s*$
+failregex = ^%(__prefix_line)s(?:\s*fail2ban(?:\.actions)?\s*%(__pid_re)s?:\s+)?(?:NOTICE\s+)?\[<_jailname>\]\s+Ban\s+<HOST>
 
 [lt_journal]
 _daemon = <lt_short/_daemon>

config/filter.d/roundcube-auth.conf

@@ -13,10 +13,9 @@ before = common.conf
 
 [Definition]
 
-prefregex = ^\s*(\[\])?(%(__hostname)s\s*(?:roundcube(?:\[(\d*)\])?:)?\s*(<[\w]+>)? IMAP Error)?: <F-CONTENT>.+</F-CONTENT>$
+prefregex = ^\s*(\[\])?(%(__hostname)s\s*(?:roundcube(?:\[(\d*)\])?:)?\s*(<[\w]+>)? IMAP Error)?: (?:<[\w]+> )?<F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^(?:FAILED login|Login failed) for <F-USER>.*</F-USER> from <HOST>(?:(?:\([^\)]*\))?\. (?:(?! from ).)*(?: user=(?P=user))? in \S+\.php on line \d+ \(\S+ \S+\))?$
-            ^(?:<[\w]+> )?Failed login for <F-USER>.*</F-USER> from <HOST> in session \w+( \(error: \d\))?$
+failregex = ^(?:Login failed|(?i:Failed) login) for <F-USER>(?:(?P<simple>\S+)|.*)</F-USER> (?:against \S+ )?from <ADDR>(?:(?:\([^\)]*\))?\.(?! from ) (?(simple)(?:\S+(?! from ) )*|(?:(?! from ).)*(?: user=(?P=user))? )in \S+\.php on line \d+| in session \w+)?(?: \([^\)]*\))?$
 
 ignoreregex = 
 

config/filter.d/sendmail-reject.conf

@@ -22,21 +22,28 @@ before = common.conf
 _daemon = (?:(sm-(mta|acceptingconnections)|sendmail))
 __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )?
 addr = (?:(?:IPv6:)?<IP6>|<IP4>)
+# mta_dname -- matches name of MTA daemon (typically specified in DAEMON_OPTIONS),
+# normally something without spaces like MTA-v4 or Deamon0, etc. If it'd contain spaces, one can
+# rewrite it in jail using  `filter = %(known/filter)s[mta_dname="[^,]+"]` or in .local overwrite
+# of the filter. (we would not use catch-alls here to satisfy obscure artificial case).
+mta_dname = \S+
 
-prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$
+prefregex = ^\s*(?:<mail\.[^\>]+> )?<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$
 
-cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(?:550 5\.7\.1(?: (?P=email)\.\.\.)?(?: Relaying denied\.)? (?:IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\]|Fix reverse DNS for \S+)|553 5\.1\.8(?: (?P=email)\.\.\.)? Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
+cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(?:550 5\.7\.1(?: (?P=email)\.\.\.)?(?: Relaying denied\.)? (?:IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\]|Fix reverse DNS for \S+)|[45]5[13] [45]\.1\.8(?: (?P=email)\.\.\.)? Domain of sender address \S+ does not (?:exist|resolve)|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
             ^ruleset=check_relay(?:, arg\d+=\S*)*, relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
             ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$
             ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
-            ^<[^@]+@[^>]+>\.\.\. No such user here$
-            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[%(addr)s\]$
+            ^<[^@]+@[^>]+>\.\.\. (?:No such user here|User unknown)$
+            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+,(?: bodytype=\w+,)? proto=E?SMTP, daemon=%(mta_dname)s, relay=(?:\S+ )?\[%(addr)s\]$
 
 mdre-normal =
 
 mdre-extra = ^(?:\S+ )?\[%(addr)s\](?: \(may be forged\))? did not issue \S+ during connection
 
 mdre-aggressive = %(mdre-extra)s
+                  ^lost input channel from (?:\S+ )?\[%(addr)s\] to %(mta_dname)s after rcpt$
+                  ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(?:450 4\.4\.0(?: (?P=email)\.\.\.)?(?: Relaying temporarily denied\.)?(?: Cannot resolve PTR record for (\d+\.){3}\d+))$
 
 failregex = %(cmnfailre)s
             <mdre-<mode>>
@@ -63,6 +70,8 @@ journalmatch = SYSLOG_IDENTIFIER=sm-mta + _SYSTEMD_UNIT=sendmail.service
 # Note the capture <F-MLFID>, includes both the __prefix_lines (which includes
 # the sendmail PID), but also the `\w{14}` which the the sendmail assigned 
 # mail ID (todo: check this is necessary, possible obsolete).
+# Avoid moving <F-MLFID> into the entire prefregex because the grouped messages we 
+# need have different syslog levels (info vs notice) that break the group if BSD verbose format is set
 #
 # Author: Daniel Black, Fabian Wenk and Sergey Brester aka sebres.
 # Rewritten using prefregex by Serg G. Brester.

config/filter.d/sshd.conf

@@ -16,7 +16,7 @@ before = common.conf
 
 [DEFAULT]
 
-_daemon = sshd
+_daemon = sshd(?:-session)?
 
 # optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: "
 __pref = (?:(?:error|fatal): (?:PAM: )?)?
@@ -70,7 +70,7 @@ mdre-normal =
 # used to differentiate "connection closed" with and without `[preauth]` (fail/nofail cases in ddos mode)
 mdre-normal-other = ^<F-NOFAIL><F-MLFFORGET>(?:Connection (?:closed|reset)|Disconnect(?:ed|ing))</F-MLFFORGET></F-NOFAIL>%(__authng_user)s <ADDR>%(__on_port_opt)s(?:: (?!Too many authentication failures)[^\[]+)?(?: \[preauth\])?\s*$
 
-mdre-ddos = ^(?:Did not receive identification string from|Timeout before authentication for) <HOST>
+mdre-ddos = ^(?:Did not receive identification string from|Timeout before authentication for(?: connection from)?) <HOST>
             ^kex_exchange_identification: (?:read: )?(?:[Cc]lient sent invalid protocol identifier|[Cc]onnection (?:closed by remote host|reset by peer))
             ^Bad protocol version identification '(?:[^']|.*?)' (?:from )?<HOST>%(__suff)s$
             ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
@@ -126,7 +126,7 @@ ignoreregex =
 
 maxlines = 1
 
-journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd
+journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd + _COMM=sshd-session
 
 # DEV Notes:
 #

config/filter.d/vaultwarden.conf

@@ -0,0 +1,8 @@
+# Fail2Ban filter for unsuccessful Vaultwarden authentication attempts
+# Logged in /var/log/vaultwarden.log
+# Author: LearningSpot
+
+[Definition]
+
+failregex = ^\s*(?:\[\]\s*)?\[vaultwarden::api::(?:identity|admin|core::two_factor::authenticator)?\]\[ERROR\] (?:Invalid admin token|Invalid TOTP code|Username or password is incorrect)[\.!](?:\s+(?!IP:)\S+)* IP: <ADDR>(?:\. Username: <F-USER>\S+</F-USER>)?
+ignoreregex =

config/filter.d/vsftpd.conf

@@ -10,13 +10,13 @@ before = common.conf
 
 [Definition]
 
-__pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:?
+__pam_re=(?:\(?%(__pam_auth)s(?:\(\S+\))?\)?:?\s+)?
 _daemon =  vsftpd
 
-failregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=(ftp)? ruser=\S* rhost=<HOST>(?:\s+user=.*)?\s*$
-            ^ \[pid \d+\] \[[^\]]+\] FAIL LOGIN: Client "<HOST>"(?:\s*$|,)
+failregex = ^%(__prefix_line)s%(__pam_re)sauthentication failure; logname=<F-ALT_USER1>\S*</F-ALT_USER1> uid=\S* euid=\S* tty=(?:ftp)? ruser=<F-USER>\S*</F-USER> rhost=<HOST>(?:\s+user=<F-ALT_USER>\S*</F-ALT_USER>)?\s*$
+            ^(?:\s*\[pid \d+\] |%(__prefix_line)s)\[<F-USER>[^\]]+</F-USER>\] FAIL LOGIN: Client "<HOST>"(?:\s*$|,)
 
-ignoreregex = 
+ignoreregex =
 
-# Author: Cyril Jaquier
+# Authors: Cyril Jaquier, Lucian Maly <lmaly@redhat.com>
 # Documentation from fail2ban wiki

config/filter.d/xrdp.conf

@@ -0,0 +1,35 @@
+#
+# Fail2Ban filter for XRDP
+#
+# Detects login attempts with invalid credentials
+#
+# Requirements:
+#   - xrdp >= 0.9.19
+#   - The log level in sesman.ini should be set to `INFO` or higher
+#     to emit the log messages needed for this filter.
+#
+# Author: Evan Linde
+#
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+
+[DEFAULT]
+
+_daemon = xrdp-sesman
+
+
+[Definition]
+
+authfail_re = \[INFO \] AUTHFAIL: user=<F-USER>(?:\S+|.+)</F-USER> ip=<ADDR> time=\d+
+
+failregex = ^%(__prefix_line)s%(authfail_re)s$
+
+ignoreregex =
+
+datepattern = ^\[?%%ExY%%Exm%%Exd-%%ExH:%%ExM:%%ExS\]?
+              ^{DATE}

config/jail.conf

@@ -205,8 +205,8 @@ fail2ban_agent = Fail2Ban/%(fail2ban_version)s
 # iptables-multiport, shorewall, etc) It is used to define
 # action_* variables. Can be overridden globally or per
 # section within jail.local file
-banaction = iptables-multiport
-banaction_allports = iptables-allports
+#banaction = iptables-multiport
+#banaction_allports = iptables-allports
 
 # The simplest action to take: ban only
 action_ = %(banaction)s[port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
@@ -227,14 +227,11 @@ action_mwl = %(action_)s
 action_xarf = %(action_)s
              xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"]
 
-# ban & send a notification to one or more of the 50+ services supported by Apprise.
-# See https://github.com/caronc/apprise/wiki for details on what is supported.
-#
-# You may optionally over-ride the default configuration line (containing the Apprise URLs)
-# by using 'apprise[config="/alternate/path/to/apprise.cfg"]' otherwise
-# /etc/fail2ban/apprise.conf is sourced for your supported notification configuration.
+# ban & send a notification to one or more of the 120+ services supported by Apprise.
 # action = %(action_)s
-#          apprise
+#          apprise[config="/alternate/path/to/apprise.yaml", args='--tag fail2ban']
+# See https://appriseit.com/services/ for details on what is supported.
+# Or action.d/apprise.conf for more details how to configure or customize it.
 
 # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines
 # to the destemail.
@@ -501,7 +498,7 @@ backend = %(syslog_backend)s
 [froxlor-auth]
 
 port    = http,https
-logpath  = %(syslog_authpriv)s
+logpath  = %(syslog_user)s
 backend  = %(syslog_backend)s
 
 
@@ -785,17 +782,11 @@ logpath  = /var/lib/znc/moddata/adminlog/znc.log
 
 # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld] or
 # equivalent section:
-# log-warnings = 2
-#
-# for syslog (daemon facility)
-# [mysqld_safe]
-# syslog
-#
-# for own logfile
-# [mysqld]
-# log-error=/var/log/mysqld.log
+#   log_error_verbosity = 3
+# for older versions:
+#   log-warnings = 2
+# Also check whether `log_error` (or `log-error`) system variable match the `logpath`.
 [mysqld-auth]
-
 port     = 3306
 logpath  = %(mysql_log)s
 backend  = %(mysql_backend)s
@@ -978,6 +969,10 @@ logpath = %(apache_error_log)s
 port    = http,https
 logpath = /var/log/traefik/access.log
 
+[openvpn]
+port = 443
+logpath = /var/log/syslog
+
 [scanlogd]
 logpath = %(syslog_local0)s
 banaction = %(banaction_allports)s
@@ -990,3 +985,14 @@ logpath = /var/log/monitorix-httpd
 port    = 1080
 logpath = %(syslog_daemon)s
 
+[proxmox]
+port = https,http,8006
+logpath = /var/log/daemon.log
+
+[vaultwarden]
+port = http,https
+logpath = /var/log/vaultwarden.log
+
+[xrdp]
+port    = 3389
+logpath = /var/log/xrdp-sesman.log

config/paths-common.conf

@@ -87,7 +87,12 @@ dovecot_backend = %(default_backend)s
 # Seems to be set at compile time only to LOG_LOCAL0 (src/const.h) at Notice level
 solidpop3d_log = %(syslog_local0)s
 
-mysql_log = %(syslog_daemon)s
+mysql_log = /var/log/mariadb/mariadb.log
+            /var/log/mariadb/error.log
+            /var/log/mysql/mysqld.log
+            /var/log/mysql/error.log
+            /var/log/mysqld.log
+            
 mysql_backend = %(default_backend)s
 
 roundcube_errors_log = /var/log/roundcube/errors

config/paths-debian.conf

@@ -9,6 +9,12 @@ after  = paths-overrides.local
 
 [DEFAULT]
 
+banaction = nftables
+banaction_allports = nftables[type=allports]
+
+sshd_backend = systemd
+postfix_backend = systemd
+
 syslog_mail = /var/log/mail.log
 
 # control the `mail.warn` setting, see `/etc/rsyslog.d/50-default.conf` (if commented `mail.*` wins).

doc/conf.py

@@ -47,12 +47,9 @@ copyright = u'2014'
 #
 
 from fail2ban.version import version as fail2ban_version
-from distutils.version import LooseVersion
-
-fail2ban_loose_version = LooseVersion(fail2ban_version)
 
 # The short X.Y version.
-version = ".".join(str(_) for _ in fail2ban_loose_version.version[:2])
+version = ".".join(str(_) for _ in fail2ban_version.split(".")[:2])
 # The full version, including alpha/beta/rc tags.
 release = fail2ban_version
 

fail2ban/client/beautifier.py

@@ -21,8 +21,10 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko"
 __license__ = "GPL"
 
+import sys
+
 from ..exceptions import UnknownJailException, DuplicateJailException
-from ..helpers import getLogger, logging
+from ..helpers import getLogger, logging, PREFER_ENC
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -36,6 +38,11 @@ logSys = getLogger(__name__)
 
 class Beautifier:
 
+	stdoutEnc = PREFER_ENC
+	if sys.stdout and sys.stdout.encoding is not None:
+		stdoutEnc = sys.stdout.encoding
+	encUtf = 1 if stdoutEnc.lower() == 'utf-8' else 0
+
 	def __init__(self, cmd = None):
 		self.__inputCmd = cmd
 
@@ -104,7 +111,11 @@ class Beautifier:
 							jail_stat(j, "      " if i == len(jstat) else "   |  ")
 				msg = "\n".join(msg)
 			elif inC[0:1] == ['stats'] or inC[0:1] == ['statistics']:
-				def _statstable(response):
+				chrTable = [
+					['|', '-', '|', 'x', 'x', '-', '|', '-'], ## ascii
+					["\u2551", "\u2550", "\u255F", "\u256B", "\u256C", "\u2569", "\u2502", "\u2500"] ## utf-8
+				];
+				def _statstable(response, ct):
 					tophead = ["Jail", "Backend", "Filter", "Actions"]
 					headers = ["", "", "cur", "tot", "cur", "tot"]
 					minlens = [8, 8, 3, 3, 3, 3]
@@ -120,29 +131,31 @@ class Beautifier:
 						f = "%%%ds" if ralign[i] else "%%-%ds"
 						rfmt.append(f % lens[i])
 						hfmt.append(f % lens[i])
-					rfmt = [rfmt[0], rfmt[1], "%s \u2502 %s" % (rfmt[2], rfmt[3]), "%s \u2502 %s" % (rfmt[4], rfmt[5])]
-					hfmt = [hfmt[0], hfmt[1], "%s \u2502 %s" % (hfmt[2], hfmt[3]), "%s \u2502 %s" % (hfmt[4], hfmt[5])]
+					rfmt = [rfmt[0], rfmt[1], "%s %s %s" % (rfmt[2], ct[6], rfmt[3]), "%s %s %s" % (rfmt[4], ct[6], rfmt[5])]
+					hfmt = [hfmt[0], hfmt[1], "%s %s %s" % (hfmt[2], ct[6], hfmt[3]), "%s %s %s" % (hfmt[4], ct[6], hfmt[5])]
 					tlens = [lens[0], lens[1], 3 + lens[2] + lens[3], 3 + lens[4] + lens[5]]
 					tfmt = [hfmt[0], hfmt[1], "%%-%ds" % (tlens[2],), "%%-%ds" % (tlens[3],)]
 					tsep = tfmt[0:2]
-					rfmt = " \u2551 ".join(rfmt)
-					hfmt = " \u2551 ".join(hfmt)
-					tfmt = " \u2551 ".join(tfmt)
-					tsep = " \u2551 ".join(tsep)
-					separator = ((tsep % tuple(tophead[0:2])) + " \u255F\u2500" + 
-						("\u2500\u256B\u2500".join(['\u2500' * n for n in tlens[2:]])) + '\u2500')
+					rfmt = (" "+ct[0]+" ").join(rfmt)
+					hfmt = (" "+ct[0]+" ").join(hfmt)
+					tfmt = (" "+ct[0]+" ").join(tfmt)
+					tsep = (" "+ct[0]+" ").join(tsep)
+					separator = ((tsep % tuple(tophead[0:2])) + " "+ct[2]+ct[7] + 
+						((ct[7]+ct[3]+ct[7]).join([ct[7] * n for n in tlens[2:]])) + ct[7])
 					ret = []
-					ret.append(tfmt % tuple(["", ""]+tophead[2:]))
-					ret.append(separator)
-					ret.append(hfmt % tuple(headers))
-					separator = "\u2550\u256C\u2550".join(['\u2550' * n for n in tlens]) + '\u2550'
-					ret.append(separator)
+					ret.append(" "+tfmt % tuple(["", ""]+tophead[2:]))
+					ret.append(" "+separator)
+					ret.append(" "+hfmt % tuple(headers))
+					separator = (ct[1]+ct[4]+ct[1]).join([ct[1] * n for n in tlens]) + ct[1]
+					ret.append(ct[1]+separator)
 					for row in rows:
-						ret.append(rfmt % tuple(row))
-					separator = "\u2550\u2569\u2550".join(['\u2550' * n for n in tlens]) + '\u2550'
-					ret.append(separator)
+						ret.append(" "+rfmt % tuple(row))
+					separator = (ct[1]+ct[5]+ct[1]).join([ct[1] * n for n in tlens]) + ct[1]
+					ret.append(ct[1]+separator)
 					return ret
-				msg = "\n".join(_statstable(response))
+				if not response:
+					return "No jails found."
+				msg = "\n".join(_statstable(response, chrTable[self.encUtf]))
 			elif len(inC) < 2:
 				pass # to few cmd args for below
 			elif inC[1] == "syslogsocket":
@@ -199,8 +212,8 @@ class Beautifier:
 				else:
 					msg = "These IP addresses/networks are ignored:\n"
 					for ip in response[:-1]:
-						msg += "|- " + ip + "\n"
-					msg += "`- " + response[-1]
+						msg += "|- " + str(ip) + "\n"
+					msg += "`- " + str(response[-1])
 			elif inC[2] in ("failregex", "addfailregex", "delfailregex",
 							"ignoreregex", "addignoreregex", "delignoreregex"):
 				if len(response) == 0:

fail2ban/client/configreader.py

@@ -230,6 +230,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes):
 			missed = [ cf for cf in config_files if cf not in config_files_read ]
 			if missed:
 				logSys.error("Could not read config files: %s", ', '.join(missed))
+				return False
 			if config_files_read:
 				return True
 			logSys.error("Found no accessible config files for %r under %s",
@@ -353,6 +354,11 @@ class DefinitionInitConfigReader(ConfigReader):
 					if v is None: v = getopt(opt)
 					self._initOpts['known/'+opt] = v
 				if opt not in self._initOpts:
+					# overwrite also conditional init options (from init?... section):
+					cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(opt)
+					if cond:
+						optc, cond = cond.groups()
+						v = pOpts.get(optc, v)
 					if v is None: v = getopt(opt)
 					self._initOpts[opt] = v
 		if all and self.has_section("Definition"):
@@ -406,7 +412,7 @@ class DefinitionInitConfigReader(ConfigReader):
 			if cond:
 				n, cond = cond.groups()
 				ignore.add(n)
-		# substiture options already specified direct:
+		# substitute options already specified direct:
 		opts = substituteRecursiveTags(combinedopts, 
 			ignore=ignore, addrepl=self.getCombOption)
 		if not opts:

fail2ban/client/configurator.py

@@ -63,11 +63,13 @@ class Configurator:
 		return fail2ban_basedir
 	
 	def readEarly(self):
-		self.__fail2ban.read()
+		if not self.__fail2ban.read():
+			raise LookupError("Read fail2ban configuration failed.")
 	
 	def readAll(self):
 		self.readEarly()
-		self.__jails.read()
+		if not self.__jails.read():
+			raise LookupError("Read jails configuration failed.")
 	
 	def getEarlyOptions(self):
 		return self.__fail2ban.getEarlyOptions()

fail2ban/client/fail2banclient.py

@@ -406,6 +406,8 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				if ret is not None:
 					if ret:
 						return True
+					if self._conf.get("test", False) and not self._args: # test only
+						return False
 					raise ServerExecutionException("Init of command line failed")
 
 			# Commands

fail2ban/client/fail2bancmdline.py

@@ -260,12 +260,15 @@ class Fail2banCmdLine():
 				if readcfg:
 					readcfg = False
 					ret, stream = self.readConfig()
-				if not ret:
-					raise ServerExecutionException("ERROR: test configuration failed")
 				# exit after test if no commands specified (test only):
 				if not len(self._args):
-					output("OK: configuration test is successful")
+					if ret:
+						output("OK: configuration test is successful")
+					else:
+						output("ERROR: test configuration failed")
 					return ret
+				if not ret:
+					raise ServerExecutionException("ERROR: test configuration failed")
 
 			# Nothing to do here, process in client/server
 			return None

fail2ban/client/fail2banreader.py

@@ -37,7 +37,7 @@ class Fail2banReader(ConfigReader):
 		ConfigReader.__init__(self, **kwargs)
 	
 	def read(self):
-		ConfigReader.read(self, "fail2ban")
+		return ConfigReader.read(self, "fail2ban")
 	
 	def getEarlyOptions(self):
 		opts = [

fail2ban/client/fail2banregex.py

@@ -106,7 +106,7 @@ usage = lambda: "%s [OPTIONS] <LOG> <REGEX> [IGNOREREGEX]" % sys.argv[0]
 
 class _f2bOptParser(OptionParser):
 	def format_help(self, *args, **kwargs):
-		""" Overwritten format helper with full ussage."""
+		""" Overwritten format helper with full usage."""
 		self.usage = ''
 		return "Usage: " + usage() + "\n" + __doc__ + """
 LOG:
@@ -118,12 +118,11 @@ LOG:
 
 REGEX:
   string                a string representing a 'failregex'
-  filter                name of filter, optionally with options (sshd[mode=aggressive])
+  filter                name of jail or filter, optionally with options (sshd[mode=aggressive])
   filename              path to a filter file (filter.d/sshd.conf)
 
 IGNOREREGEX:
   string                a string representing an 'ignoreregex'
-  filename              path to a filter file (filter.d/sshd.conf)
 \n""" + OptionParser.format_help(self, *args, **kwargs) + """\n
 Report bugs to https://github.com/fail2ban/fail2ban/issues\n
 """ + __copyright__ + "\n"
@@ -173,6 +172,8 @@ def get_opt_parser():
 			   help="Disable check for all regex's"),
 		Option("-o", "--out", action="store", dest="out", default=None,
 			   help="Set token to print failure information only (row, id, ip, msg, host, ip4, ip6, dns, matches, ...)"),
+		Option("-i", "--invert", action="store_true", dest="invert",
+			   help="Invert the sense of matching, to output non-matching lines."),
 		Option("--print-no-missed", action='store_true',
 			   help="Do not print any missed lines"),
 		Option("--print-no-ignored", action='store_true',
@@ -370,6 +371,9 @@ class Fail2banRegex(object):
 					output("       while parsing: %s" % (value,))
 					if self._verbose: raise(e)
 					return False
+		elif self._ignoreregex:
+			# clear ignoreregex that could be previously loaded from filter:
+			self._filter.delIgnoreRegex()
 		
 		readercommands = None
 		# if it is jail:
@@ -432,8 +436,8 @@ class Fail2banRegex(object):
 			# to stream:
 			readercommands = reader.convert()
 
+		regex_values = {}
 		if readercommands:
-			regex_values = {}
 			for opt in readercommands:
 				if opt[0] == 'multi-set':
 					optval = opt[3]
@@ -473,7 +477,7 @@ class Fail2banRegex(object):
 
 		else:
 			self.output( "Use %11s line : %s" % (regex, shortstr(value)) )
-			regex_values = {regextype: [RegexStat(value)]}
+			regex_values[regextype] = [RegexStat(value)]
 
 		for regextype, regex_values in regex_values.items():
 			regex = regextype + 'regex'
@@ -527,7 +531,7 @@ class Fail2banRegex(object):
 		except RegexException as e: # pragma: no cover
 			output( 'ERROR: %s' % e )
 			return None, 0, None
-		if self._filter.getMaxLines() > 1:
+		if self._filter.getMaxLines() > 1 and not self._opts.out:
 			for bufLine in orgLineBuffer[int(fullBuffer):]:
 				if bufLine not in self._filter._Filter__lineBuffer:
 					try:
@@ -617,8 +621,10 @@ class Fail2banRegex(object):
 
 	def process(self, test_lines):
 		t0 = time.time()
+		out = None
 		if self._opts.out: # get out function
 			out = self._prepaireOutput()
+			outinv = self._opts.invert
 		for line in test_lines:
 			if isinstance(line, tuple):
 				line_datetimestripped, ret, is_ignored = self.testRegex(line[0], line[1])
@@ -630,8 +636,13 @@ class Fail2banRegex(object):
 					continue
 				line_datetimestripped, ret, is_ignored = self.testRegex(line)
 
-			if self._opts.out: # (formatted) output:
-				if len(ret) > 0 and not is_ignored: out(ret)
+			if out: # (formatted) output:
+				if len(ret) > 0 and not is_ignored:
+					if not outinv: out(ret)
+				elif outinv: # inverted output (currently only time and message as matches):
+					if not len(ret): # [failRegexIndex, fid, date, fail]
+						ret = [[-1, "", self._filter._Filter__lastDate, {"fid":"", "matches":[line]}]]
+					out(ret)
 				continue
 
 			if is_ignored:

fail2ban/client/filterreader.py

@@ -89,7 +89,7 @@ class FilterReader(DefinitionInitConfigReader):
 				stream.insert(0 if opt == 'usedns' else prio0idx,
 					["set", jailName, opt, value])
 				prio0idx += 1
-			elif opt in ('datepattern'):
+			elif opt == 'datepattern':
 				stream.append(["set", jailName, opt, value])
 			elif opt == 'journalmatch':
 				for match in value.split("\n"):

fail2ban/client/jailreader.py

@@ -116,11 +116,16 @@ class JailReader(ConfigReader):
 		"logtimezone": ["string", None],
 		"logencoding": ["string", None],
 		"logpath": ["string", None],
+		"skip_if_nologs": ["bool", False],
+		"systemd_if_nologs": ["bool", True],
 		"action": ["string", ""]
 	}
 	_configOpts.update(FilterReader._configOpts)
 
-	_ignoreOpts = set(['action', 'filter', 'enabled', 'backend'] + list(FilterReader._configOpts.keys()))
+	_ignoreOpts = set(
+		['action', 'filter', 'enabled', 'backend', 'skip_if_nologs', 'systemd_if_nologs'] +
+		list(FilterReader._configOpts.keys())
+	)
 
 	def getOptions(self, addOpts=None):
 
@@ -235,7 +240,7 @@ class JailReader(ConfigReader):
 			return self.__opts
 		return _merge_dicts(self.__opts, self.__filter.getCombined())
 
-	def convert(self, allow_no_files=False):
+	def convert(self, allow_no_files=False, systemd_if_nologs=True):
 		"""Convert read before __opts to the commands stream
 
 		Parameters
@@ -273,10 +278,26 @@ class JailReader(ConfigReader):
 						stream2.append(
 							["set", self.__name, "addlogpath", p, tail])
 				if not found_files:
-					msg = "Have not found any log file for %s jail" % self.__name
-					if not allow_no_files:
+					msg = "Have not found any log file for '%s' jail." % self.__name
+					skip_if_nologs = self.__opts.get('skip_if_nologs', False)
+					# if auto and we can switch to systemd backend (only possible if jail have journalmatch):
+					if backend.startswith("auto") and systemd_if_nologs and (
+					  self.__opts.get('systemd_if_nologs', True) and
+					  self.__opts.get('journalmatch', None) is not None
+					):
+						# switch backend to systemd:
+						backend = 'systemd'
+						msg += " Jail will monitor systemd journal."
+						skip_if_nologs = False
+					elif not allow_no_files and not skip_if_nologs:
 						raise ValueError(msg)
 					logSys.warning(msg)
+					if skip_if_nologs:
+						self.__opts['runtime-error'] = msg
+						msg = "Jail '%s' skipped, because of missing log files." % (self.__name,)
+						logSys.warning(msg)
+						stream = [['config-error', msg]]
+						return stream
 			elif opt == "ignoreip":
 				stream.append(["set", self.__name, "addignoreip"] + splitwords(value))
 			elif opt not in JailReader._ignoreOpts:

fail2ban/client/jailsreader.py

@@ -88,7 +88,7 @@ class JailsReader(ConfigReader):
 				parse_status |= 2
 		return ((ignoreWrong and parse_status & 1) or not (parse_status & 2))
 
-	def convert(self, allow_no_files=False):
+	def convert(self, allow_no_files=False, systemd_if_nologs=True):
 		"""Convert read before __opts and jails to the commands stream
 
 		Parameters
@@ -101,11 +101,14 @@ class JailsReader(ConfigReader):
 		stream = list()
 		# Convert jails
 		for jail in self.__jails:
-			stream.extend(jail.convert(allow_no_files=allow_no_files))
+			stream.extend(jail.convert(allow_no_files, systemd_if_nologs))
 		# Start jails
 		for jail in self.__jails:
-			if not jail.options.get('config-error'):
+			if not jail.options.get('config-error') and not jail.options.get('runtime-error'):
 				stream.append(["start", jail.getName()])
+			else:
+				# just delete rtm-errors (to check next time if cached)
+				jail.options.pop('runtime-error', None)
 
 		return stream
 

fail2ban/helpers.py

@@ -40,6 +40,14 @@ except:
 	_libcap = None
 
 
+# some modules (like pyinotify, see #3487) may have dependency to asyncore, so ensure we've a path
+# to compat folder, otherwise python 3.12+ could miss them:
+def __extend_compat_path():
+	cp = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'compat')
+	if cp not in sys.path:
+		sys.path.append(cp)
+__extend_compat_path()
+
 PREFER_ENC = locale.getpreferredencoding()
 # correct preferred encoding if lang not set in environment:
 if PREFER_ENC.startswith('ANSI_'): # pragma: no cover
@@ -274,7 +282,18 @@ def excepthook(exctype, value, traceback):
 		"Unhandled exception in Fail2Ban:", exc_info=True)
 	return sys.__excepthook__(exctype, value, traceback)
 
-def splitwords(s):
+RE_REM_COMMENTS = re.compile(r'(?m)(?:^|\s)[\#;].*')
+def removeComments(s):
+	"""Helper to remove comments:
+		# comment ...
+		; comment ...
+		no comment # comment ...
+		no comment ; comment ...
+	"""
+	return RE_REM_COMMENTS.sub('', s)
+
+RE_SPLT_WORDS = re.compile(r'[\s,]+')
+def splitwords(s, ignoreComments=False):
 	"""Helper to split words on any comma, space, or a new line
 
 	Returns empty list if input is empty (or None) and filters
@@ -282,7 +301,9 @@ def splitwords(s):
 	"""
 	if not s:
 		return []
-	return list(filter(bool, [v.strip() for v in re.split(r'[\s,]+', s)]))
+	if ignoreComments:
+		s = removeComments(s)
+	return list(filter(bool, [v.strip() for v in RE_SPLT_WORDS.split(s)]))
 
 def _merge_dicts(x, y):
 	"""Helper to merge dicts.
@@ -302,15 +323,17 @@ def _merge_copy_dicts(x, y):
 
 # regex, to extract list of options:
 OPTION_CRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL)
+# regex, matching option name (inclusive conditional option, like n?family=inet6):
+OPTION_NAME_CRE = r'[\w\-_\.]+(?:\?[\w\-_\.]+=[\w\-_\.]+)?'
 # regex, to iterate over single option in option list, syntax:
 # `action = act[p1="...", p2='...', p3=...]`, where the p3=... not contains `,` or ']'
 # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax 
 # `action = act[p1=...][p2=...]`
 OPTION_EXTRACT_CRE = re.compile(
-	r'\s*([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$|(?P<wrngA>.+))|,?\s*$|(?P<wrngB>.+)', re.DOTALL)
+	r'\s*('+OPTION_NAME_CRE+r')=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$|(?P<wrngA>.+))|,?\s*$|(?P<wrngB>.+)', re.DOTALL)
 # split by new-line considering possible new-lines within options [...]:
 OPTION_SPLIT_CRE = re.compile(
-	r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL)
+	r'(?:[^\[\s]+(?:\s*\[\s*(?:'+OPTION_NAME_CRE+r'=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL)
 
 def extractOptions(option):
 	match = OPTION_CRE.match(option)

fail2ban/protocol.py

@@ -58,11 +58,10 @@ protocol = [
 ["banned", "return jails with banned IPs as dictionary"],
 ["banned <IP> ... <IP>]", "return list(s) of jails where given IP(s) are banned"],
 ["status", "gets the current status of the server"], 
-["status --all [FLAVOR]", "gets the current status of all jails, with optional flavor or extended info"],
+["status --all [FLAVOR]", "gets the current status of all jails, with optional output style [FLAVOR]. Flavors: 'basic' (default), 'cymru', 'short', 'stats'"],
 ["stat[istic]s", "gets the current statistics of all jails as table"],
 ["ping", "tests if the server is alive"],
 ["echo", "for internal usage, returns back and outputs a given string"],
-["help", "return this output"], 
 ["version", "return the server version"],
 ['', "LOGGING", ""],
 ["set loglevel <LEVEL>", "sets logging level to <LEVEL>. Levels: CRITICAL, ERROR, WARNING, NOTICE, INFO, "
@@ -84,7 +83,7 @@ protocol = [
 ["add <JAIL> <BACKEND>", "creates <JAIL> using <BACKEND>"], 
 ["start <JAIL>", "starts the jail <JAIL>"], 
 ["stop <JAIL>", "stops the jail <JAIL>. The jail is removed"], 
-["status <JAIL> [FLAVOR]", "gets the current status of <JAIL>, with optional flavor or extended info"],
+["status <JAIL> [FLAVOR]", "gets the current status of <JAIL>, with optional output style [FLAVOR]. Flavors: 'basic' (default), 'cymru', 'short', 'stats'"],
 ['', "JAIL CONFIGURATION", ""],
 ["set <JAIL> idle on|off", "sets the idle state of <JAIL>"], 
 ["set <JAIL> ignoreself true|false", "allows the ignoring of own IP addresses"], 

fail2ban/server/actions.py

@@ -640,12 +640,12 @@ class Actions(JailThread, Mapping):
 		If actions specified, don't flush list - just execute unban for 
 		given actions (reload, obsolete resp. removed actions).
 		"""
-		log = True
+		log = "Unban" if not stop else "Repeal Ban"
 		if actions is None:
 			logSys.debug("  Flush ban list")
 			lst = self.banManager.flushBanList()
 		else:
-			log = False # don't log "[jail] Unban ..." if removing actions only.
+			log = None # don't log "[jail] Unban ..." if removing actions only.
 			lst = iter(self.banManager)
 		cnt = 0
 		# first we'll execute flush for actions supporting this operation:
@@ -686,7 +686,7 @@ class Actions(JailThread, Mapping):
 			cnt, self.banManager.size(), self._jail.name)
 		return cnt
 
-	def __unBan(self, ticket, actions=None, log=True):
+	def __unBan(self, ticket, actions=None, log="Unban"):
 		"""Unbans host corresponding to the ticket.
 
 		Executes the actions in order to unban the host given in the
@@ -704,7 +704,7 @@ class Actions(JailThread, Mapping):
 		ip = ticket.getID()
 		aInfo = self._getActionInfo(ticket)
 		if log:
-			logSys.notice("[%s] Unban %s", self._jail.name, ip)
+			logSys.notice("[%s] %s %s", self._jail.name, log, ip)
 		for name, action in unbactions.items():
 			try:
 				logSys.debug("[%s] action %r: unban %s", self._jail.name, name, ip)

fail2ban/server/asyncserver.py

@@ -25,14 +25,6 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
 from pickle import dumps, loads, HIGHEST_PROTOCOL
-try:
-	import asynchat
-except ImportError:
-	from ..compat import asynchat
-try:
-	import asyncore
-except ImportError:
-	from ..compat import asyncore
 import errno
 import fcntl
 import os
@@ -45,6 +37,13 @@ from .utils import Utils
 from ..protocol import CSPROTO
 from ..helpers import logging, getLogger, formatExceptionInfo
 
+# load asyncore and asynchat after helper to ensure we've a path to compat folder:
+import asynchat
+if asynchat.asyncore:
+	asyncore = asynchat.asyncore
+else: # pragma: no cover - normally unreachable
+	import asyncore
+
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 

fail2ban/server/database.py

@@ -725,7 +725,7 @@ class Fail2BanDb(object):
 						failures = 0
 						tickdata = {}
 					m = data.get('matches', [])
-					# pre-insert "maxadd" enries (because tickets are ordered desc by time)
+					# pre-insert "maxadd" entries (because tickets are ordered desc by time)
 					maxadd = self.maxMatches - len(matches)
 					if maxadd > 0:
 						if len(m) <= maxadd:

fail2ban/server/datedetector.py

@@ -165,8 +165,8 @@ class DateDetectorCache(object):
 		r"%b %d, %ExY %I:%M:%S %p",
 		# ASSP: Apr-27-13 02:33:06
 		r"^%b-%d-%Exy %k:%M:%S",
-		# 20050123T215959, 20050123 215959, 20050123  85959
-		r"%ExY%Exm%Exd(?:T|  ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?",
+		# 20050123T215959, 20050123 215959, 20050123  85959, 20050123-21:59:59
+		r"%ExY%Exm%Exd(?:-|T|  ?)%ExH:?%ExM:?%ExS(?:[.,]%f)?(?:\s*%z)?",
 		# prefixed with optional named time zone (monit):
 		# PDT Apr 16 21:05:29
 		r"(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?",
@@ -252,6 +252,8 @@ class DateDetector(object):
 				"There is already a template with name %s" % name)
 		self.__known_names.add(name)
 		self.__templates.append(DateDetectorTemplate(template))
+		logSys.debug("  date pattern regex for `%s`: `%s`",
+			getattr(template, 'pattern', ''), template.regex)
 
 	def appendTemplate(self, template):
 		"""Add a date template to manage and use in search of dates.
@@ -289,16 +291,15 @@ class DateDetector(object):
 
 			DD_patternCache.set(key, template)
 
-		self._appendTemplate(template)
-		logSys.info("  date pattern `%r`: `%s`",
+		logSys.info("  date pattern `%s`: `%s`",
 			getattr(template, 'pattern', ''), template.name)
-		logSys.debug("  date pattern regex for %r: %s",
-			getattr(template, 'pattern', ''), template.regex)
+		self._appendTemplate(template)
 
 	def addDefaultTemplate(self, filterTemplate=None, preMatch=None, allDefaults=True):
 		"""Add Fail2Ban's default set of date templates.
 		"""
 		ignoreDup = len(self.__templates) > 0
+		cnt = 0
 		for template in (
 			DateDetector._defCache.templates if allDefaults else DateDetector._defCache.defaultTemplates
 		):
@@ -311,6 +312,11 @@ class DateDetector(object):
 					wrap=lambda s: RE_DATE_PREMATCH.sub(lambda m: DateTemplate.unboundPattern(s), preMatch))
 			# append date detector template (ignore duplicate if some was added before default):
 			self._appendTemplate(template, ignoreDup=ignoreDup)
+			cnt += 1
+		if preMatch:
+			logSys.info("  default date pattern for `%r`: %d template(s)", preMatch, cnt)
+		else:
+			logSys.info("  default %sdate pattern: %d template(s)", "filtered " if filterTemplate else "", cnt)
 
 	@property
 	def templates(self):

fail2ban/server/datetemplate.py

@@ -35,7 +35,7 @@ logSys = getLogger(__name__)
 # check already grouped contains "(", but ignores char "\(" and conditional "(?(id)...)":
 RE_GROUPED = re.compile(r'(?<!(?:\(\?))(?<!\\)\((?!\?)')
 RE_GROUP = ( re.compile(r'^((?:\(\?\w+\))?\^?(?:\(\?\w+\))?)(.*?)(\$?)$'), r"\1(\2)\3" )
-RE_GLOBALFLAGS = re.compile(r'((?:^|(?!<\\))\(\?[a-z]+\))')
+RE_GLOBALFLAGS = re.compile(r'((?:^|(?<!\\))\(\?[a-z]+\))')
 
 RE_EXLINE_NO_BOUNDS = re.compile(r'^\{UNB\}')
 RE_EXLINE_BOUND_BEG = re.compile(r'^\{\^LN-BEG\}')

fail2ban/server/filter.py

@@ -32,7 +32,7 @@ import time
 
 from .actions import Actions
 from .failmanager import FailManagerEmpty, FailManager
-from .ipdns import DNSUtils, IPAddr
+from .ipdns import DNSUtils, IPAddr, FileIPAddrSet
 from .observer import Observers
 from .ticket import FailTicket
 from .jailthread import JailThread
@@ -475,6 +475,10 @@ class Filter(JailThread):
 		# Generate the failure attempt for the IP:
 		unixTime = MyTime.time()
 		ticket = FailTicket(ip, unixTime, matches=matches)
+		# check it shall be ignored:
+		if self._inIgnoreIPList(ip, ticket):
+			return 0
+		# add attempt (found failure):
 		logSys.info(
 			"[%s] Attempt %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
 		)
@@ -482,7 +486,9 @@ class Filter(JailThread):
 		# Perform the ban if this attempt is resulted to:
 		if attempts >= self.failManager.getMaxRetry():
 			self.performBan(ip)
-
+		# report to observer - failure was found, for possibly increasing of it retry counter (asynchronous)
+		if Observers.Main is not None:
+			Observers.Main.add('failureFound', self.jail, ticket)
 		return 1
 
 	##
@@ -507,6 +513,12 @@ class Filter(JailThread):
 		# An empty string is always false
 		if ipstr == "":
 			return
+		# File?
+		ip = FileIPAddrSet.RE_FILE_IGN_IP.match(ipstr)
+		if ip:
+			ip = DNSUtils.getIPsFromFile(ip.group(1)) # FileIPAddrSet
+			self.__ignoreIpList.append(ip)
+			return
 		# Create IP address object
 		ip = IPAddr(ipstr)
 		# Avoid exact duplicates
@@ -529,6 +541,11 @@ class Filter(JailThread):
 			return
 		# delete by ip:
 		logSys.debug("  Remove %r from ignore list", ip)
+		# File?
+		if FileIPAddrSet.RE_FILE_IGN_IP.match(ip):
+			self.__ignoreIpList.remove(ip)
+			return
+		# IP / DNS
 		if ip in self.__ignoreIpSet:
 			self.__ignoreIpSet.remove(ip)
 		else:
@@ -585,7 +602,7 @@ class Filter(JailThread):
 			return True
 		for net in self.__ignoreIpList:
 			if ip.isInNet(net):
-				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns"))
+				self.logIgnoreIp(ip, log_ignore, ignore_source=(net.instanceType))
 				if self.__ignoreCache: c.set(key, True)
 				return True
 
@@ -1097,8 +1114,8 @@ class FileFilter(Filter):
 	def getFailures(self, filename, inOperation=None):
 		if self.idle: return False
 		log = self.getLog(filename)
-		if log is None:
-			logSys.error("Unable to get failures in %s", filename)
+		if log is None and self.active:
+			logSys.log(logging.MSG, "Unable to get failures in %s", filename)
 			return False
 		# We should always close log (file), otherwise may be locked (log-rotate, etc.)
 		try:
@@ -1274,24 +1291,15 @@ class FileFilter(Filter):
 				break
 			db.updateLog(self.jail, log)
 			
-	def onStop(self):
+	def afterStop(self):
 		"""Stop monitoring of log-file(s). Invoked after run method.
 		"""
-		# ensure positions of pending logs are up-to-date:
-		if self._pendDBUpdates and self.jail.database:
-			self._updateDBPending()
 		# stop files monitoring:
 		for path in list(self.__logs.keys()):
 			self.delLogPath(path)
-
-	def stop(self):
-		"""Stop filter
-		"""
-		# normally onStop will be called automatically in thread after its run ends, 
-		# but for backwards compatibilities we'll invoke it in caller of stop method.
-		self.onStop()
-		# stop thread:
-		super(Filter, self).stop()
+		# ensure positions of pending logs are up-to-date:
+		if self._pendDBUpdates and self.jail.database:
+			self._updateDBPending()
 
 ##
 # FileContainer class.

fail2ban/server/filterpyinotify.py

@@ -24,22 +24,18 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Lee Clemens, 2012 Y
 __license__ = "GPL"
 
 import logging
-from distutils.version import LooseVersion
 import os
 from os.path import dirname, sep as pathsep
 
-import pyinotify
-
 from .failmanager import FailManagerEmpty
 from .filter import FileFilter
 from .mytime import MyTime, time
 from .utils import Utils
 from ..helpers import getLogger
 
-
-if not hasattr(pyinotify, '__version__') \
-  or LooseVersion(pyinotify.__version__) < '0.8.3': # pragma: no cover
-  raise ImportError("Fail2Ban requires pyinotify >= 0.8.3")
+# pyinotify may have dependency to asyncore, so import it after helper to ensure 
+# we've a path to compat folder:
+import pyinotify
 
 # Verify that pyinotify is functional on this system
 # Even though imports -- might be dysfunctional, e.g. as on kfreebsd
@@ -371,19 +367,18 @@ class FilterPyinotify(FileFilter):
 				self.commonError("unhandled", e)
 			
 		logSys.debug("[%s] filter exited (pyinotifier)", self.jailName)
-		self.__notifier = None
+		self.done()
 
 		return True
 
 	##
-	# Call super.stop() and then stop the 'Notifier'
+	# Clean-up: then stop the 'Notifier'
 
-	def stop(self):
-		# stop filter thread:
-		super(FilterPyinotify, self).stop()
+	def afterStop(self):
 		try:
 			if self.__notifier: # stop the notifier
 				self.__notifier.stop()
+				self.__notifier = None
 		except AttributeError: # pragma: no cover
 			if self.__notifier: raise
 

fail2ban/server/filtersystemd.py

@@ -24,22 +24,65 @@ __license__ = "GPL"
 
 import os
 import time
-from distutils.version import LooseVersion
 
+from glob import glob
 from systemd import journal
-if LooseVersion(getattr(journal, '__version__', "0")) < '204':
-	raise ImportError("Fail2Ban requires systemd >= 204")
 
 from .failmanager import FailManagerEmpty
 from .filter import JournalFilter, Filter
 from .mytime import MyTime
 from .utils import Utils
-from ..helpers import getLogger, logging, splitwords, uni_decode
+from ..helpers import getLogger, logging, splitwords, uni_decode, _as_bool
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
 
+_systemdPathCache = Utils.Cache()
+def _getSystemdPath(path):
+	"""Get systemd path using systemd-path command (cached)"""
+	p = _systemdPathCache.get(path)
+	if p: return p
+	p = Utils.executeCmd('systemd-path %s' % path, timeout=10, shell=True, output=True)
+	if p and p[0]:
+		p = str(p[1].decode('utf-8')).split('\n')[0]
+		_systemdPathCache.set(path, p)
+		return p
+	p = '/var/log' if path == 'system-state-logs' else ('/run/log' if path == 'system-runtime-logs' else None)
+	_systemdPathCache.set(path, p)
+	return p
+
+def _globJournalFiles(flags=None, path=None):
+	"""Get journal files without rotated files."""
+	filesSet = set()
+	_join = os.path.join
+	def _addJF(filesSet, p, flags):
+		"""add journal files to set corresponding path and flags (without rotated *@*.journal)"""
+		# system journal:
+		if (flags is None) or (flags & journal.SYSTEM_ONLY):
+			filesSet |= set(glob(_join(p,'system.journal'))) - set(glob(_join(p,'system*@*.journal')))
+		# current user-journal:
+		if (flags is not None) and (flags & journal.CURRENT_USER):
+			uid = os.geteuid()
+			filesSet |= set(glob(_join(p,('user-%s.journal' % uid)))) - set(glob(_join(p,('user-%s@*.journal' % uid))))
+		# all local journals:
+		if (flags is None) or not (flags & (journal.SYSTEM_ONLY|journal.CURRENT_USER)):
+			filesSet |= set(glob(_join(p,'*.journal'))) - set(glob(_join(p,'*@*.journal')))
+	if path:
+		# journals relative given path only:
+		_addJF(filesSet, path, flags)
+	else:
+		# persistent journals corresponding flags:
+		if (flags is None) or not (flags & journal.RUNTIME_ONLY):
+			_addJF(filesSet, _join(_getSystemdPath('system-state-logs'), 'journal/*'), flags)
+		# runtime journals corresponding flags:
+		_addJF(filesSet, _join(_getSystemdPath('system-runtime-logs'), 'journal/*'), flags)
+	# if not root, filter readable only:
+	if os.geteuid() != 0:
+		filesSet = [f for f in filesSet if os.access(f, os.R_OK)]
+	return filesSet if filesSet else None
+
+
 ##
 # Journal reader class.
 #
@@ -55,12 +98,13 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	# @param jail the jail object
 
 	def __init__(self, jail, **kwargs):
-		jrnlargs = FilterSystemd._getJournalArgs(kwargs)
+		self.__jrnlargs = FilterSystemd._getJournalArgs(kwargs)
 		JournalFilter.__init__(self, jail, **kwargs)
 		self.__modified = 0
 		# Initialise systemd-journal connection
-		self.__journal = journal.Reader(**jrnlargs)
+		self.__journal = journal.Reader(**self.__jrnlargs)
 		self.__matches = []
+		self.__bypassInvalidateMsg = 0
 		self.setDatePattern(None)
 		logSys.debug("Created FilterSystemd")
 
@@ -77,31 +121,88 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		except KeyError:
 			pass
 		else:
-			import glob
 			p = args['files']
 			if not isinstance(p, (list, set, tuple)):
 				p = splitwords(p)
 			files = []
 			for p in p:
-				files.extend(glob.glob(p))
+				files.extend(glob(p))
 			args['files'] = list(set(files))
 
-		# Default flags is SYSTEM_ONLY(4). This would lead to ignore user session files,
-		# so can prevent "Too many open files" errors on a lot of user sessions (see gh-2392):
+		rotated = _as_bool(kwargs.pop('rotated', 0))
+		# Default flags is SYSTEM_ONLY(4) or LOCAL_ONLY(1), depending on rotated parameter. 
+		# This could lead to ignore user session files, so together with ignoring rotated
+		# files would prevent "Too many open files" errors on a lot of user sessions (see gh-2392):
 		try:
 			args['flags'] = int(kwargs.pop('journalflags'))
 		except KeyError:
 			# be sure all journal types will be opened if files/path specified (don't set flags):
-			if ('files' not in args or not len(args['files'])) and ('path' not in args or not args['path']):
-				args['flags'] = int(os.getenv("F2B_SYSTEMD_DEFAULT_FLAGS", 4))
-				
+			if (not args.get('files') and not args.get('path')):
+				args['flags'] = os.getenv("F2B_SYSTEMD_DEFAULT_FLAGS", None)
+				if args['flags'] is not None:
+					args['flags'] = int(args['flags'])
+				elif rotated:
+					args['flags'] = journal.SYSTEM_ONLY
+
 		try:
 			args['namespace'] = kwargs.pop('namespace')
 		except KeyError:
 			pass
 
+		# To avoid monitoring rotated logs, as prevention against "Too many open files",
+		# set the files to system.journal and user-*.journal (without rotated *@*.journal):
+		if not rotated and not args.get('files') and not args.get('namespace'):
+			args['files'] = _globJournalFiles(
+				args.get('flags', journal.LOCAL_ONLY), args.get('path'))
+			if args['files']:
+				args['files'] = list(args['files'])
+				# flags and path cannot be specified simultaneously with files:
+				args['flags'] = None;
+				args['path'] = None; 
+			else:
+				args['files'] = None
+
 		return args
 
+	@property
+	def _journalAlive(self):
+		"""Checks journal is online.
+		"""
+		try:
+			# open?
+			if self.__journal.closed: # pragma: no cover
+				return False
+			# has cursor? if it is broken (e. g. no descriptor) - it'd raise this:
+			# OSError: [Errno 99] Cannot assign requested address
+			if self.__journal._get_cursor():
+				return True
+		except OSError: # pragma: no cover
+			pass
+		return False
+
+	def _reopenJournal(self): # pragma: no cover
+		"""Reopen journal (if it becomes offline after rotation)
+		"""
+		if self.__journal.closed:
+			# recreate reader:
+			self.__journal = journal.Reader(**self.__jrnlargs)
+		else:
+			try:
+				# workaround for gh-3929 (no journal descriptor after rotation),
+				# to reopen journal we'd simply invoke inherited init again:
+				self.__journal.close()
+				ja = self.__jrnlargs
+				super(journal.Reader, self.__journal).__init__(
+					ja.get('flags', 0), ja.get('path'), ja.get('files'), ja.get('namespace'))
+			except:
+				# cannot reopen in that way, so simply recreate reader:
+				self.closeJournal()
+				self.__journal = journal.Reader(**self.__jrnlargs)
+		# restore journalmatch specified for the jail:
+		self.resetJournalMatches()
+		# just to avoid "Invalidate signaled" happening again after reopen:
+		self.__bypassInvalidateMsg = MyTime.time() + 1
+
 	##
 	# Add a journal match filters from list structure
 	#
@@ -260,6 +361,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 	def inOperationMode(self):
 		self.inOperation = True
 		logSys.info("[%s] Jail is in operation now (process new journal entries)", self.jailName)
+		# just to avoid "Invalidate signaled" happening often at start:
+		self.__bypassInvalidateMsg = MyTime.time() + 1
 
 	##
 	# Main loop.
@@ -317,6 +420,14 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 		while self.active:
 			# wait for records (or for timeout in sleeptime seconds):
 			try:
+				if self.idle:
+					# because journal.wait will returns immediately if we have records in journal,
+					# just wait a little bit here for not idle, to prevent hi-load:
+					if not Utils.wait_for(lambda: not self.active or not self.idle, 
+						self.sleeptime * 10, self.sleeptime
+					):
+						self.ticks += 1
+						continue
 				## wait for entries using journal.wait:
 				if wcode == journal.NOP and self.inOperation:
 					## todo: find better method as wait_for to break (e.g. notify) journal.wait(self.sleeptime),
@@ -331,8 +442,10 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 					## if invalidate (due to rotation, vacuuming or journal files added/removed etc):
 					if self.active and wcode == journal.INVALIDATE:
 						if self.ticks:
-							logSys.log(logging.DEBUG, "[%s] Invalidate signaled, take a little break (rotation ends)", self.jailName)
+							if not self.__bypassInvalidateMsg or MyTime.time() > self.__bypassInvalidateMsg:
+								logSys.log(logging.MSG, "[%s] Invalidate signaled, take a little break (rotation ends)", self.jailName)
 							time.sleep(self.sleeptime * 0.25)
+							self.__bypassInvalidateMsg = 0
 						Utils.wait_for(lambda: not self.active or \
 							self.__journal.wait(Utils.DEFAULT_SLEEP_INTERVAL) != journal.INVALIDATE,
 							self.sleeptime * 3, 0.00001)
@@ -343,14 +456,11 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 								if self.__journal.get_previous(): self.__journal.get_next()
 							except OSError:
 								pass
-				if self.idle:
-					# because journal.wait will returns immediately if we have records in journal,
-					# just wait a little bit here for not idle, to prevent hi-load:
-					if not Utils.wait_for(lambda: not self.active or not self.idle, 
-						self.sleeptime * 10, self.sleeptime
-					):
-						self.ticks += 1
-						continue
+						# if it is not alive - reopen:
+						if not self._journalAlive:
+							logSys.log(logging.MSG, "[%s] Journal reader seems to be offline, reopen journal", self.jailName)
+							self._reopenJournal()
+							wcode = journal.NOP
 				self.__modified = 0
 				while self.active:
 					logentry = None
@@ -411,8 +521,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 
 		logSys.debug("[%s] filter terminated", self.jailName)
 
-		# close journal:
-		self.closeJournal()
+		# call afterStop once (close journal, etc):
+		self.done()
 
 		logSys.debug("[%s] filter exited (systemd)", self.jailName)
 		return True
@@ -446,12 +556,10 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 				break
 			db.updateJournal(self.jail, log, *args)
 
-	def onStop(self):
-		"""Stop monitoring of journal. Invoked after run method.
-		"""
+	def afterStop(self):
+		"""Cleanup"""
 		# close journal:
 		self.closeJournal()
 		# ensure positions of pending logs are up-to-date:
 		if self._pendDBUpdates and self.jail.database:
 			self._updateDBPending()
-

fail2ban/server/ipdns.py

@@ -23,10 +23,11 @@ __license__ = "GPL"
 
 import socket
 import struct
+import os
 import re
 
 from .utils import Utils
-from ..helpers import getLogger
+from ..helpers import getLogger, MyTime, splitwords
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -79,6 +80,8 @@ class DNSUtils:
 	# todo: make configurable the expired time and max count of cache entries:
 	CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60)
 	CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60)
+	# static cache used to hold sets read from files:
+	CACHE_fileToIp = Utils.Cache(maxCount=100, maxTime=5*60)
 
 	@staticmethod
 	def dnsToIp(dns):
@@ -229,6 +232,20 @@ class DNSUtils:
 		DNSUtils.CACHE_nameToIp.set(DNSUtils._getSelfIPs_key, ips)
 		return ips
 
+	@staticmethod
+	def getIPsFromFile(fileName, noError=True):
+		"""Get set of IP addresses or subnets from file"""
+		# to find cached IPs:
+		ips = DNSUtils.CACHE_fileToIp.get(fileName)
+		if ips is not None:
+			return ips
+		# try to obtain set from file:
+		ips = FileIPAddrSet(fileName)
+		#ips.load() - load on demand
+		# cache and return :
+		DNSUtils.CACHE_fileToIp.set(fileName, ips)
+		return ips
+
 	_IPv6IsAllowed = None
 
 	@staticmethod
@@ -457,6 +474,10 @@ class IPAddr(object):
 	def familyStr(self):
 		return IPAddr.FAM2STR.get(self._family)
 
+	@property
+	def instanceType(self):
+		return "ip" if self.isValid else "dns"
+
 	@property
 	def plen(self):
 		return self._plen
@@ -598,6 +619,9 @@ class IPAddr(object):
 	def isInNet(self, net):
 		"""Return either the IP object is in the provided network
 		"""
+		# if addr-set:
+		if isinstance(net, IPAddrSet):
+			return self in net
 		# if it isn't a valid IP address, try DNS resolution
 		if not net.isValid and net.raw != "":
 			# Check if IP in DNS
@@ -675,15 +699,32 @@ IPAddr.IP6_4COMPAT = IPAddr("::ffff:0:0", 96)
 
 class IPAddrSet(set):
 
-	hasSubNet = False
+	hasSubNet = 0
 
 	def __init__(self, ips=[]):
+		ips, subnet = IPAddrSet._list2set(ips)
+		set.__init__(self, ips)
+		self.hasSubNet = subnet
+
+	@staticmethod
+	def _list2set(ips):
 		ips2 = set()
+		subnet = 0
 		for ip in ips:
 			if not isinstance(ip, IPAddr): ip = IPAddr(ip)
 			ips2.add(ip)
-			self.hasSubNet |= not ip.isSingle
-		set.__init__(self, ips2)
+			subnet += not ip.isSingle
+		return ips2, subnet
+
+	@property
+	def instanceType(self):
+		return "ip-set"
+
+	def set(self, ips):
+		ips, subnet = IPAddrSet._list2set(ips)
+		self.clear()
+		self.update(ips)
+		self.hasSubNet = subnet
 
 	def add(self, ip):
 		if not isinstance(ip, IPAddr): ip = IPAddr(ip)
@@ -696,6 +737,79 @@ class IPAddrSet(set):
 		return set.__contains__(self, ip) or (self.hasSubNet and any(n.contains(ip) for n in self))
 
 
+class FileIPAddrSet(IPAddrSet):
+
+	# RE matching file://... (absolute as well as relative file name)
+	RE_FILE_IGN_IP = re.compile(r'^file:(?:/{0,2}(?=/(?!/|.{1,2}/))|/{0,2})(.*)$')
+
+	fileName = ''
+	_reprName = None
+	maxUpdateLatency = 1 # latency in seconds to update by changes
+	_nextCheck = 0
+	_fileStats = ()
+
+	def __init__(self, fileName=''):
+		self.fileName = fileName
+		# self.load() - lazy load on demand by first check (in, __contains__ etc)
+
+	@property
+	def instanceType(self):
+		return repr(self)
+
+	def __eq__(self, other):
+		if id(self) == id(other): return 1
+		# to allow remove file-set from list (delIgnoreIP) by its name:
+		if isinstance(other, FileIPAddrSet):
+			return self.fileName == other.fileName
+		m = FileIPAddrSet.RE_FILE_IGN_IP.match(other)
+		if m:
+			return self.fileName == m.group(1)
+
+	def _isModified(self):
+		"""Check whether the file is modified (file stats changed)
+
+		Side effect: if modified, _fileStats will be updated to last known stats of file
+		"""
+		tm = MyTime.time()
+		# avoid to check it always (not often than maxUpdateLatency):
+		if tm <= self._nextCheck:
+			return None; # no check needed
+		self._nextCheck = tm + self.maxUpdateLatency
+		stats = os.stat(self.fileName)
+		stats = stats.st_mtime, stats.st_ino, stats.st_size
+		if self._fileStats != stats:
+			self._fileStats = stats
+			return True; # modified, needs to be reloaded
+		return False; # unmodified
+
+	def load(self, forceReload=False, noError=True):
+		"""Load set from file (on demand if needed or by forceReload)
+		"""
+		try:
+			# load only if needed and modified (or first time load on demand)
+			if self._isModified() or forceReload:
+				with open(self.fileName, 'r') as f:
+					ips = f.read()
+				ips = splitwords(ips, ignoreComments=True)
+				self.set(ips)
+		except Exception as e: # pragma: no cover
+			self._nextCheck += 60; # increase interval to check (to 1 minute, to avoid log flood on errors)
+			if not noError: raise e
+			logSys.warning("Retrieving IPs set from %r failed: %s", self.fileName, e)
+
+	def __repr__(self):
+		if self._reprName is None:
+			self._reprName = 'file:' + ('/' if self.fileName.startswith('/') else '') + self.fileName
+		return self._reprName
+
+	def __contains__(self, ip):
+		# load if needed:
+		if self.fileName:
+			self.load()
+		# inherited contains:
+		return IPAddrSet.__contains__(self, ip)
+
+
 def _NetworkInterfacesAddrs(withMask=False):
 
 	# Closure implementing lazy load modules and libc and define _NetworkInterfacesAddrs on demand:

fail2ban/server/jail.py

@@ -335,7 +335,9 @@ class Jail(object):
 			try:
 				## signal to stop filter / actions:
 				if stop:
-					obj.stop()
+					if obj.isAlive():
+						obj.stop()
+					obj.done(); # and clean-up everything
 				## wait for end of threads:
 				if join:
 					obj.join()

fail2ban/server/jailthread.py

@@ -103,7 +103,21 @@ class JailThread(Thread):
 	def stop(self):
 		"""Sets `active` property to False, to flag run method to return.
 		"""
-		self.active = False
+		if self.active: self.active = False
+		# normally onStop will be called automatically in thread after its run ends,
+		# but for backwards compatibilities we'll invoke it in caller of stop method.
+		self.onStop()
+		self.onStop = lambda:()
+		self.done()
+
+	def done(self):
+		self.done = lambda:()
+		# if still runniung - wait a bit before initiate clean-up:
+		if self.is_alive():
+			Utils.wait_for(lambda: not self.is_alive(), 5)
+		# now clean-up everything:
+		self.afterStop()
+
 
 	@abstractmethod
 	def run(self): # pragma: no cover - absract
@@ -111,11 +125,15 @@ class JailThread(Thread):
 		"""
 		pass
 
+	def afterStop(self):
+		"""Cleanup resources."""
+		pass
+
 	def join(self):
 		""" Safer join, that could be called also for not started (or ended) threads (used for cleanup).
 		"""
-		## if cleanup needed - create derivate and call it before join...
-
+		## if cleanup needed - create derivative and call it before join...
+		self.done()
 		## if was really started - should call join:
 		if self.active is not None:
 			super(JailThread, self).join()

fail2ban/server/transmitter.py

@@ -353,7 +353,6 @@ class Transmitter:
 			return self.__server.getBanTime(name)
 		elif command[1] == "attempt":
 			value = command[2:]
-			if self.__quiet: return
 			return self.__server.addAttemptIP(name, *value)
 		elif command[1].startswith("bantime."):
 			value = command[2]

fail2ban/server/utils.py

@@ -29,6 +29,7 @@ import subprocess
 import sys
 from	 threading import Lock
 import time
+import types
 from ..helpers import getLogger, _merge_dicts, uni_decode
 from collections import OrderedDict
 
@@ -352,6 +353,7 @@ class Utils():
 	def load_python_module(pythonModule):
 		pythonModuleName = os.path.splitext(
 			os.path.basename(pythonModule))[0]
-		mod = importlib.machinery.SourceFileLoader(
-			pythonModuleName, pythonModule).load_module()
+		ldr = importlib.machinery.SourceFileLoader(pythonModuleName, pythonModule)
+		mod = types.ModuleType(ldr.name)
+		ldr.exec_module(mod)
 		return mod

fail2ban/tests/action_d/test_smtp.py

@@ -22,6 +22,7 @@ import threading
 import unittest
 import re
 import sys
+import types
 import importlib
 
 from ..dummyjail import DummyJail
@@ -175,7 +176,8 @@ try:
 			super(SMTPActionTest, self).tearDown()
 
 except ImportError as e:
-	print("I: Skipping smtp tests: %s" % e)
+	if tuple(sys.version_info) <= (3, 11):
+		print("I: Skipping smtp tests: %s" % e)
 
 
 try:
@@ -295,10 +297,11 @@ try:
 			self.jail = DummyJail()
 			pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py")
 			pythonModuleName = os.path.basename(pythonModule.rstrip(".py"))
-			customActionModule = importlib.machinery.SourceFileLoader(
-				pythonModuleName, pythonModule).load_module()
+			ldr = importlib.machinery.SourceFileLoader(pythonModuleName, pythonModule)
+			mod = types.ModuleType(ldr.name)
+			ldr.exec_module(mod)
 
-			self.action = customActionModule.Action(
+			self.action = mod.Action(
 				self.jail, "test", host="localhost:%i" % self.port)
 
 			self.action.ssl = True
@@ -309,4 +312,5 @@ try:
 			super(AIOSMTPActionTest, self).tearDown()
 	
 except ImportError as e:
-	print("I: Skipping SSL smtp tests: %s" % e)
+	if tuple(sys.version_info) >= (3, 10):
+		print("I: Skipping SSL smtp tests: %s" % e)

fail2ban/tests/actionstestcase.py

@@ -66,7 +66,7 @@ class ExecuteActions(LogCaptureTestCase):
 		self.__actions.add('test')
 		self.assertTrue(self.__actions['test'])
 		self.assertIn('test', self.__actions)
-		self.assertNotIn('nonexistant action', self.__actions)
+		self.assertNotIn('nonexistent action', self.__actions)
 		self.__actions.add('test1')
 		del self.__actions['test']
 		del self.__actions['test1']
@@ -287,7 +287,7 @@ class ExecuteActions(LogCaptureTestCase):
 		# no flush for inet4 (already successfully flushed):
 		self.assertNotLogged("ERROR",
 			"stdout: %r" % 'ip flush inet4',
-			'Unban tickets each individualy',
+			'Unban tickets each individually',
 			all=True)
 
 	def testActionsConsistencyCheckDiffFam(self):
@@ -401,7 +401,7 @@ class ExecuteActions(LogCaptureTestCase):
 		# no flush for inet4 (already successfully flushed):
 		self.assertNotLogged("ERROR",
 			"stdout: %r" % 'ip flush inet4',
-			'Unban tickets each individualy',
+			'Unban tickets each individually',
 			all=True)
 
 	@with_alt_time

fail2ban/tests/clientbeautifiertestcase.py

@@ -25,7 +25,7 @@ import unittest
 
 from ..client.beautifier import Beautifier
 from ..version import version
-from ..server.ipdns import IPAddr
+from ..server.ipdns import IPAddr, FileIPAddrSet
 from ..exceptions import UnknownJailException, DuplicateJailException
 
 class BeautifierTest(unittest.TestCase):
@@ -34,6 +34,7 @@ class BeautifierTest(unittest.TestCase):
 		""" Call before every test case """
 		super(BeautifierTest, self).setUp()
 		self.b = Beautifier()
+		self.b.encUtf = 0; ## we prefer ascii in test suite (see #3750)
 
 	def tearDown(self):
 		""" Call after every test case """
@@ -170,22 +171,25 @@ class BeautifierTest(unittest.TestCase):
 
 	def testStatusStats(self):
 		self.b.setInputCmd(["stats"])
+		## no jails:
+		self.assertEqual(self.b.beautify({}), "No jails found.")
+		## 3 jails:
 		response = {
 			"ssh": ["systemd", (3, 6), (12, 24)],
 			"exim4": ["pyinotify", (6, 12), (20, 20)],
 			"jail-with-long-name": ["polling", (0, 0), (0, 0)]
 		}
 		output = (""
-			+ "                    ?           ? Filter    ? Actions  \n"
-			+ "Jail                ? Backend   ????????????????????????\n"
-			+ "                    ?           ? cur ? tot ? cur ? tot\n"
-			+ "????????????????????????????????????????????????????????\n"
-			+ "ssh                 ? systemd   ?   3 ?   6 ?  12 ?  24\n"
-			+ "exim4               ? pyinotify ?   6 ?  12 ?  20 ?  20\n"
-			+ "jail-with-long-name ? polling   ?   0 ?   0 ?   0 ?   0\n"
-			+ "????????????????????????????????????????????????????????"
+			+ "                     |           | Filter    | Actions  \n"
+			+ " Jail                | Backend   |-----------x-----------\n"
+			+ "                     |           | cur | tot | cur | tot\n"
+			+ "---------------------x-----------x-----------x-----------\n"
+			+ " ssh                 | systemd   |   3 |   6 |  12 |  24\n"
+			+ " exim4               | pyinotify |   6 |  12 |  20 |  20\n"
+			+ " jail-with-long-name | polling   |   0 |   0 |   0 |   0\n"
+			+ "---------------------------------------------------------"
 		)
-		response = self.b.beautify(response).encode('ascii', 'replace').decode('ascii')
+		response = self.b.beautify(response)
 		self.assertEqual(response, output)
 			
 
@@ -293,6 +297,13 @@ class BeautifierTest(unittest.TestCase):
 		output += "`- 10.0.2.1"
 		self.assertEqual(self.b.beautify(response), output)
 
+	def testIgnoreIPFile(self):
+		self.b.setInputCmd(["set", "sshd", "addignoreip"])
+		response = [FileIPAddrSet("/test/file-ipaddr-set")]
+		output = ("These IP addresses/networks are ignored:\n"
+			"`- file://test/file-ipaddr-set")
+		self.assertEqual(self.b.beautify(response), output)
+
 	def testFailRegex(self):
 		self.b.setInputCmd(["get", "sshd", "failregex"])
 		output = "No regular expression is defined"

fail2ban/tests/clientreadertestcase.py

@@ -564,7 +564,7 @@ class FilterReaderTest(LogCaptureTestCase):
 		self.assertNotEqual(opts['maxlines'], 'X'); # wrong int value 'X' for 'maxlines'
 		self.assertLogged("Wrong int value 'X' for 'maxlines'. Using default one:")
 
-	def testFilterReaderSubstitionDefault(self):
+	def testFilterReaderSubstitutionDefault(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=<IP>']]
 		filterReader = FilterReader('substitution', "jailname", {},
 		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
@@ -584,7 +584,7 @@ class FilterReaderTest(LogCaptureTestCase):
 		opts = filterReader.getCombined()
 		self.assertTrue('sshd' in opts['failregex'])
 		
-	def testFilterReaderSubstitionSet(self):
+	def testFilterReaderSubstitutionSet(self):
 		output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=<IP>']]
 		filterReader = FilterReader('substitution', "jailname", {'honeypot': 'sour@example.com'},
 		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
@@ -593,7 +593,7 @@ class FilterReaderTest(LogCaptureTestCase):
 		c = filterReader.convert()
 		self.assertSortedEqual(c, output)
 
-	def testFilterReaderSubstitionKnown(self):
+	def testFilterReaderSubstitutionKnown(self):
 		output = [['set', 'jailname', 'addfailregex', '^to=test,sweet@example.com,test2,sweet@example.com fromip=<IP>$']]
 		filterName, filterOpt = extractOptions(
 			'substitution[failregex="^<known/failregex>$", honeypot="<sweet>,<known/honeypot>", sweet="test,<known/honeypot>,test2"]')
@@ -604,7 +604,7 @@ class FilterReaderTest(LogCaptureTestCase):
 		c = filterReader.convert()
 		self.assertSortedEqual(c, output)
 
-	def testFilterReaderSubstitionSection(self):
+	def testFilterReaderSubstitutionSection(self):
 		output = [['set', 'jailname', 'addfailregex', '^\\s*to=fail2ban@localhost fromip=<IP>\\s*$']]
 		filterName, filterOpt = extractOptions(
 			'substitution[failregex="^\\s*<Definition/failregex>\\s*$", honeypot="<default/honeypot>"]')
@@ -615,7 +615,7 @@ class FilterReaderTest(LogCaptureTestCase):
 		c = filterReader.convert()
 		self.assertSortedEqual(c, output)
 
-	def testFilterReaderSubstitionFail(self):
+	def testFilterReaderSubstitutionFail(self):
 		# directly subst the same var :
 		filterReader = FilterReader('substitution', "jailname", {'honeypot': '<honeypot>'},
 		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
@@ -719,50 +719,67 @@ class JailsReaderTest(LogCaptureTestCase):
 		jails = JailsReader(basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG)
 		self.assertTrue(jails.read())
 		self.assertFalse(jails.getOptions(ignoreWrong=False))
-		self.assertRaises(ValueError, jails.convert)
-		comm_commands = jails.convert(allow_no_files=True)
-		self.maxDiff = None
-		self.assertSortedEqual(comm_commands,
-			[['add', 'emptyaction', 'auto'],
-			 ['add', 'test-known-interp', 'auto'],
-			 ['multi-set', 'test-known-interp', 'addfailregex', [
-			   'failure test 1 (filter.d/test.conf) <HOST>',
-			   'failure test 2 (filter.d/test.local) <HOST>',
-			   'failure test 3 (jail.local) <HOST>'
-			 ]],
-			 ['start', 'test-known-interp'],
-			 ['add', 'missinglogfiles', 'auto'],
-			 ['set', 'missinglogfiles', 'addfailregex', '<IP>'],
-			 ['add', 'brokenaction', 'auto'],
-			 ['set', 'brokenaction', 'addfailregex', '<IP>'],
-			 ['set', 'brokenaction', 'addaction', 'brokenaction'],
-			 ['multi-set', 'brokenaction', 'action', 'brokenaction', [
-				 ['actionban', 'hit with big stick <ip>'],
-				 ['actname', 'brokenaction'],
-				 ['name', 'brokenaction']
-			 ]],
-			 ['add', 'parse_to_end_of_jail.conf', 'auto'],
-			 ['set', 'parse_to_end_of_jail.conf', 'addfailregex', '<IP>'],
-			 ['set', 'tz_correct', 'addfailregex', '<IP>'],
-			 ['set', 'tz_correct', 'logtimezone', 'UTC+0200'],
-			 ['start', 'emptyaction'],
-			 ['start', 'missinglogfiles'],
-			 ['start', 'brokenaction'],
-			 ['start', 'parse_to_end_of_jail.conf'],
-		         ['add', 'tz_correct', 'auto'],
-			 ['start', 'tz_correct'],
-			 ['config-error',
-				"Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo': unexpected option syntax"],
-			 ['config-error',
-				"Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test': unexpected option syntax"],
-			 ['config-error',
-				"Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"],
-			 ['config-error',
-				"Jail 'missingbitsjail' skipped, because of wrong configuration: Unable to read the filter 'catchallthebadies'"],
-			 ])
+		self.assertRaises(ValueError, lambda: jails.convert(systemd_if_nologs=False))
 		self.assertLogged("Errors in jail 'missingbitsjail'.")
 		self.assertNotLogged("Skipping...")
+		# check with allow no files (just to cover other jail problems), but without switch to systemd:
+		self.pruneLog('[test-phase] allow no files, no switch to systemd ...')
+		comm_commands = jails.convert(allow_no_files=True, systemd_if_nologs=False)
+		self.maxDiff = None
+		def _checkStream(comm_commands, backend='auto'):
+			self.assertSortedEqual(comm_commands,
+				[['add', 'emptyaction', 'auto'],
+				 ['add', 'test-known-interp', 'auto'],
+				 ['multi-set', 'test-known-interp', 'addfailregex', [
+				   'failure test 1 (filter.d/test.conf) <HOST>',
+				   'failure test 2 (filter.d/test.local) <HOST>',
+				   'failure test 3 (jail.local) <HOST>'
+				 ]],
+				 ['start', 'test-known-interp'],
+				 ['add', 'missinglogfiles', backend], # can switch backend because have journalmatch
+				 ['set', 'missinglogfiles', 'addfailregex', '<IP>'],
+				 ['set', 'missinglogfiles', 'addjournalmatch', '_COMM=test'],
+				 ['config-error', "Jail 'missinglogfiles_skip' skipped, because of missing log files."],
+				 ['add', 'brokenaction', 'auto'],
+				 ['set', 'brokenaction', 'addfailregex', '<IP>'],
+				 ['set', 'brokenaction', 'addaction', 'brokenaction'],
+				 ['multi-set', 'brokenaction', 'action', 'brokenaction', [
+					 ['actionban', 'hit with big stick <ip>'],
+					 ['actname', 'brokenaction'],
+					 ['name', 'brokenaction']
+				 ]],
+				 ['add', 'parse_to_end_of_jail.conf', 'auto'],
+				 ['set', 'parse_to_end_of_jail.conf', 'addfailregex', '<IP>'],
+				 ['set', 'tz_correct', 'addfailregex', '<IP>'],
+				 ['set', 'tz_correct', 'logtimezone', 'UTC+0200'],
+				 ['start', 'emptyaction'],
+				 ['start', 'missinglogfiles'],
+				 ['start', 'brokenaction'],
+				 ['start', 'parse_to_end_of_jail.conf'],
+			   ['add', 'tz_correct', 'auto'],
+				 ['start', 'tz_correct'],
+				 ['config-error',
+					"Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo': unexpected option syntax"],
+				 ['config-error',
+					"Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test': unexpected option syntax"],
+				 ['config-error',
+					"Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"],
+				 ['config-error',
+					"Jail 'missingbitsjail' skipped, because of wrong configuration: Unable to read the filter 'catchallthebadies'"],
+				 ])
+		_checkStream(comm_commands, backend='auto')
+		self.assertNotLogged("Have not found any log file for 'missinglogfiles' jail. Jail will monitor systemd journal.")
 		self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction")
+		self.assertLogged("Jail 'missinglogfiles_skip' skipped, because of missing log files.")
+		# switch backend auto to systemd if no files found, note that jail "missinglogfiles_skip" will be skipped yet,
+		# because for this jail configured skip_if_nologs = true, all other jails shall switch to systemd with warning
+		self.pruneLog('[test-phase] auto -> systemd')
+		comm_commands = jails.convert(allow_no_files=True)
+		_checkStream(comm_commands, backend='systemd')
+		self.assertNotLogged("Errors in jail 'missingbitsjail'.")
+		self.assertLogged("Have not found any log file for 'missinglogfiles' jail. Jail will monitor systemd journal.")
+		self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction")
+		self.assertLogged("Jail 'missinglogfiles_skip' skipped, because of missing log files.")
 
 	def testReadStockActionConf(self):
 		unittest.F2B.SkipIfCfgMissing(stock=True)
@@ -861,7 +878,7 @@ class JailsReaderTest(LogCaptureTestCase):
 		self.assertTrue(jails.getOptions())       # reads fine
 		# grab all filter names
 		filters = set(os.path.splitext(os.path.split(a)[1])[0]
-			for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
+			for a in glob.glob(os.path.join(CONFIG_DIR, 'filter.d', '*.conf'))
 				if not (a.endswith('common.conf') or a.endswith('-aggressive.conf')))
 		# get filters of all jails (filter names without options inside filter[...])
 		filters_jail = set(
@@ -1022,6 +1039,11 @@ filter = testfilter1
 		self.assertRaisesRegex(ValueError, r"Have not found any log file for .* jail", 
 			self._testLogPath, backend='polling')
 
+	def testLogPathSkipJailIfNoLogs(self):
+		s = self._testLogPath(backend='polling', skip_if_nologs=True)
+		self.assertLogged('Have not found any log file for')
+		self.assertEqual(s, [['config-error', "Jail 'testjail1' skipped, because of missing log files."]])
+
 	def testLogPathSystemdBackend(self):
 		try: # pragma: systemd no cover
 			from ..server.filtersystemd import FilterSystemd
@@ -1031,7 +1053,7 @@ filter = testfilter1
 		self._testLogPath(backend='systemd[journalflags=2]')
 	
 	@with_tmpdir
-	def _testLogPath(self, basedir, backend):
+	def _testLogPath(self, basedir, backend, skip_if_nologs=False):
 		jailfd = open(os.path.join(basedir, "jail.conf"), 'w')
 		jailfd.write("""
 [testjail1]
@@ -1043,8 +1065,10 @@ action =
 filter = 
 failregex = test <HOST>
 """ % (backend, basedir))
+		if skip_if_nologs:
+			jailfd.write("skip_if_nologs = true\n")
 		jailfd.close()
 		jails = JailsReader(basedir=basedir)
 		self.assertTrue(jails.read())
 		self.assertTrue(jails.getOptions())
-		jails.convert()
+		return jails.convert()

fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf

@@ -9,7 +9,7 @@ before = ../../../../config/filter.d/common.conf
 
 [DEFAULT]
 
-_daemon = sshd
+_daemon = sshd(?:-session)?
 
 # optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: "
 __pref = (?:(?:error|fatal): (?:PAM: )?)?

fail2ban/tests/config/jail.conf

@@ -21,6 +21,12 @@ failregex = %(known/failregex)s
 
 [missinglogfiles]
 enabled = true
+journalmatch = _COMM=test ;# allow to switch to systemd (by backend = `auto` and no logs found)
+logpath = /weapons/of/mass/destruction
+
+[missinglogfiles_skip]
+enabled = true
+skip_if_nologs = true
 logpath = /weapons/of/mass/destruction
 
 [brokenactiondef]

fail2ban/tests/datedetectortestcase.py

@@ -580,7 +580,7 @@ class CustomDateFormatsTest(unittest.TestCase):
 			else: # pragma: no cover
 				self.assertEqual(date, None)
 
-#	def testDefaultTempate(self):
+#	def testDefaultTemplate(self):
 #		self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 #		self.__datedetector.setDefaultPattern("%b %d %H:%M:%S")
 #		

fail2ban/tests/fail2banclienttestcase.py

@@ -826,6 +826,23 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"Errors in jail 'broken-jail'.",
 			"ERROR: test configuration failed", all=True)
 
+		# disable jail in .local (shall be again OK):
+		self.pruneLog("[test-phase 1]")
+		_write_file(pjoin(cfg, "jail.local"), "a", "",
+			"[broken-jail]", "enabled = false")
+		self.execCmd(SUCCESS, startparams, "--test")
+		self.assertLogged("OK: configuration test is successful")
+
+		# generate decoding error: ('utf-8' codec can't decode byte 0xfd):
+		self.pruneLog("[test-phase 1a]")
+		with open(pjoin(cfg, "jail.local"), "ab") as f:
+			f.write(b"\n# invalid char \xfd")
+		self.execCmd(FAILED, startparams, "-t")
+		self.assertLogged("Could not read config files",
+			"Read jails configuration failed.",
+			"ERROR: test configuration failed", all=True)
+
+
 	@with_tmpdir
 	def testKillAfterStart(self, tmp):
 		try:
@@ -1108,9 +1125,9 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"3 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME)
 		# stop/start and unban/restore ban:
 		self.assertLogged(
-			"[test-jail2] Unban 192.0.2.4",
-			"[test-jail2] Unban 192.0.2.8",
-			"[test-jail2] Unban 192.0.2.9",
+			"[test-jail2] Repeal Ban 192.0.2.4",
+			"[test-jail2] Repeal Ban 192.0.2.8",
+			"[test-jail2] Repeal Ban 192.0.2.9",
 			"Jail 'test-jail2' stopped",
 			"Jail 'test-jail2' started",
 			"[test-jail2] Restore Ban 192.0.2.4",

fail2ban/tests/fail2banregextestcase.py

@@ -148,7 +148,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertLogged("Unable to compile regular expression")
 		self.assertLogged("unknown group name", "at position 23", all=False); # details of failed compilation
 
-	def testWrongIngnoreRE(self):
+	def testWrongIgnoreRE(self):
 		self.assertFalse(_test_exec(
 			"--datepattern", "{^LN-BEG}EPOCH",
 			"test", r".*? from <HOST>$", r".**"
@@ -316,6 +316,20 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		"-l", "notice", # put down log-level, because of too many debug-messages
 			FILENAME_ZZZ_GEN, FILTER_ZZZ_GEN+"[mode=test]"
 		))
+		self.assertLogged("Ignoreregex: 2 total",
+			"Lines: 23 lines, 2 ignored, 16 matched, 5 missed", all=True)
+		# cover filter ignoreregex gets overwritten by command argument:
+		self.pruneLog("[test-phase 2]")
+		self.assertTrue(_test_exec(
+		"-l", "notice", # put down log-level, because of too many debug-messages
+			"[Jun 21 16:56:03] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.251\n"
+			"[Jun 21 16:56:04] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.252\n"
+			"[Jun 21 16:56:05] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.255\n",
+			FILTER_ZZZ_GEN+"[mode=test]",
+			"F2B: error from 192.0.2.255$"
+		))
+		self.assertLogged("Use ignoreregex line", "Ignoreregex: 1 total",
+			"Lines: 3 lines, 1 ignored, 2 matched, 0 missed", all=True)
 
 	def testDirectMultilineBuf(self):
 		# test it with some pre-lines also to cover correct buffer scrolling (all multi-lines printed):
@@ -418,8 +432,16 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertLogged('output: %s' % "['192.0.2.0'", "'ip4': '192.0.2.0'", "'user': 'kevin'", all=True)
 		self.pruneLog()
 		# log msg :
-		self.assertTrue(_test_exec('-o', 'msg', STR_00, RE_00_USER))
+		nmline = "Dec 31 12:00:00 [sshd] error: PAM: No failure for user from 192.0.2.123"
+		lines = STR_00+"\n"+nmline
+		self.assertTrue(_test_exec('-o', 'msg', lines, RE_00_USER))
 		self.assertLogged('output: %s' % STR_00)
+		self.assertNotLogged('output: %s' % nmline)
+		self.pruneLog()
+		# log msg (inverted) :
+		self.assertTrue(_test_exec('-o', 'msg', '-i', lines, RE_00_USER))
+		self.assertLogged('output: %s' % nmline)
+		self.assertNotLogged('output: %s' % STR_00)
 		self.pruneLog()
 		# item of match (user):
 		self.assertTrue(_test_exec('-o', 'user', STR_00, RE_00_USER))
@@ -429,6 +451,17 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <family>', STR_00, RE_00_USER))
 		self.assertLogged('output: %s' % '192.0.2.0, kevin, inet4')
 		self.pruneLog()
+		# log msg :
+		lines = nmline+"\n"+STR_00; # just reverse lines (to cover possible order dependencies)
+		self.assertTrue(_test_exec('-o', '<time> : <msg>', lines, RE_00_USER))
+		self.assertLogged('output: %s : %s' % (1104490799.0, STR_00))
+		self.assertNotLogged('output: %s' % nmline)
+		self.pruneLog()
+		# log msg (inverted) :
+		self.assertTrue(_test_exec('-o', '<time> : <msg>', '-i', lines, RE_00_USER))
+		self.assertLogged('output: %s : %s' % (1104490800.0, nmline))
+		self.assertNotLogged('output: %s' % STR_00)
+		self.pruneLog()
 
 	def testStalledIPByNoFailFrmtOutput(self):
 		opts = (
@@ -567,7 +600,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 			FILENAME_ZZZ_GEN, FILENAME_ZZZ_GEN
 		))
 
-	def testWronChar(self):
+	def testWrongChar(self):
 		unittest.F2B.SkipIfCfgMissing(stock=True)
 		self.assertTrue(_test_exec(
 		"-l", "notice", # put down log-level, because of too many debug-messages
@@ -582,7 +615,7 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertLogged('Nov  8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco')
 		self.assertLogged('Nov  8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco')
 
-	def testWronCharDebuggex(self):
+	def testWrongCharDebuggex(self):
 		unittest.F2B.SkipIfCfgMissing(stock=True)
 		self.assertTrue(_test_exec(
 		"-l", "notice", # put down log-level, because of too many debug-messages

fail2ban/tests/files/config/apache-auth/README

@@ -4,7 +4,7 @@ Apache Auth.
 This directory contains the configuration file of Apache's Web Server to
 simulate authentication files.
 
-These assumed that /var/www/html is the web root and AllowOverides is "All".
+These assumed that /var/www/html is the web root and AllowOverrides is "All".
 
 The subdirectories here are copied to the /var/www/html directory.
 

fail2ban/tests/files/logs/apache-badbots

@@ -6,3 +6,6 @@
 
 # failJSON: { "time": "2007-03-05T14:41:21", "match": true , "host": "1.2.3.4" }
 1.2.3.4 - - [05/Mar/2007:14:41:21 +0100] "HEAD /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02"
+
+# failJSON: { "time": "2024-08-18T22:08:39", "match": true , "host": "192.0.2.222", "desc": "vhost in accesslog, gh-1594" }
+www.sitename.com 192.0.2.222 - - [18/Aug/2024:21:08:39 +0100] "GET /filename.jpg HTTP/1.1" 403 332 "-" "TrackBack/1.02"

fail2ban/tests/files/logs/apache-fakegooglebot

@@ -1,5 +1,7 @@
 # Apache 2.2
 # failJSON: { "time": "2015-01-31T14:29:44", "match": true, "host": "66.249.66.1" }
-66.249.66.1 - - - [31/Jan/2015:14:29:44 ] example.com "GET / HTTP/1.1" 200 814 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + 293 1149 546
-# failJSON: { "time": "2015-01-31T14:29:44", "match": false, "host": "93.184.215.14" }
-93.184.215.14 - - - [31/Jan/2015:14:29:44 ] example.com "GET / HTTP/1.1" 200 814 "-" "NOT A __GOOGLE_BOT__" + 293 1149 546
+66.249.66.1 - - - [31/Jan/2015:14:29:44 ] fail2ban.org "GET / HTTP/1.1" 200 814 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + 293 1149 546
+# failJSON: { "time": "2015-01-31T14:29:44", "match": false, "host": "51.159.55.100" }
+51.159.55.100 - - - [31/Jan/2015:14:29:44 ] fail2ban.org "GET / HTTP/1.1" 200 814 "-" "NOT A __GOOGLE_BOT__" + 293 1149 546
+# failJSON: { "time": "2024-08-18T22:08:39", "match": true , "host": "192.0.2.222", "desc": "vhost in accesslog, gh-1594" }
+www.sitename.com 192.0.2.222 - - [18/Aug/2024:21:08:39 +0100] "GET / HTTP/1.1" 403 332 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"

fail2ban/tests/files/logs/apache-noscript

@@ -23,3 +23,6 @@
 
 # failJSON: { "time": "2020-08-11T08:56:17", "match": true , "host": "192.0.2.1", "desc": "script not found with AH02811 and cgi-bin path segment in script (gh-2805)" }
 [Tue Aug 11 08:56:17.580412 2020] [cgi:error] [pid 27550:tid 140110750279424] [client 192.0.2.1:18071] AH02811: script not found or unable to stat: /usr/lib/cgi-bin/kerbynet
+
+# failJSON: { "time": "2024-12-18T23:58:03", "match": true , "host": "192.0.2.74", "desc": "script not found, changed log-format with stderr from (gh-3900)" }
+[Wed Dec 18 23:58:03.148113 2024] [cgi:error] [pid 16720:tid 1878] [client 192.0.2.74:35092] AH02811: stderr from /usr/lib/cgi-bin/luci: script not found or unable to stat

fail2ban/tests/files/logs/apache-overflows

@@ -25,3 +25,11 @@
 # https://issues.apache.org/bugzilla/show_bug.cgi?id=46123
 # failJSON: { "time": "2008-10-29T11:55:14", "match": true , "host": "127.0.0.1" }
 [Wed Oct 29 11:55:14 2008] [error] [client 127.0.0.1] Invalid method in request \x16\x03\x01 - possible attempt to establish SSL connection when the server isn't expecting it
+
+# failJSON: { "time": "2024-06-26T05:20:26", "match": true , "host": "192.0.2.39", "desc": "AH10244: invalid URI path, gh-3778" }
+[Wed Jun 26 05:20:26.182799 2024] [core:error] [pid 2928] [client 192.0.2.39:37924] AH10244: invalid URI path (/cgi-bin/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/bin/sh)
+
+# failJSON: { "time": "2024-12-18T15:23:15", "match": true , "host": "192.0.2.74", "desc": "coverage for another log-format (gh-3900)" }
+[Wed Dec 18 15:23:15.495667 2024] [core:error] [pid 1672:tid 1839] [client 192.0.2.74:39140] AH10244: invalid URI path (/cgi-bin/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/bin/sh)
+# failJSON: { "time": "2024-12-18T15:23:16", "match": true , "host": "192.0.2.74", "desc": "coverage for another log-format (gh-3900)" }
+[Wed Dec 18 15:23:16.304454 2024] [core:error] [pid 1673:tid 1845] [client 192.0.2.74:35446] AH10244: invalid URI path (/cgi-bin/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/.%2e/bin/sh)

fail2ban/tests/files/logs/asterisk

@@ -21,6 +21,8 @@
 [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'.
 # failJSON: { "time": "2005-01-18T17:39:50", "match": true , "host": "1.2.3.4" }
 [Jan 18 17:39:50] NOTICE[12049]: res_pjsip_session.c:2337 new_invite: Call from 'anonymous' (TCP:[1.2.3.4]:61470) to extension '9011+442037690237' rejected because extension not found in context 'default'.
+# failJSON: { "time": "2005-06-12T15:13:54", "match": true , "host": "1.2.3.4" }
+[Jun 12 15:13:54] NOTICE[3980] res_pjsip_session.c:  anonymous: Call (UDP:1.2.3.4:65049) to extension '001447441452805' rejected because extension not found in context 'inbound-foo-bar'.
 # failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" }
 [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found
 # failJSON: { "time": "2013-05-13T07:10:53", "match": true , "host": "1.2.3.4" }
@@ -106,6 +108,8 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han
 # PJSip Errors
 # failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.6" }
 [2016-05-06 07:08:09] NOTICE[17103] res_pjsip/pjsip_distributor.c: Request from '"test1" <sip:test1@2.3.4.5>' failed for '192.0.2.6:5678' (callid: deadbeef) - No matching endpoint found
+# failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.7", "desc": "Test for No matching endpoint found with retry counts (pattern 1)" }
+[2016-05-06 07:08:09] NOTICE[17103] res_pjsip/pjsip_distributor.c: Request 'INVITE' from '"test2" <sip:test2@3.4.5.6>' failed for '192.0.2.7:5679' (callid: cafebabe) - No matching endpoint found after 5 tries in 2.500 ms
 
 # # FreePBX Warnings
 # #_dis_failJSON: { "time": "2016-05-06T07:08:09", "match": true, "host": "192.0.2.4" }

fail2ban/tests/files/logs/dovecot

@@ -22,6 +22,12 @@ Jun 14 00:48:21 platypus dovecot: imap-login: Disconnected (auth failed, 1 attem
 # failJSON: { "time": "2005-06-23T00:52:43", "match": true , "host": "193.95.245.163" }
 Jun 23 00:52:43 vhost1-ua dovecot: pop3-login: Disconnected: Inactivity (auth failed, 1 attempts): user=<info>, method=PLAIN, rip=193.95.245.163, lip=176.214.13.210
 
+# Dovecot version 2.4
+# failJSON: { "time": "2005-06-12T19:07:29", "match": true , "host": "192.0.2.241" }
+Jun 12 19:07:29 hostname dovecot[241]: imap-login: Login aborted: Connection closed (auth failed, 3 attempts in 16 secs) (auth_failed): user=<test>, method=PLAIN, rip=192.0.2.241, lip=203.0.113.104, TLS, session=<9ZHq02g3J8S60fan>
+# failJSON: { "time": "2005-06-13T16:35:56", "match": true , "host": "192.0.2.241" }
+Jun 13 16:35:56 mx dovecot[241]: managesieve-login: Login aborted: Logged out (auth failed, 1 attempts in 10 secs) (auth_failed): user=<user@domain>, method=PLAIN, rip=192.0.2.241, lip=203.0.113.104, TLS, session=<Dp8j1Ho3suQYdo+k>
+
 # failJSON: { "time": "2005-07-02T13:49:31", "match": true , "host": "192.51.100.13" }
 Jul 02 13:49:31 hostname dovecot[442]: pop3-login: Aborted login (auth failed, 1 attempts in 17 secs): user=<test>, method=PLAIN, rip=192.51.100.13, lip=203.0.113.17, session=<YADINsQCDs5BH8Pg>
 
@@ -141,6 +147,9 @@ Aug 29 15:33:53 server dovecot: managesieve-login: Disconnected: Too many invali
 
 # failJSON: { "time": "2004-08-29T01:49:33", "match": true , "host": "192.0.2.5", "desc": "matches in aggressive mode, avoid slow RE, gh-3370" }
 Aug 29 01:49:33 server dovecot[459]: imap-login: Disconnected: Connection closed: read(size=1026) failed: Connection reset by peer (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1, TLS handshaking: read(size=1026) failed: Connection reset by peer
+# failJSON: { "time": "2004-08-22T12:09:41", "match": true , "host": "192.0.2.6", "desc": "matches in aggressive mode, new format with `Login aborted` and `(no_auth_attempts)`" }
+Aug 22 12:09:41 server dovecot: imap-login: Login aborted: Connection closed: read(size=653) failed: Connection reset by peer (no auth attempts in 1 secs) (no_auth_attempts): user=<>, rip=192.0.2.6, lip=192.0.2.1, TLS: read(size=653) failed: Connection reset by peer, session=<qMLsbvY8PLSkXGoP>
+
 # failJSON: { "time": "2004-08-29T01:49:33", "match": true , "host": "192.0.2.5", "desc": "matches in aggressive mode, avoid slow RE, gh-3370" }
 Aug 29 01:49:33 server dovecot[459]: imap-login: Disconnected: Connection closed: SSL_accept() failed: error:1408F10B:SSL routines:ssl3_get_record:wrong version number (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1, TLS handshaking: SSL_accept() failed: error:1408F10B:SSL routines:ssl3_get_record:wrong version number
 # failJSON: { "time": "2004-08-29T01:49:33", "match": true , "host": "192.0.2.5", "desc": "matches in aggressive mode, avoid slow RE, gh-3370" }
@@ -156,3 +165,8 @@ Aug 29 16:06:58 s166-62-100-187 dovecot: imap-login: Disconnected (disconnected
 Aug 31 16:15:10 s166-62-100-187 dovecot: imap-login: Disconnected (client didn't finish SASL auth, waited 2 secs): user=<>, method=PLAIN, rip=192.0.2.6, lip=192.168.1.2, TLS: SSL_read() syscall failed: Connection reset by peer, TLSv1.2 with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)
 # failJSON: { "time": "2004-08-31T16:21:53", "match": true , "host": "192.0.2.7" }
 Aug 31 16:21:53 s166-62-100-187 dovecot: imap-login: Disconnected (no auth attempts in 4 secs): user=<>, rip=192.0.2.7, lip=192.168.1.2, TLS handshaking: SSL_accept() syscall failed: Connection reset by peer
+
+# failJSON: { "time": "2004-08-22T12:09:41", "match": true , "host": "192.0.2.7", "desc": "disconnected during TLS handshake (no application protocol)" }
+Aug 22 12:09:41 server dovecot: imap-login: Login aborted: Connection closed: SSL_accept() failed: error:0A0000EB:SSL routines::no application protocol (disconnected during TLS handshake) (tls_handshake_not_finished): user=<>, rip=192.0.2.7, lip=192.0.2.1, TLS handshaking: SSL_accept() failed: error:0A0000EB:SSL routines::no application protocol, session=<J5HwbvY8QrSkXGoP>
+# failJSON: { "time": "2004-08-22T12:09:41", "match": true , "host": "192.0.2.7", "desc": "disconnected during TLS handshake (no shared cipher)" }
+Aug 22 12:09:41 server dovecot: imap-login: Login aborted: Connection closed: SSL_accept() failed: error:0A0000C1:SSL routines::no shared cipher (disconnected during TLS handshake) (tls_handshake_not_finished): user=<>, rip=192.0.2.7, lip=192.0.2.1, TLS handshaking: SSL_accept() failed: error:0A0000C1:SSL routines::no shared cipher, session=<Q23ybvY8RLSkXGoP>

fail2ban/tests/files/logs/dropbear

@@ -13,3 +13,11 @@ Jul 27 01:04:12 fail2ban-test dropbear[1335]: Bad password attempt for 'root' fr
 Jul 27 01:04:22 fail2ban-test dropbear[1335]: Exit before auth (user 'root', 10 fails): Max auth tries reached - user 'root' from 1.2.3.4:60588
 # failJSON: { "time": "2005-07-27T01:18:59", "match": true , "host": "1.2.3.4" }
 Jul 27 01:18:59 fail2ban-test dropbear[1477]: Login attempt for nonexistent user from 1.2.3.4:60794
+
+# failJSON: { "time": "2005-07-10T23:53:52", "match": true , "host": "1.2.3.4", "desc": "extra pid/timestamp may be logged into journal, gh-3597" }
+Jul 10 23:53:52 fail2ban-test dropbear[825]: [825] Jul 10 23:53:52 Bad password attempt for 'root' from 1.2.3.4:52289
+
+# failJSON: { "time": "2005-07-10T23:57:29", "match": true , "host": "192.0.2.3", "desc": "different message format, gh-3791" }
+Jul 10 23:57:29 fail2ban-test dropbear[825]: [825] Jul 10 23:57:29 Exit before auth from <192.0.2.3:52289>: (user 'root', 10 fails): Max auth tries reached - user 'root'
+# failJSON: { "time": "2005-07-10T23:59:24", "match": true , "host": "192.0.2.3", "desc": "different message format, gh-3791" }
+Jul 10 23:59:24 fail2ban-test dropbear[826]: [826] Jul 10 23:59:24 Exit before auth from <192.0.2.3:52325>: Max auth tries reached - user 'is invalid'

fail2ban/tests/files/logs/exim

@@ -6,6 +6,41 @@
 2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s)
 # failJSON: { "time": "2013-06-12T13:18:11", "match": true , "host": "101.66.165.86" }
 2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data
+
+# 'https://github.com/fail2ban/fail2ban/pull/251#issuecomment-23001227' 
+# failJSON: { "time": "2013-08-20T07:48:02", "match": true , "host": "85.25.92.177" }
+2013-08-20 07:48:02 login authenticator failed for static-ip-85-25-92-177.inaddr.ip-pool.com (USER) [85.25.92.177]: 535 Incorrect authentication data: 1 Time(s)
+# failJSON: { "time": "2013-08-20T23:30:05", "match": true , "host": "91.218.72.71" }
+2013-08-20 23:30:05 plain authenticator failed for ([192.168.2.102]) [91.218.72.71]: 535 Incorrect authentication data: 1 Time(s)
+# failJSON: { "time": "2013-09-02T09:19:07", "match": true , "host": "118.233.20.68" }
+2013-09-02 09:19:07 login authenticator failed for (gkzwsoju) [118.233.20.68]: 535 Incorrect authentication data
+# failJSON: { "time": "2014-01-12T02:07:48", "match": true , "host": "85.214.85.40" }
+2014-01-12 02:07:48 dovecot_login authenticator failed for h1832461.stratoserver.net (User) [85.214.85.40]: 535 Incorrect authentication data (set_id=scanner)
+# failJSON: { "time": "2019-10-22T03:39:17", "match": true , "host": "192.0.2.37", "desc": "pid-prefix in form of 'hostname exim[...]:', gh-2553" }
+2019-10-22 03:39:17 mx1.fqdn.local exim[29786]: dovecot_login authenticator failed for (User) [192.0.2.37]: 535 Incorrect authentication data (set_id=test@domain.com)
+# failJSON: { "time": "2014-12-02T03:00:23", "match": true , "host": "193.254.202.35" }
+2014-12-02 03:00:23 auth_plain authenticator failed for (rom182) [193.254.202.35]:41556 I=[10.0.0.1]:25: 535 Incorrect authentication data (set_id=webmaster)
+# failJSON: { "time": "2017-04-23T22:45:59", "match": true , "host": "192.0.2.2", "desc": "optional part (...)" }
+2017-04-23 22:45:59 fixed_login authenticator failed for bad.host.example.com [192.0.2.2]:54412 I=[172.89.0.6]:587: 535 Incorrect authentication data (set_id=user@example.com)
+# failJSON: { "time": "2024-03-21T19:26:06", "match": true , "host": "194.169.175.1" }
+2024-03-21 19:26:06 dovecot_login authenticator failed for (User) [194.169.175.1]:21298 I=[22.33.44.55]:465 Ci=30416: 535 Incorrect authentication data (set_id=uaf589@example.com)
+
+## no matches with `mode = normal`:
+
+# failJSON: { "match": false , "desc": "aggressive mode only" }
+2017-12-03 08:32:00 no host name found for IP address 192.0.2.8
+# failJSON: { "match": false , "desc": "aggressive mode only" }
+2017-12-03 08:51:35 no IP address found for host test.example.com (during SMTP connection from [192.0.2.9])
+# failJSON: { "match": false , "desc": "aggressive mode only" }
+2022-04-03 21:53:53 no IP address found for host hos-t.example.tld (during SMTP connection from [63.85.123.6]:49390 I=[31.130.202.17]:25)
+
+# filterOptions: {"logtype": "journal"}
+
+# failJSON: { "match": true , "host": "192.0.2.27", "desc": "systemd-journal entry with additional timestamp, gh-3060" }
+mail.example.com exim[3751842]: 2021-07-17 23:20:49 plain_server authenticator failed for ([192.0.2.17]) [192.0.2.27]: 535 Incorrect authentication data
+
+# filterOptions: [{"mode": "more"}, {"mode": "aggressive"}]
+
 # failJSON: { "time": "2013-06-10T10:10:59", "match": true , "host": "193.169.56.211" }
 2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for <user@example.com>: Unrouteable address
 # http://forum.lissyara.su/viewtopic.php?f=20&t=29857
@@ -35,24 +70,6 @@
 # failJSON: { "time": "2013-06-15T16:36:49", "match": true , "host": "111.67.203.116" }
 2013-06-15 16:36:49 H=altmx.marsukov.com [111.67.203.116] F=<kadrofutcheti@mail.ru> rejected RCPT <oksana@birzhatm.ua>: Unknown user
 
-# 'https://github.com/fail2ban/fail2ban/pull/251#issuecomment-23001227' 
-# failJSON: { "time": "2013-08-20T07:48:02", "match": true , "host": "85.25.92.177" }
-2013-08-20 07:48:02 login authenticator failed for static-ip-85-25-92-177.inaddr.ip-pool.com (USER) [85.25.92.177]: 535 Incorrect authentication data: 1 Time(s)
-# failJSON: { "time": "2013-08-20T23:30:05", "match": true , "host": "91.218.72.71" }
-2013-08-20 23:30:05 plain authenticator failed for ([192.168.2.102]) [91.218.72.71]: 535 Incorrect authentication data: 1 Time(s)
-
-# failJSON: { "time": "2013-09-02T09:19:07", "match": true , "host": "118.233.20.68" }
-2013-09-02 09:19:07 login authenticator failed for (gkzwsoju) [118.233.20.68]: 535 Incorrect authentication data
-
-# failJSON: { "time": "2014-01-12T02:07:48", "match": true , "host": "85.214.85.40" }
-2014-01-12 02:07:48 dovecot_login authenticator failed for h1832461.stratoserver.net (User) [85.214.85.40]: 535 Incorrect authentication data (set_id=scanner)
-
-# failJSON: { "time": "2019-10-22T03:39:17", "match": true , "host": "192.0.2.37", "desc": "pid-prefix in form of 'hostname exim[...]:', gh-2553" }
-2019-10-22 03:39:17 mx1.fqdn.local exim[29786]: dovecot_login authenticator failed for (User) [192.0.2.37]: 535 Incorrect authentication data (set_id=test@domain.com)
-
-# failJSON: { "time": "2014-12-02T03:00:23", "match": true , "host": "193.254.202.35" }
-2014-12-02 03:00:23 auth_plain authenticator failed for (rom182) [193.254.202.35]:41556 I=[10.0.0.1]:25: 535 Incorrect authentication data (set_id=webmaster)
-
 # failJSON: { "time": "2016-03-18T00:34:06", "match": true , "host": "45.32.34.167" }
 2016-03-18 00:34:06 [7513] SMTP protocol error in "AUTH LOGIN" H=(ylmf-pc) [45.32.34.167]:60723 I=[172.89.0.6]:587 AUTH command used when not advertised
 # failJSON: { "time": "2016-03-19T18:40:44", "match": true , "host": "92.45.204.170" }
@@ -86,8 +103,6 @@
 # failJSON: { "time": "2016-03-27T16:48:48", "match": true , "host": "192.0.2.1" }
 2016-03-27 16:48:48 [21478] 1akDqs-0005aQ-9b SMTP connection from host.example.com (SERVER) [192.0.2.1]:47714 I=[172.89.0.6]:25 closed by DROP in ACL
 
-# failJSON: { "time": "2017-04-23T22:45:59", "match": true , "host": "192.0.2.2", "desc": "optional part (...)" }
-2017-04-23 22:45:59 fixed_login authenticator failed for bad.host.example.com [192.0.2.2]:54412 I=[172.89.0.6]:587: 535 Incorrect authentication data (set_id=user@example.com)
 # failJSON: { "time": "2017-05-01T07:42:42", "match": true , "host": "192.0.2.3", "desc": "rejected RCPT - Unrouteable address" }
 2017-05-01 07:42:42 H=some.rev.dns.if.found (the.connector.reports.this.name) [192.0.2.3] F=<some.name@some.domain> rejected RCPT <some.invalid.name@a.domain>: Unrouteable address
 
@@ -98,20 +113,9 @@
 # failJSON: { "time": "2017-11-28T14:14:32", "match": true , "host": "192.0.2.6", "desc": "quoted injecting on AUTH command" }
 2017-11-28 14:14:32 SMTP protocol error in "aUtH lOgIn" H=(test) [8.8.8.8]" H=(roxzgj) [192.0.2.6] AUTH command used when not advertised
 
-# failJSON: { "time": "2024-03-21T19:26:06", "match": true , "host": "194.169.175.1" }
-2024-03-21 19:26:06 dovecot_login authenticator failed for (User) [194.169.175.1]:21298 I=[22.33.44.55]:465 Ci=30416: 535 Incorrect authentication data (set_id=uaf589@example.com)
 # failJSON: { "time": "2024-03-21T09:18:51", "match": true , "host": "9.12.1.21" }
 2024-03-21 09:18:51 H=m05.horp.tld [9.12.1.21]:43030 I=[194.169.175.2]:25 Ci=7326 CV=no SNI=mail.leone.tld F=<user@example.tld> rejected RCPT <locus@leone.tld>: relay not permitted
 
-## no matches with `mode = normal`:
-
-# failJSON: { "match": false , "desc": "aggressive mode only" }
-2017-12-03 08:32:00 no host name found for IP address 192.0.2.8
-# failJSON: { "match": false , "desc": "aggressive mode only" }
-2017-12-03 08:51:35 no IP address found for host test.example.com (during SMTP connection from [192.0.2.9])
-# failJSON: { "match": false , "desc": "aggressive mode only" }
-2022-04-03 21:53:53 no IP address found for host hos-t.example.tld (during SMTP connection from [63.85.123.6]:49390 I=[31.130.202.17]:25)
-
 # filterOptions: [{"mode": "aggressive"}]
 
 # failJSON: { "time": "2017-12-03T08:32:00", "match": true , "host": "192.0.2.8", "desc": "no host found for IP" }
@@ -120,8 +124,5 @@
 2017-12-03 08:51:35 no IP address found for host test.example.com (during SMTP connection from [192.0.2.9])
 # failJSON: { "time": "2022-04-03T21:53:53", "match": true , "host": "63.85.123.6", "desc": "no IP found for host long" }
 2022-04-03 21:53:53 no IP address found for host hos-t.example.tld (during SMTP connection from [63.85.123.6]:49390 I=[31.130.202.17]:25)
-
-# filterOptions: {"logtype": "journal"}
-
-# failJSON: { "match": true , "host": "192.0.2.27", "desc": "systemd-journal entry with additional timestamp, gh-3060" }
-mail.example.com exim[3751842]: 2021-07-17 23:20:49 plain_server authenticator failed for ([192.0.2.17]) [192.0.2.27]: 535 Incorrect authentication data
+# failJSON: { "time": "2022-04-03T21:53:54", "match": true , "host": "192.0.2.101", "desc": "dropped by ACL" }
+2022-04-03 21:53:54 H=[192.0.2.101]:62839 dropped by 'connect' ACL: Country is banned

fail2ban/tests/files/logs/freeswitch

@@ -24,3 +24,8 @@
 08-03 07:56:53.026292 [WARN] [SOFIA] [sofia_reg.c:4130] Can't find user [101@148.251.134.154] from 192.0.2.3
 # failJSON: { "time": "2005-08-03T08:10:21", "match": true, "host": "192.0.2.4", "desc": "optional year in datepattern and bit different format (gh-2193)" }
 08-03 08:10:21.026299 [WARN] [SOFIA] [sofia_reg.c:2248] SIP auth failure (INVITE) on sofia profile 'external' for [41801148436701961@148.251.134.154] from ip 192.0.2.4
+
+# failJSON: { "time": "2021-10-29T21:04:58", "match": true, "host": "192.0.2.5", "desc": "percent in prefix (gh-3143)" }
+2021-10-29 21:04:58.150982 00.00% [WARNING] sofia_reg.c:2889 Can't find user [201@::1] from 192.0.2.5
+# failJSON: { "time": "2021-10-29T21:05:58", "match": true, "host": "192.0.2.5", "desc": "syslog (and extra date), percent in prefix (gh-3143)" }
+2021-10-29 21:05:58.894719 www.srv.tld freeswitch[123456]: 2021-10-29 14:13:24.894719 82.43% [WARNING] sofia_reg.c:2889 Can't find user [201@::1] from 192.0.2.5

fail2ban/tests/files/logs/froxlor-auth

@@ -1,5 +1,11 @@
+# filterOptions: [{"type": "1"}]
 # failJSON: { "time": "2005-05-21T00:56:27", "match": true , "host": "1.2.3.4" }
 May 21 00:56:27 jomu Froxlor: [Login Action 1.2.3.4] Unknown user 'user' tried to login.
 # failJSON: { "time": "2005-05-21T00:57:38", "match": true , "host": "1.2.3.4" }
 May 21 00:57:38 jomu Froxlor: [Login Action 1.2.3.4] User 'admin' tried to login with wrong password.
 
+# filterOptions: [{}, {"type": "2"}]
+# failJSON: { "time": "2025-09-21T17:46:18", "match": true , "host": "1.2.3.4" }
+2025-09-21T17:46:18.311379+02:00 hostname froxlor[1055219]: froxlor.WARNING: User tried to login with wrong password. {"source":"login","action":"50","user":"1.2.3.4"} []
+# failJSON: { "time": "2025-09-21T16:30:13", "match": true , "host": "1.2.3.4" }
+2025-09-21T16:30:13.118232+02:00 hostname froxlor[1054438]: froxlor.WARNING: Unknown user tried to login. {"source":"login","action":"50","user":"1.2.3.4"} []

fail2ban/tests/files/logs/lighttpd-auth

@@ -12,3 +12,7 @@
 2021-09-30 17:44:37: (mod_auth.c.791) digest: auth failed for  tester : wrong password, IP: 192.0.2.3
 # failJSON: { "time": "2021-09-30T17:44:37", "match": true , "host": "192.0.2.4", "desc": "gh-3116" }
 2021-09-30 17:44:37: (mod_auth.c.791) digest: auth failed: uri mismatch (/uri1 != /uri2), IP: 192.0.2.4
+
+# systemd-journal
+# failJSON: { "time": "2025-03-04T02:11:57", "match": true , "host": "192.0.2.211", "desc": "gh-3955" }
+2025-03-04T02:11:57.602061 ip-172-31-3-150.ap-southeast-2.compute.internal lighttpd[764]: (mod_auth.c.853) password doesn't match for / username: user1 IP: 192.0.2.211