[2.4.x] further mod_auth_digest improvements

.gitleaks.toml

@@ -0,0 +1,11 @@
+# Gitleaks scanner config to prevent false positives
+# see https://github.com/gitleaks/gitleaks
+[allowlist]
+description = "Global Allowlist"
+
+# Ignore based on any subset of the file path
+paths = [
+# Ignore all authentication tests which do contain
+# embedded test passwords.
+'''test\/modules\/aaa\/*\.py''',
+]

changes-entries/digest-shmem-size.txt

@@ -0,0 +1,3 @@
+  *) mod_auth_digest: Increase the default AuthDigestShmemSize to 8192
+     bytes, tracking around 140 clients rather than around 12.
+     [Joe Orton]

docs/manual/mod/mod_auth_digest.xml

@@ -240,7 +240,7 @@ authentication</description>
 <description>The amount of shared memory to allocate for keeping track
 of clients</description>
 <syntax>AuthDigestShmemSize <var>size</var></syntax>
-<default>AuthDigestShmemSize 1000</default>
+<default>AuthDigestShmemSize 8192</default>
 <contextlist><context>server config</context></contextlist>
 
 <usage>
@@ -254,6 +254,13 @@ of clients</description>
     <code>0</code> and read the error message after trying to start the
     server.</p>
 
+    <p>The default holds roughly 140 clients. A client which is discarded
+    to make room for another is not denied access: it is issued a new
+    nonce with <code>stale=true</code>, which costs it one extra request.
+    Note that a request which does not authenticate also takes an entry,
+    since the challenge sent back to it carries the identifier the client
+    is tracked by.</p>
+
     <p>The <var>size</var> is normally expressed in Bytes, but you
     may follow the number with a <code>K</code> or an <code>M</code> to
     express your value as KBytes or MBytes. For example, the following

test/modules/aaa/__init__.py

@@ -0,0 +1 @@
+

test/modules/aaa/conftest.py

@@ -0,0 +1,105 @@
+import logging
+import os
+import sys
+
+import pytest
+
+from .env import AAATestEnv
+from pyhttpd.conf import HttpdConf
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
+
+
+def pytest_report_header(config, start_path):
+    env = AAATestEnv()
+    return f"mod_auth_digest [apache: {env.get_httpd_version()}, mpm: {env.mpm_module}, {env.prefix}]"
+
+
+def _digest_dir(docs, path, extra_lines):
+    lines = [
+        f'<Directory "{docs}/digest/{path}">',
+        '    AuthType Digest',
+        f'    AuthName "{AAATestEnv.REALM}"',
+    ]
+    lines.extend(f"    {l}" for l in extra_lines)
+    lines.append('    Require valid-user')
+    lines.append('</Directory>')
+    return lines
+
+
+@pytest.fixture(scope="package")
+def env(pytestconfig) -> AAATestEnv:
+    level = logging.INFO
+    console = logging.StreamHandler()
+    console.setLevel(level)
+    console.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+    logging.getLogger('').addHandler(console)
+    logging.getLogger('').setLevel(level=level)
+    env = AAATestEnv(pytestconfig=pytestconfig)
+    env.setup_httpd()
+    env.apache_access_log_clear()
+    env.httpd_error_log.clear_log()
+
+    docs = env.server_docs_dir
+    pwfile = env.digest_pwfile
+    conf = HttpdConf(env)
+    # Pin the client table to its historical size, ~12 entries, rather than
+    # the current default of ~140: the tests which need an entry to be
+    # garbage collected (085) drive that by filling the table with bare
+    # requests, and a table an order of magnitude larger makes them an order
+    # of magnitude slower for nothing.
+    conf.add('AuthDigestShmemSize 1000')
+    conf.add(_digest_dir(docs, "default", [
+        'AuthDigestProvider file',
+        f'AuthUserFile "{pwfile}"',
+    ]))
+    conf.add(_digest_dir(docs, "nccheck", [
+        'AuthDigestProvider file',
+        f'AuthUserFile "{pwfile}"',
+        'AuthDigestNcCheck On',
+    ]))
+    conf.add(_digest_dir(docs, "nccheck-shortlife", [
+        'AuthDigestProvider file',
+        f'AuthUserFile "{pwfile}"',
+        'AuthDigestNcCheck On',
+        'AuthDigestNonceLifetime 2',
+    ]))
+    conf.add(_digest_dir(docs, "shortlife", [
+        'AuthDigestProvider file',
+        f'AuthUserFile "{pwfile}"',
+        'AuthDigestNonceLifetime 2',
+    ]))
+    conf.add(_digest_dir(docs, "neverexpire", [
+        'AuthDigestProvider file',
+        f'AuthUserFile "{pwfile}"',
+        'AuthDigestNonceLifetime -1',
+    ]))
+    conf.add(_digest_dir(docs, "onetime", [
+        'AuthDigestProvider file',
+        f'AuthUserFile "{pwfile}"',
+        'AuthDigestNonceLifetime 0',
+    ]))
+    conf.add(_digest_dir(docs, "onetime-nccheck", [
+        'AuthDigestProvider file',
+        f'AuthUserFile "{pwfile}"',
+        'AuthDigestNonceLifetime 0',
+        'AuthDigestNcCheck On',
+    ]))
+    conf.add(_digest_dir(docs, "domain", [
+        'AuthDigestProvider file',
+        f'AuthUserFile "{pwfile}"',
+        'AuthDigestDomain "/digest/domain/" "https://mirror.example.org/other/"',
+    ]))
+    conf.add(_digest_dir(docs, "noprovider", [
+        # AuthDigestProvider intentionally omitted: falls back to "file".
+        f'AuthUserFile "{pwfile}"',
+    ]))
+    conf.install()
+    assert env.apache_restart() == 0
+    return env
+
+
+@pytest.fixture(autouse=True, scope="package")
+def _stop_package_scope(env):
+    yield
+    assert env.apache_stop() == 0

test/modules/aaa/digest_client.py

@@ -0,0 +1,134 @@
+"""Minimal hand-rolled RFC 2617 Digest auth client.
+
+curl's own `--digest` handles the challenge/response handshake transparently,
+which is no good for testing edge cases (tampered nonces, replayed
+nonce-counts, wrong realms, bad algorithm tokens, ...). This module lets
+tests parse a WWW-Authenticate challenge, compute the expected response by
+hand, and build a (possibly deliberately broken) Authorization header.
+
+mod_auth_digest here only implements qop="auth" (see modules/aaa/mod_auth_digest.c
+Open Issues: "MD5-sess and auth-int are not yet implemented"), so this client
+only implements the qop=auth request-digest/response-auth formulas from
+RFC 2617 section 3.2.2.
+"""
+
+import hashlib
+import re
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+_PARAM_RE = re.compile(r'(\w+)=(?:"([^"]*)"|([^\s,]+))\s*,?\s*')
+
+
+def _md5hex(s: str) -> str:
+    return hashlib.md5(s.encode('utf-8')).hexdigest()
+
+
+def parse_params(value: str) -> Dict[str, str]:
+    """Parse a comma-separated key=value / key="value" list, as used by
+    both WWW-Authenticate and Authentication-Info header values."""
+    params = {}
+    for m in _PARAM_RE.finditer(value):
+        key = m.group(1)
+        val = m.group(2) if m.group(2) is not None else m.group(3)
+        params[key.lower()] = val
+    return params
+
+
+@dataclass
+class DigestChallenge:
+    realm: Optional[str]
+    nonce: Optional[str]
+    algorithm: Optional[str] = None
+    opaque: Optional[str] = None
+    domain: Optional[str] = None
+    qop: Optional[str] = None
+    stale: bool = False
+    raw: str = ""
+
+    @staticmethod
+    def parse(www_authenticate: str) -> 'DigestChallenge':
+        assert www_authenticate.startswith("Digest "), \
+            f"not a Digest challenge: {www_authenticate}"
+        params = parse_params(www_authenticate[len("Digest "):])
+        return DigestChallenge(
+            realm=params.get('realm'),
+            nonce=params.get('nonce'),
+            algorithm=params.get('algorithm'),
+            opaque=params.get('opaque'),
+            domain=params.get('domain'),
+            qop=params.get('qop'),
+            stale=params.get('stale', '').lower() == 'true',
+            raw=www_authenticate,
+        )
+
+    def domain_list(self) -> List[str]:
+        return self.domain.split() if self.domain else []
+
+
+def ha1(username: str, realm: str, password: str) -> str:
+    return _md5hex(f"{username}:{realm}:{password}")
+
+
+def ha2(method: str, uri: str) -> str:
+    return _md5hex(f"{method}:{uri}")
+
+
+def request_digest(ha1_hex: str, nonce: str, nc: str, cnonce: str,
+                    qop: str, ha2_hex: str) -> str:
+    return _md5hex(f"{ha1_hex}:{nonce}:{nc}:{cnonce}:{qop}:{ha2_hex}")
+
+
+def rspauth_digest(ha1_hex: str, nonce: str, nc: str, cnonce: str,
+                    qop: str, uri: str) -> str:
+    """Authentication-Info's rspauth uses A2 = ':' + uri (no method)."""
+    ha2_hex = _md5hex(f":{uri}")
+    return _md5hex(f"{ha1_hex}:{nonce}:{nc}:{cnonce}:{qop}:{ha2_hex}")
+
+
+def build_authorization(username: str, challenge: DigestChallenge, password: str,
+                         method: str, uri: str, nc: str = "00000001",
+                         cnonce: str = "0a4f113b3c2e7a1d", qop: Optional[str] = "auth",
+                         realm: Optional[str] = None, nonce_val: Optional[str] = None,
+                         algorithm: Optional[str] = None, response: Optional[str] = None,
+                         opaque: Optional[str] = None, include_opaque: bool = True,
+                         include_qop_fields: bool = True, extra: Optional[List[str]] = None
+                         ) -> str:
+    """Build a Digest Authorization header value.
+
+    By default this builds a *correct* response for the given challenge and
+    credentials. Any of realm=/nonce_val=/algorithm=/response=/opaque= can be
+    overridden to construct deliberately invalid headers, and qop=None with
+    include_qop_fields=False builds a legacy RFC 2069-style header (no qop,
+    cnonce, or nc) to prove that path is rejected.
+    """
+    eff_realm = challenge.realm if realm is None else realm
+    eff_nonce = challenge.nonce if nonce_val is None else nonce_val
+    if response is None:
+        h1 = ha1(username, eff_realm, password)
+        h2 = ha2(method, uri)
+        if qop:
+            response = request_digest(h1, eff_nonce, nc, cnonce, qop, h2)
+        else:
+            # legacy RFC 2069: MD5(HA1:nonce:HA2), no qop/cnonce/nc
+            response = _md5hex(f"{h1}:{eff_nonce}:{h2}")
+
+    parts = [
+        f'username="{username}"',
+        f'realm="{eff_realm}"',
+        f'nonce="{eff_nonce}"',
+        f'uri="{uri}"',
+        f'response="{response}"',
+    ]
+    if algorithm is not None:
+        parts.append(f'algorithm={algorithm}')
+    if qop and include_qop_fields:
+        parts.append(f'qop={qop}')
+        parts.append(f'nc={nc}')
+        parts.append(f'cnonce="{cnonce}"')
+    eff_opaque = challenge.opaque if (opaque is None and include_opaque) else opaque
+    if eff_opaque:
+        parts.append(f'opaque="{eff_opaque}"')
+    if extra:
+        parts.extend(extra)
+    return "Digest " + ", ".join(parts)

test/modules/aaa/env.py

@@ -0,0 +1,79 @@
+import hashlib
+import inspect
+import logging
+import os
+from typing import List, Optional
+
+from pyhttpd.env import HttpdTestEnv, HttpdTestSetup
+from pyhttpd.result import ExecResult
+
+log = logging.getLogger(__name__)
+
+
+class AAATestSetup(HttpdTestSetup):
+
+    def __init__(self, env: 'HttpdTestEnv'):
+        super().__init__(env=env)
+        self.add_source_dir(os.path.dirname(inspect.getfile(AAATestSetup)))
+        self.add_modules(["auth_digest", "authn_file", "authn_core",
+                          "authz_core", "authz_user"])
+
+
+class AAATestEnv(HttpdTestEnv):
+
+    REALM = "AAA Digest Realm"
+    DIGEST_USER = "digestuser"
+    DIGEST_PASSWORD = "digestpass2617"
+    DIGEST_USER2 = "otheruser"
+    DIGEST_PASSWORD2 = "otherpass2617"
+
+    def __init__(self, pytestconfig=None):
+        super().__init__(pytestconfig=pytestconfig)
+        self.add_httpd_log_modules(["auth_digest", "authn_file", "authz_core"])
+        self._digest_pwfile = f"{self.server_dir}/digest.passwd"
+
+    def setup_httpd(self, setup: HttpdTestSetup = None):
+        super().setup_httpd(setup=AAATestSetup(env=self))
+        self._write_digest_pwfile()
+
+    def _write_digest_pwfile(self):
+        def ha1(user, password):
+            return hashlib.md5(
+                f"{user}:{self.REALM}:{password}".encode()).hexdigest()
+
+        with open(self._digest_pwfile, 'w') as fd:
+            fd.write(f"{self.DIGEST_USER}:{self.REALM}:"
+                     f"{ha1(self.DIGEST_USER, self.DIGEST_PASSWORD)}\n")
+            fd.write(f"{self.DIGEST_USER2}:{self.REALM}:"
+                     f"{ha1(self.DIGEST_USER2, self.DIGEST_PASSWORD2)}\n")
+
+    @property
+    def digest_pwfile(self) -> str:
+        return self._digest_pwfile
+
+    def configtest(self, directory_lines: List[str], extra_top_lines: Optional[List[str]] = None
+                    ) -> ExecResult:
+        """Run `httpd -t` against a minimal, standalone config built from the
+        already-generated modules.conf plus `directory_lines` wrapped in a
+        <Directory> block over the shared docroot. Used to test directives
+        that are rejected at config-check time (e.g. AuthDigestQop values
+        other than 'auth') without touching the package's running server.
+        """
+        conf_path = os.path.join(self.gen_dir, "digest-configtest.conf")
+        modules_conf = os.path.join(self.server_conf_dir, "modules.conf")
+        lines = [
+            f'ServerRoot "{self.server_dir}"',
+            f'Include "{modules_conf}"',
+            f'DocumentRoot "{self.server_docs_dir}"',
+            f'Listen {self.http_port2}',
+        ]
+        if extra_top_lines:
+            lines.extend(extra_top_lines)
+        lines.append(f'<Directory "{self.server_docs_dir}">')
+        lines.extend(f"    {l}" for l in directory_lines)
+        lines.append('</Directory>')
+        with open(conf_path, 'w') as fd:
+            fd.write('\n'.join(lines))
+            fd.write('\n')
+        httpd_bin = os.path.join(self.bin_dir, 'httpd')
+        return self.run([httpd_bin, '-t', '-f', conf_path])

test/modules/aaa/htdocs/digest/default/secret.txt

@@ -0,0 +1 @@
+digest-default-secret

test/modules/aaa/htdocs/digest/domain/nested/secret.txt

@@ -0,0 +1 @@
+digest-domain-nested-secret

test/modules/aaa/htdocs/digest/domain/secret.txt

@@ -0,0 +1 @@
+digest-domain-secret

test/modules/aaa/htdocs/digest/nccheck-shortlife/secret.txt

@@ -0,0 +1 @@
+digest-nccheck-secret

test/modules/aaa/htdocs/digest/nccheck/secret.txt

@@ -0,0 +1 @@
+digest-nccheck-secret

test/modules/aaa/htdocs/digest/neverexpire/secret.txt

@@ -0,0 +1 @@
+digest-neverexpire-secret

test/modules/aaa/htdocs/digest/noprovider/secret.txt

@@ -0,0 +1 @@
+digest-noprovider-secret

test/modules/aaa/htdocs/digest/onetime-nccheck/secret.txt

@@ -0,0 +1 @@
+digest-onetime-secret

test/modules/aaa/htdocs/digest/onetime/secret.txt

@@ -0,0 +1 @@
+digest-onetime-secret

test/modules/aaa/htdocs/digest/shortlife/secret.txt

@@ -0,0 +1 @@
+digest-shortlife-secret