diff --git a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst index 9f35dc57bcb..7bc6d06f0db 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst @@ -43,13 +43,21 @@ DIRAC_HTTPS_SSL_METHOD_MIN If set, overrides the lowest supported TLS version when using HTTPS. It should be a valid value of :py:class:`ssl.TLSVersion`. DIRAC_M2CRYPTO_SPLIT_HANDSHAKE - If ``true`` or ``yes`` the SSL handshake is done in a new thread (default Yes) + Only relevant with ``DIRAC_USE_M2CRYPTO``. If ``true`` or ``yes`` the SSL handshake is done in a new thread (default Yes) DIRAC_M2CRYPTO_SSL_CIPHERS - If set, overwrites the default SSL ciphers accepted. It should be a colon separated list. See :py:mod:`DIRAC.Core.DISET` + Only relevant with ``DIRAC_USE_M2CRYPTO`` (but also accepted as an alias of ``DIRAC_SSL_CIPHERS`` by the default implementation). If set, overwrites the default SSL ciphers accepted. It should be a colon separated list. See :py:mod:`DIRAC.Core.DISET` DIRAC_M2CRYPTO_SSL_METHODS - If set, overwrites the default SSL methods accepted. It should be a colon separated list. See :py:mod:`DIRAC.Core.DISET` + Only relevant with ``DIRAC_USE_M2CRYPTO``. If set, overwrites the default SSL methods accepted. It should be a colon separated list. See :py:mod:`DIRAC.Core.DISET` + +DIRAC_SSL_CIPHERS + If set, overwrites the default SSL ciphers accepted by the DISET protocol. It should be an OpenSSL style colon separated list. + +DIRAC_USE_M2CRYPTO + If ``true`` or ``yes``, use the legacy M2Crypto/pyasn1 based implementation of the X509 layer and of the TLS transports (DISET and HTTPS) instead of the default one, based on pyca/cryptography and the :py:mod:`ssl` module of the standard library (default No). + The implementation is selected once, at import time, so it must be set consistently for all the components of an installation. + The optional dependencies M2Crypto, pyasn1, pyasn1-modules and (for HTTPS services) tornado-m2crypto with the patched DIRACGrid tornado fork must be installed. DIRAC_MYSQL_OPTIMIZER_TRACES_PATH If set, it should point to an existing directory, where MySQL Optimizer traces will be stored. See :py:func:`DIRAC.Core.Utilities.MySQL.captureOptimizerTraces` diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index 2246fcd5285..9a738a61858 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -313,7 +313,12 @@ How to install Tornado Requirements ************ -Two special python packages are needed: +The standard tornado package is used: the TLS layer (including grid proxy +support) relies on the ``ssl`` module of the python standard library +(python >= 3.13 is required on the server side). + +When the legacy M2Crypto implementation is enabled (``DIRAC_USE_M2CRYPTO=Yes``), +two special python packages are needed instead: * git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable : in place of the standard tornado. This adds configurable feature to tornado * git+https://github.com/DIRACGrid/tornado_m2crypto.git: this allows to use tornado with M2Crypto diff --git a/environment.yml b/environment.yml index b6c6a16716e..902151350fb 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,12 @@ dependencies: - fts3 - gitpython >=2.1.0 - invoke + - cryptography >=47.0.0 + # m2crypto, pyasn1 and pyasn1-modules are only needed for the legacy + # M2Crypto based implementation (DIRAC_USE_M2CRYPTO=Yes) - m2crypto >=0.38.0 + - pyasn1 >0.4.1 + - pyasn1-modules - matplotlib - numpy - paramiko @@ -30,8 +35,6 @@ dependencies: - pillow - prompt-toolkit >=3,<4 - psutil >=4.2.0 - - pyasn1 >0.4.1 - - pyasn1-modules - pydantic >=2 - python-json-logger >=0.1.8 - pytz >=2015.7 @@ -89,7 +92,10 @@ dependencies: - uritemplate # - readline >=6.2.4 in the standard library - simplejson >=3.8.1 - #- tornado >=5.0.0,<6.0.0 + # tornado is installed with pip below: the DIRACGrid fork is needed for the + # legacy M2Crypto based implementation (DIRAC_USE_M2CRYPTO=Yes) and behaves + # as the standard tornado >=5.1.1,<6.0.0 otherwise + #- tornado >=5.1.1,<6.0.0 - typing >=3.6.6 - rucio-clients >=34.4.2 # For mypy @@ -105,6 +111,8 @@ dependencies: - Authlib >=1.0.0 - dominate - pyjwt + # The following two packages are only needed for the legacy M2Crypto + # based implementation (DIRAC_USE_M2CRYPTO=Yes) # This is a fork of tornado with a patch to allow for configurable iostream - git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable # This is an extension of Tornado to use M2Crypto diff --git a/setup.cfg b/setup.cfg index 6c9e1c1b2bd..32dc726cd8d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ install_requires = cachetools certifi cwltool + cryptography >=47.0.0 diraccfg DIRACCommon diracx-client >=v0.0.1 @@ -41,14 +42,11 @@ install_requires = importlib_metadata >=4.4 importlib_resources invoke - M2Crypto >=0.36 packaging paramiko pexpect prompt-toolkit >=3 psutil - pyasn1 - pyasn1-modules pydantic >=2.4 pyparsing python-dateutil @@ -93,7 +91,6 @@ server = stomp.py suds tornado ~=5.1.1 - tornado-m2crypto importlib_resources testing = hypothesis @@ -104,6 +101,12 @@ testing = pytest-mock pytest-rerunfailures pycodestyle +# Only needed for the legacy M2Crypto based implementation (DIRAC_USE_M2CRYPTO=Yes) +m2crypto = + M2Crypto >=0.36 + pyasn1 + pyasn1-modules + tornado-m2crypto [options.entry_points] dirac = diff --git a/src/DIRAC/Core/DISET/__init__.py b/src/DIRAC/Core/DISET/__init__.py index 4f7bc852cf8..32fc83b15f1 100755 --- a/src/DIRAC/Core/DISET/__init__.py +++ b/src/DIRAC/Core/DISET/__init__.py @@ -5,6 +5,9 @@ #: Default timeout to establish a connection DEFAULT_CONNECTION_TIMEOUT = 10 +# The following constants are only used by the M2Crypto based SSL +# implementation (DIRAC_USE_M2CRYPTO=Yes) + #: Default SSL Ciher accepted. Current default is for pyGSI/M2crypto compatibility #: Can be changed with DIRAC_M2CRYPTO_SSL_CIPHERS #: Recommandation (incompatible with pyGSI) diff --git a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py old mode 100755 new mode 100644 index f5b42f03393..d411886b5ce --- a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py @@ -1,3 +1,13 @@ +"""SSL Transport selector. + +Two implementations of the SSL transport are available: the default one, +based on the standard library ssl module (:py:mod:`.StdSSLTransport`), and +the legacy one, based on M2Crypto (:py:mod:`.M2SSLTransport`). The +implementation is selected once, at import time, with the DIRAC_USE_M2CRYPTO +environment variable (default No). + +The helpers defined here (delegate, checkSanity) are implementation agnostic. +""" import os from DIRAC import gLogger @@ -7,9 +17,19 @@ from DIRAC.Core.Security.X509Certificate import X509Certificate # pylint: disable=import-error -# Eventhough SSLTransport is not used in this file, it is imported in other module from there, -# so do not remove these imports ! -from DIRAC.Core.DISET.private.Transports.M2SSLTransport import SSLTransport +# Even though SSLTransport, getSSLContext and getPeerInfo are not all used in +# this file, they are imported from here by other modules, so do not remove +# these imports! +if os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true"): + from DIRAC.Core.DISET.private.Transports.M2SSLTransport import SSLTransport + from DIRAC.Core.DISET.private.Transports.SSL.M2Utils import getM2SSLContext as getSSLContext + from DIRAC.Core.DISET.private.Transports.SSL.M2Utils import getM2PeerInfo as getPeerInfo +else: + from DIRAC.Core.DISET.private.Transports.StdSSLTransport import ( + SSLTransport, + getSSLContext, + getPeerInfo, + ) def delegate(delegationRequest, kwargs): diff --git a/src/DIRAC/Core/DISET/private/Transports/StdSSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/StdSSLTransport.py new file mode 100755 index 00000000000..29457fe3bf5 --- /dev/null +++ b/src/DIRAC/Core/DISET/private/Transports/StdSSLTransport.py @@ -0,0 +1,518 @@ +"""SSL Transport implementation using the standard library ssl module. + +The peer chain validation (including RFC 3820 proxy support) is delegated to +OpenSSL through ``ssl.SSLContext``: + +* proxy certificates are accepted by setting the ``X509_V_FLAG_ALLOW_PROXY_CERTS`` + verify flag (``ssl.VERIFY_ALLOW_PROXY_CERTS``) +* the peer chain is retrieved after the handshake with + ``ssl.SSLSocket.get_unverified_chain`` (python >= 3.13), and the DIRAC + credentials (DN, group, ...) are extracted from it with + :py:class:`DIRAC.Core.Security.X509Chain.X509Chain` +""" +import os +import socket +import ssl +import tempfile +import threading +import time +from collections import OrderedDict + +from DIRAC import gLogger +from DIRAC.Core.DISET import DEFAULT_CONNECTION_TIMEOUT, DEFAULT_RPC_TIMEOUT +from DIRAC.Core.DISET.private.Transports.BaseTransport import BaseTransport +from DIRAC.Core.Security import Locations + +# The pyca based X509Chain is imported through its fully qualified path (like +# M2Utils does with the m2crypto one): getPeerInfo relies on +# generateX509ChainFromDERList, which only the pyca implementation provides +from DIRAC.Core.Security.pyca.X509Chain import X509Chain +from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK + +# Verify depth of peer certs +VERIFY_DEPTH = 50 + +# Re-exported for convenience (exists since python 3.10) +VERIFY_ALLOW_PROXY_CERTS = ssl.VERIFY_ALLOW_PROXY_CERTS + + +def __loadHostCertificate(ctx): + """Load hostcert & key from the default location and set them as the + credentials for SSL context ctx. + Returns None. + """ + certKeyTuple = Locations.getHostCertificateAndKeyLocation() + if not certKeyTuple: + raise RuntimeError("Hostcert/key location not set") + hostcert, hostkey = certKeyTuple + if not os.path.isfile(hostcert): + raise RuntimeError(f"Hostcert file ({hostcert}) is missing") + if not os.path.isfile(hostkey): + raise RuntimeError(f"Hostkey file ({hostkey}) is missing") + ctx.load_cert_chain(hostcert, keyfile=hostkey) + + +def __loadProxy(ctx, proxyPath=None): + """Load proxy from proxyPath (or default location if not specified) and + set it as the certificate & key to use for this SSL context. + Returns None. + """ + if not proxyPath: + proxyPath = Locations.getProxyLocation() + if not proxyPath: + raise RuntimeError("Proxy location not set") + if not os.path.isfile(proxyPath): + raise RuntimeError(f"Proxy file ({proxyPath}) is missing") + # A proxy file contains the leaf certificate, its key, and the rest + # of the chain in a single file, which load_cert_chain handles natively + ctx.load_cert_chain(proxyPath) + + +def getSSLContext(**kwargs): + """Gets an ssl.SSLContext configured using the standard + DIRAC connection keywords from kwargs. The keywords are: + + - bServerMode: Boolean, if True the context is setup for a server + (hostcert is always used). + - useCertificates: Boolean, Set to true to use hostcerts in client + mode. + - proxyString: String, allow a literal proxy string to be provided. + - proxyLocation: String, Path to file to use as proxy, defaults to + usual location(s) if not set. + - skipCACheck: Boolean, if True, don't verify peer certificates. + - optionalClientCert: Boolean, in server mode, request but do not require + the client certificate (used by the HTTPS services, + where token/visitor authentication also exists). + - sslCiphers: String, OpenSSL style cipher string of ciphers to allow + on this connection. + + Returns the new context. + """ + serverMode = kwargs.get("bServerMode", False) + + if serverMode: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + else: + # PROTOCOL_TLS_CLIENT enables check_hostname and CERT_REQUIRED by default + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + + # Set certificates for connection + if serverMode or (kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False)): + # Server mode always uses hostcert + __loadHostCertificate(ctx) + + else: + # Client mode has a choice of possible options + if kwargs.get("proxyString", None): + # ssl cannot take an in-memory location or a string, + # so write it to a temp file and use proxyLocation + with tempfile.NamedTemporaryFile(mode="w") as tmpFile: + tmpFile.write(kwargs["proxyString"]) + # Flush, otherwise the file is empty in the subsequent call + tmpFile.flush() + __loadProxy(ctx, proxyPath=tmpFile.name) + else: + # Use normal proxy + __loadProxy(ctx, proxyPath=kwargs.get("proxyLocation", None)) + + # Set peer verification + if kwargs.get("skipCACheck", False): + # Don't validate peer + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + else: + # Do validate peer + if serverMode and kwargs.get("optionalClientCert", False): + ctx.verify_mode = ssl.CERT_OPTIONAL + else: + ctx.verify_mode = ssl.CERT_REQUIRED + # Allow proxy certificates to be used + ctx.verify_flags |= VERIFY_ALLOW_PROXY_CERTS + # Set CA location + caPath = Locations.getCAsLocation() + if not caPath: + raise RuntimeError("Failed to find CA location") + if not os.path.isdir(caPath): + raise RuntimeError(f"CA path ({caPath}) is not a valid directory") + ctx.load_verify_locations(capath=caPath) + + # DIRAC_M2CRYPTO_SSL_CIPHERS is accepted for backward compatibility + ciphers = ( + kwargs.get("sslCiphers") or os.environ.get("DIRAC_SSL_CIPHERS") or os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") + ) + if ciphers: + ctx.set_ciphers(ciphers) + + return ctx + + +# Client-side SSL contexts are cached and shared between connections, as +# building one means reading and parsing the credential files for every RPC. +# An entry is invalidated when the credential file changes (e.g. proxy +# renewal), and expires after _CLIENT_CTX_CACHE_TTL seconds so that changes +# in the CA directory are eventually picked up. +_CLIENT_CTX_CACHE: OrderedDict = OrderedDict() +_CLIENT_CTX_CACHE_LOCK = threading.Lock() +_CLIENT_CTX_CACHE_MAX = 8 +_CLIENT_CTX_CACHE_TTL = 300 + + +def _clientContextCacheKey(kwargs): + """Compute the cache key for a client SSL context, or None if the + configuration is not cacheable. The credential selection mirrors + :py:func:`getSSLContext`, and the key contains the stat info of the + credential files so the cache is invalidated when they change. + + :param kwargs: the connection keywords, as given to getSSLContext + """ + if kwargs.get("bServerMode") or kwargs.get("proxyString"): + return None + if kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False): + credentialFiles = Locations.getHostCertificateAndKeyLocation() + if not credentialFiles: + return None + else: + proxyPath = kwargs.get("proxyLocation") or Locations.getProxyLocation() + if not proxyPath: + return None + credentialFiles = (proxyPath,) + + fileStats = [] + try: + for path in credentialFiles: + st = os.stat(path) + fileStats.append((path, st.st_mtime_ns, st.st_size, st.st_ino)) + except OSError: + return None + + skipCACheck = bool(kwargs.get("skipCACheck", False)) + caPath = None if skipCACheck else Locations.getCAsLocation() + ciphers = ( + kwargs.get("sslCiphers") or os.environ.get("DIRAC_SSL_CIPHERS") or os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") + ) + return (tuple(fileStats), skipCACheck, caPath, ciphers) + + +def _getClientSSLContext(**kwargs): + """Return a client SSL context, shared from the cache whenever possible + (ssl.SSLContext objects are safe to share between connections and threads) + """ + cacheKey = _clientContextCacheKey(kwargs) + if cacheKey is None: + return getSSLContext(**kwargs) + + now = time.monotonic() + with _CLIENT_CTX_CACHE_LOCK: + cached = _CLIENT_CTX_CACHE.get(cacheKey) + if cached and now - cached[0] < _CLIENT_CTX_CACHE_TTL: + _CLIENT_CTX_CACHE.move_to_end(cacheKey) + return cached[1] + + ctx = getSSLContext(**kwargs) + + with _CLIENT_CTX_CACHE_LOCK: + _CLIENT_CTX_CACHE[cacheKey] = (now, ctx) + _CLIENT_CTX_CACHE.move_to_end(cacheKey) + while len(_CLIENT_CTX_CACHE) > _CLIENT_CTX_CACHE_MAX: + _CLIENT_CTX_CACHE.popitem(last=False) + return ctx + + +def getPeerInfo(sslSocket): + """Gets the details of the current peer as a standard dict. The peer + details are obtained from the supplied ssl.SSLSocket. + The details returned are those from ~X509Chain.getCredentials, without Registry info: + + DN - Full peer DN as string + x509Chain - Full chain of peer + isProxy - Boolean, True if chain ends with proxy + isLimitedProxy - Boolean, True if chain ends with limited proxy + group - String, DIRAC group for this peer, if known + + Returns a dict of details. + """ + if not hasattr(sslSocket, "get_unverified_chain"): + raise RuntimeError("Extracting the peer certificate chain requires python >= 3.13 (ssl get_unverified_chain)") + # The chain, as sent by the peer (leaf first). It was already validated + # by OpenSSL during the handshake, and the proxy-ness of it is + # anyway cryptographically re-checked by X509Chain + derChain = sslSocket.get_unverified_chain() + if not derChain: + return {} + chain = X509Chain.generateX509ChainFromDERList(derChain) + creds = chain.getCredentials(withRegistryInfo=False) + if not creds["OK"]: + raise RuntimeError(f"Failed to get SSL peer info ({creds['Message']}).") + peer = creds["Value"] + + # getCredentials already resolves DN (identity for proxies, subject + # otherwise), isProxy and isLimitedProxy + peer["x509Chain"] = chain + + return peer + + +class SSLTransport(BaseTransport): + """SSL Transport implementation based on the standard library ssl module.""" + + # This name is the same as BaseClient, + # and is used a bit everywhere, so it should be factorized out + # eventually + KW_TIMEOUT = "timeout" + + def __init__(self, *args, **kwargs): + """Create an SSLTransport object, parameters are the same + as for other transports. If ctx is specified (as an instance of + ssl.SSLContext) then use that rather than creating a new context. + + kwargs can contain all the parameters defined in BaseClient, + in particular timeout + """ + self.remoteAddress = None + self.peerCredentials = {} + + # The timeout is to be understood here as the timeout for socket + # operations involved in the RPC call, but NOT the establishment + # of the connection, for which there is a different timeout + # (DEFAULT_CONNECTION_TIMEOUT) + self.__timeout = kwargs.get(SSLTransport.KW_TIMEOUT, DEFAULT_RPC_TIMEOUT) + + self.__locked = False # We don't support locking, so this is always false. + + self.__ctx = kwargs.pop("ctx", None) + if not self.__ctx: + if kwargs.get("bServerMode", False): + self.__ctx = getSSLContext(**kwargs) + else: + self.__ctx = _getClientSSLContext(**kwargs) + + # Note that kwargs is already kept in BaseTransport + # as self.extraArgsDict, but at least I am sure that + # self.__kwargs will never be modified + self.__kwargs = kwargs + + BaseTransport.__init__(self, *args, **kwargs) + + def setSocketTimeout(self, timeout): + """Set the timeout for RPC calls. + + .. warning: This needs to be called before initAsClient. + It is used as a timeout for RPC calls, not connection. + + :param timeout: timeout for socket operation in seconds + + """ + self.__timeout = timeout + + def initAsClient(self): + """Prepare this client socket for use.""" + if self.serverMode(): + raise RuntimeError("SSLTransport is in server mode.") + + errors = [] + host, port = self.stServerAddress + + # Get all available addresses (IPv6 and IPv4) and try them in order + try: + addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except OSError as e: + return S_ERROR(f"DNS lookup failed {e!r}") + for family, socketType, proto, _canonname, socketAddress in addrInfoList: + rawSocket = None + try: + rawSocket = socket.socket(family, socketType, proto) + + # First set a short connection timeout, that will be used + # for the TCP connection and the TLS handshake + rawSocket.settimeout(DEFAULT_CONNECTION_TIMEOUT) + + # Enable keepAlive, with default options + # (see more comments about keepalive in :py:meth:`.acceptConnection`) + rawSocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + + rawSocket.connect(socketAddress) + + # server_hostname takes care of both SNI and the hostname check + self.oSocket = self.__ctx.wrap_socket(rawSocket, server_hostname=host) + + # Once the connection is established, we can use the timeout + # asked for RPC + self.oSocket.settimeout(self.__timeout) + + self.remoteAddress = self.oSocket.getpeername() + + return S_OK() + # warning: do NOT catch SSL related errors here + # They should be propagated upwards and caught by the BaseClient + # not to enter the retry loop + except ssl.SSLError: + if rawSocket is not None: + rawSocket.close() + raise + except OSError as e: + errors.append(f"{socketAddress} {e}:{repr(e)}") + if self.oSocket is not None: + self.close() + elif rawSocket is not None: + rawSocket.close() + + return S_ERROR("; ".join(errors)) + + def initAsServer(self): + """Prepare this server socket for use. + + Contrary to the client mode, the listening socket is a plain TCP + socket: the TLS layer is only added on the accepted connections, + in :py:meth:`.handshake` + """ + if not self.serverMode(): + raise RuntimeError("SSLTransport is in client mode.") + + try: + self.oSocket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + except OSError: + # Maybe no IPv6 support? Try IPv4 only socket. + self.oSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + # Make sure reuse address is set correctly + param = 1 if self.bAllowReuseAddress else 0 + self.oSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, param) + + self.oSocket.bind(self.stServerAddress) + self.oSocket.listen(self.iListenQueueSize) + return S_OK() + + def close(self): + """Close this socket.""" + if self.oSocket: + try: + self.oSocket.shutdown(socket.SHUT_RDWR) + except OSError: + # The socket may already be in a closed state + pass + self.oSocket.close() + self.oSocket = None + return S_OK() + + def renewServerContext(self): + """Renews the server context. + This reloads the certificates and the CA store by re-creating + the SSL context, which will be used for the subsequent connections. + """ + if not self.serverMode(): + raise RuntimeError("SSLTransport is in client mode.") + super().renewServerContext() + try: + self.__ctx = getSSLContext(**self.__kwargs) + except Exception as e: + gLogger.error("Failed to renew the server context", repr(e)) + return S_ERROR(f"Failed to renew the server context: {e!r}") + + return S_OK() + + def handshake(self): + """Perform the SSL handshake. + This has to be called after the connection was accepted (acceptConnection) + + The remote credentials are gathered here + """ + try: + # Warning: this method is called on the object returned + # by acceptConnection, which shares the context of the + # listening object + self.oSocket = self.__ctx.wrap_socket(self.oSocket, server_side=True) + + self.peerCredentials = getPeerInfo(self.oSocket) + + # Now that the handshake has been performed on the server + # we can set the timeout for the RPC operations. + # In practice, since we are on the server side, the + # timeout we set here represents the timeout for receiving + # the arguments and sending back the response. This should + # in principle be reasonably quick, but just to be sure + # we can set it to the DEFAULT_RPC_TIMEOUT + self.oSocket.settimeout(DEFAULT_RPC_TIMEOUT) + + return S_OK() + except (OSError, RuntimeError) as e: + return S_ERROR(f"Error in handshake: {e} {repr(e)}") + + def setClientSocket(self, oSocket): + """Set the inner socket of this instance to the value of oSocket. + This method is intended to be used to create client connection objects + from a server and should be considered to be an internal function. + + :param oSocket: client socket (plain, the handshake is done in :py:meth:`.handshake`) + + """ + # warning: do NOT catch socket.error here, because for who knows what reason + # exceptions are actually properly used for once, and the calling method + # relies on it (ServiceReactor.__acceptIncomingConnection) + self.oSocket = oSocket + self.remoteAddress = self.oSocket.getpeername() + + def acceptConnection(self): + """Accept a new client, returns a new SSLTransport object representing + the client connection. + + The connection is accepted, but no SSL handshake is performed + + :returns: S_OK(SSLTransport object) + """ + try: + oClient, _ = self.oSocket.accept() + + # Set the keep alive to true. This keepalive will ensure that we + # detect remote peer crashing or network interruption + # Note that this is ineffective if we are in the middle of blocking + # operations. + oClient.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + + # Here we set the timeout server side. + # We first set the connection timeout, which will + # be effective for the TLS handshake + oClient.settimeout(DEFAULT_CONNECTION_TIMEOUT) + + oClientTrans = SSLTransport(self.stServerAddress, ctx=self.__ctx) + oClientTrans.setClientSocket(oClient) + return S_OK(oClientTrans) + except (OSError, RuntimeError) as e: + return S_ERROR(f"Error in acceptConnection: {e} {repr(e)}") + + def _read(self, bufSize=4096, skipReadyCheck=False): + """Read bufSize bytes from the buffer. + + :param bufSize: size of the buffer to read + :param skipReadyCheck: ignored. + + + :returns: S_OK(data read) + """ + try: + read = self.oSocket.recv(bufSize) + return S_OK(read) + except OSError as e: + return S_ERROR(f"Error in _read: {e} {repr(e)}") + + def isLocked(self): + """Returns if this instance is locked. + Always returns false. + + :returns: False + """ + return self.__locked + + def _write(self, buf): + """Write all bytes contained within iterable "buf" to the + connected peer. + + :param buf: iterable buffer + + :returns: S_OK(number of bytes written) + """ + try: + wrote = self.oSocket.send(buf) + return S_OK(wrote) + except OSError as e: + return S_ERROR(f"Error in _write: {e} {repr(e)}") diff --git a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py index 0e8134652db..c80bff60f3e 100644 --- a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py @@ -1,14 +1,15 @@ """ Test the SSLTransport mechanism """ import os import selectors +import sys import threading from diraccfg import CFG -from pytest import fixture +from pytest import fixture, mark, param from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.Core.DISET.private.Transports import M2SSLTransport, PlainTransport -from DIRAC.Core.Security.test.x509TestUtilities import CERTDIR, USERCERT, getCertOption +from DIRAC.Core.DISET.private.Transports import PlainTransport, StdSSLTransport +from DIRAC.Core.Security.test.x509TestUtilities import CERTDIR, USERCERT, getCertOption, skipm2 # TODO: Expired hostcert # TODO: Expired usercert @@ -38,7 +39,16 @@ # Transports are now tested in pairs: # "Server-Client" # Each pair is defined as a string. -TRANSPORTTESTS = ("Plain-Plain", "M2-M2") +TRANSPORTTESTS = ( + "Plain-Plain", + # The server side of the standard library ssl transport needs + # SSLSocket.get_unverified_chain, which was added in python 3.13 + param( + "SSL-SSL", + marks=mark.skipif(sys.version_info < (3, 13), reason="needs python >= 3.13 (ssl get_unverified_chain)"), + ), + param("M2-M2", marks=skipm2), +) # https://www.ibm.com/developerworks/linux/library/l-openssl/index.html @@ -121,7 +131,12 @@ def transportByName(transport): """helper function to get a transport class by 'friendly' name.""" if transport.lower() == "plain": return PlainTransport.PlainTransport + elif transport.lower() == "ssl": + return StdSSLTransport.SSLTransport elif transport.lower() == "m2": + # Imported lazily as M2Crypto may not be installed + from DIRAC.Core.DISET.private.Transports import M2SSLTransport + return M2SSLTransport.SSLTransport raise RuntimeError(f"Unknown Transport Name: {transport}") @@ -198,6 +213,27 @@ def test_simpleMessage(create_serverAndClient): assert serverAnswer == MAGIC_ANSWER +def test_clientContextCache(tmp_path): + """Client SSL contexts are shared until the credential file changes""" + from DIRAC.Core.DISET.private.Transports.StdSSLTransport import _CLIENT_CTX_CACHE, _getClientSSLContext + + proxyCopy = tmp_path / "proxy.pem" + proxyCopy.write_bytes(open(proxyFile, "rb").read()) + + _CLIENT_CTX_CACHE.clear() + ctx1 = _getClientSSLContext(proxyLocation=str(proxyCopy), skipCACheck=True) + ctx2 = _getClientSSLContext(proxyLocation=str(proxyCopy), skipCACheck=True) + assert ctx1 is ctx2 + + # Changing the credential file invalidates the cache entry + st = os.stat(proxyCopy) + os.utime(proxyCopy, ns=(st.st_atime_ns, st.st_mtime_ns + 10**9)) + ctx3 = _getClientSSLContext(proxyLocation=str(proxyCopy), skipCACheck=True) + assert ctx3 is not ctx1 + + _CLIENT_CTX_CACHE.clear() + + def test_getRemoteInfo(create_serverAndClient): """Check the information from remote peer""" serv, client = create_serverAndClient diff --git a/src/DIRAC/Core/Security/__init__.py b/src/DIRAC/Core/Security/__init__.py index 2f2209f7de1..16784f1aff7 100644 --- a/src/DIRAC/Core/Security/__init__.py +++ b/src/DIRAC/Core/Security/__init__.py @@ -1,8 +1,68 @@ """ -Just a selector for X509Certificate +DIRAC X509 security infrastructure. + +This package provides the X509Certificate, X509Chain, X509Request and X509CRL +classes as submodules, as well as a set of OID constants used for handling +proxies and VOMS extensions. + +Two implementations of the X509 classes are available: the default one, based +on pyca/cryptography (in the ``pyca`` subpackage), and the legacy one, based +on M2Crypto and pyasn1 (in the ``m2crypto`` subpackage). The implementation is +selected once, at import time, with the DIRAC_USE_M2CRYPTO environment +variable (default No). """ +import os from pkgutil import extend_path +# List of OIDs used in handling VOMS extension. +# VOMS extension is encoded in ASN.1 format and it's surprisingly hard to decode. OIDs describe content of sections +# of the data. There is no "official list of OIDs", ones used here are sourced from analyzing VOMS extensions itself +# and different pieces of code and presentations in subject of X509 certificates, certificate extensions and VOMS. +# Googling names or values of those OIDs, especially VOMS related, usually result in up to three pages of results, +# mainly Java code defining those values like code below. +# This is literally lookup table, so I know WTH is this, when I read value and see '1.3.6.1.4.1.8005.100.100.4'. + +DOMAIN_COMPONENT_OID = "0.9.2342.19200300.100.1.25" +DIRAC_GROUP_OID = "1.2.42.42" +VOMS_FQANS_OID = "1.3.6.1.4.1.8005.100.100.4" +VOMS_EXTENSION_OID = "1.3.6.1.4.1.8005.100.100.5" +VOMS_TAGS_EXT_OID = "1.3.6.1.4.1.8005.100.100.11" +COMMON_NAME_OID = "2.5.4.3" +SURNAME_OID = "2.5.4.4" +SERIALNUMBER_OID = "2.5.4.5" +COUNTRY_NAME = "2.5.4.6" +LOCALITY_NAME = "2.5.4.7" +STATE_OR_PROVINCE_NAME = "2.5.4.8" +ORGANIZATION_NAME = "2.5.4.10" +ORGANIZATIONAL_UNIT_NAME_OID = "2.5.4.11" +TITLE_OID = "2.5.4.12" +GIVEN_NAME_OID = "2.5.4.42" + +# See https://tools.ietf.org/html/rfc3820#appendix-A +PROXY_CERT_INFO_EXTENSION_OID = "1.3.6.1.5.5.7.1.14" +PROXY_OID = "1.3.6.1.5.5.7.21.1" +LIMITED_PROXY_OID = "1.3.6.1.4.1.3536.1.1.1.9" + +# Some specific distinguished names: https://www.cryptosys.net/pki/manpki/pki_distnames.html + +DN_MAPPING = { + COMMON_NAME_OID: "/CN=", + COUNTRY_NAME: "/C=", + DOMAIN_COMPONENT_OID: "/DC=", + GIVEN_NAME_OID: "/G=", + LOCALITY_NAME: "/L=", + ORGANIZATION_NAME: "/O=", + ORGANIZATIONAL_UNIT_NAME_OID: "/OU=", + SERIALNUMBER_OID: "/SERIALNUMBER=", + STATE_OR_PROVINCE_NAME: "/ST=", + SURNAME_OID: "/SN=", + TITLE_OID: "/T=", +} + + +#: Default strength of the proxy in bit +DEFAULT_PROXY_STRENGTH = 2048 + ##### # SUPER DISGUSTING HACK @@ -24,7 +84,11 @@ #### -# Add the m2crypto subpackage to the search path -# This allows imports like 'from DIRAC.Core.Security.X509Chian...' to work transparently +# If we want to use M2Crypto, we add the m2crypto subpackage to the search path, +# otherwise the default pyca/cryptography based subpackage is used. +# This allows imports like 'from DIRAC.Core.Security.X509Chain...' to work transparently # Nice kind of tricks you find in libraries like xml... -__path__ = extend_path(__path__, __name__ + ".m2crypto") +if os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true"): + __path__ = extend_path(__path__, __name__ + ".m2crypto") +else: + __path__ = extend_path(__path__, __name__ + ".pyca") diff --git a/src/DIRAC/Core/Security/pyca/X509CRL.py b/src/DIRAC/Core/Security/pyca/X509CRL.py new file mode 100644 index 00000000000..b561403e087 --- /dev/null +++ b/src/DIRAC/Core/Security/pyca/X509CRL.py @@ -0,0 +1,98 @@ +""" X509CRL is a class for managing X509CRL +This class is used to manage the revoked certificates.... +""" +import datetime + +from cryptography import x509 + +from DIRAC import S_OK, S_ERROR +from DIRAC.Core.Security.pyca import asn1_utils +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.File import secureOpenForWrite + +# pylint: disable=broad-except + + +class X509CRL: + def __init__(self, cert=None): + self.__pemData = "" + + if cert: + self.__loadedCert = True + self.__revokedCert = cert + else: + self.__loadedCert = False + + @classmethod + def instanceFromFile(cls, crlLocation): + """Instance a X509CRL from a file""" + crl = cls() + result = crl.loadCRLFromFile(crlLocation) + if not result["OK"]: + return result + return S_OK(crl) + + def loadCRLFromFile(self, crlLocation): + """ + Load a x509CRL certificate from a pem file + Return : S_OK / S_ERROR + """ + self.__loadedCert = False + try: + with open(crlLocation, "rb") as crlFile: + pemData = crlFile.read() + self.__revokedCert = x509.load_pem_x509_crl(pemData) + except Exception as e: + return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')}") + self.__loadedCert = True + self.__pemData = pemData.decode("ascii") + return S_OK() + + def __bytes__(self): + if not self.__loadedCert: + return b"No certificate loaded" + return self.__pemData.encode("ascii") + + def __str__(self): + return self.__pemData + + def dumpAllToString(self): + """ + Dump all to string + """ + if not self.__loadedCert: + return S_ERROR(DErrno.ECERTREAD, "No certificate loaded") + return S_OK(self.__pemData) + + def dumpAllToFile(self, filename=False): + """ + Dump all to file. If no filename specified a temporal one will be created + """ + if not self.__loadedCert: + return S_ERROR("No certificate loaded") + try: + with secureOpenForWrite(filename) as (fd, filename): + fd.write(self.__pemData) + except Exception as e: + return S_ERROR(DErrno.EWF, f"{filename}: {repr(e).replace(',)', ')')}") + return S_OK(filename) + + def hasExpired(self): + if not self.__loadedCert: + return S_ERROR("No certificate loaded") + nextUpdate = self.__revokedCert.next_update_utc + if nextUpdate is None: + return S_OK(False) + return S_OK(datetime.datetime.now(datetime.timezone.utc) > nextUpdate) + + def getIssuer(self): + if not self.__loadedCert: + return S_ERROR("No certificate loaded") + return S_OK(asn1_utils.nameToDN(self.__revokedCert.issuer)) + + def __repr__(self): + repStr = " X509Request -> X509Chain -> X509Certificate + from DIRAC.Core.Security.X509Request import X509Request + + req = X509Request() + req.generateProxyRequest(bitStrength=bitStrength, limited=limited) + + return S_OK(req) + + @executeOnlyIfCertLoaded + def getRemainingSecs(self): + """ + Get remaining lifetime in secs + + :returns: S_OK(remaining seconds) + """ + notAfter = self.getNotAfterDate()["Value"] + now = datetime.datetime.utcnow() + remainingSeconds = max(0, int((notAfter - now).total_seconds())) + + return S_OK(remainingSeconds) + + @executeOnlyIfCertLoaded + def getExtensions(self): + """ + Get a decoded list of extensions + + :returns: S_OK( list of tuple (extensionName, extensionValue)) + """ + extList = [] + for extension in self._certObj.extensions: + name = _EXT_OID_TO_NAME.get(extension.oid.dotted_string, "UNDEF") + try: + value = _formatExtensionValue(extension.value) + except Exception: + value = "Cannot decode value" + extList.append((name, value)) + + return S_OK(sorted(extList)) + + @executeOnlyIfCertLoaded + def verify(self, pkey): + """ + Verify the signature of the certificate using the public key provided + + :param pkey: cryptography public key object + + :returns: S_OK(bool) where the boolean shows the success of the verification + """ + try: + # The padding (PKCS1v15/PSS for RSA, ECDSA for EC) is derived from + # the signature algorithm of the certificate itself + signatureParameters = self._certObj.signature_algorithm_parameters + if isinstance(pkey, rsa.RSAPublicKey): + pkey.verify( + self._certObj.signature, + self._certObj.tbs_certificate_bytes, + signatureParameters, + self._certObj.signature_hash_algorithm, + ) + elif isinstance(pkey, ec.EllipticCurvePublicKey): + pkey.verify( + self._certObj.signature, + self._certObj.tbs_certificate_bytes, + signatureParameters, + ) + else: + pkey.verify(self._certObj.signature, self._certObj.tbs_certificate_bytes) + except Exception: + return S_OK(False) + return S_OK(True) + + @executeOnlyIfCertLoaded + def asPem(self): + """ + Return certificate as PEM string + + :returns: pem string + """ + return self._certObj.public_bytes(serialization.Encoding.PEM).decode("ascii") diff --git a/src/DIRAC/Core/Security/pyca/X509Chain.py b/src/DIRAC/Core/Security/pyca/X509Chain.py new file mode 100644 index 00000000000..d336610bf2d --- /dev/null +++ b/src/DIRAC/Core/Security/pyca/X509Chain.py @@ -0,0 +1,1051 @@ +""" X509Chain is a class for managing X509 chains with their Pkeys + +Link to the RFC 3820: https://tools.ietf.org/html/rfc3820 +In particular, limited proxy: https://tools.ietf.org/html/rfc3820#section-3.8 + +""" +from __future__ import annotations + +import copy +import hashlib +import os +import re +from collections import OrderedDict + +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +from DIRAC import S_ERROR, S_OK +from DIRAC.ConfigurationSystem.Client.Helpers import Registry +from DIRAC.Core.Security import ( + DEFAULT_PROXY_STRENGTH, + DIRAC_GROUP_OID, + LIMITED_PROXY_OID, + PROXY_CERT_INFO_EXTENSION_OID, + PROXY_OID, +) +from DIRAC.Core.Security.pyca import asn1_utils +from DIRAC.Core.Security.pyca.X509Certificate import X509Certificate +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.Decorators import executeOnlyIf +from DIRAC.Core.Utilities.File import secureOpenForWrite + +# Decorator to check that _certList is not empty +needCertList = executeOnlyIf("_certList", S_ERROR(DErrno.ENOCHAIN)) +# Decorator to check that the PKey has been loaded +needPKey = executeOnlyIf("_keyObj", S_ERROR(DErrno.ENOPKEY)) + +# Cache of parsed proxy files keyed by (path, mtime_ns, size, inode). +_PROXY_LOAD_CACHE: OrderedDict[tuple, dict] = OrderedDict() +_PROXY_LOAD_CACHE_MAX = 8 + +_PEM_KEY_PATTERN = r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----(?:.|\n)*?-----END [A-Z0-9 ]*PRIVATE KEY-----" + + +def dumpPrivateKeyPEM(keyObj): + """Serialize a private key as an unencrypted PKCS8 PEM string + + :param keyObj: cryptography private key object + + :returns: PEM encoded string + """ + return keyObj.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode("ascii") + + +class X509Chain: + """ + An X509Chain is basically a list of X509Certificate object, as well as a private key object, + which is associated to the X509Certificate the lowest in the chain. + + This is what you will want to use for user certificate (because they will turn into proxy....), and for + proxy. + + There are normally 4 ways you would instanciate an X509Chain object: + + * You are loading a proxy from a file + * Loading the chain from a file + * You are getting information about your peer during an SSL connection + * You are delegating + + Typical usages of X509Chain are illustrated below + + Loading a proxy from a file (this will load the chain and the key, assuming the key is in the same file):: + + proxy = X509Chain() + res = proxy.loadProxyFromFile(myFile) + if not res['OK']: + return res + + + Generating a proxy from a Certificate:: + + cert = X509Chain() + # Load user cert + retVal = cert.loadChainFromFile('/home/chaen/.globus/userkey.pem') + if not retVal['OK']: + return retVal + # Load the key from a different place, with a password + retVal = cert.loadKeyFromFile('/home/chaen/.globus/userkey.pem', password='MySecretKey') + if not retVal['OK']: + return res + + # Generate a limited proxy, valid one hour + retVal = cert.generateProxyToFile('/tmp/proxy.pem', + 3600, # only 1 h + diracGroup = 'lhcb_user', + strength= 2048, + limited=True) + + + Getting information from a peer in an SSL Connection:: + + # derList is the verified peer chain from ssl.SSLSocket.get_verified_chain() + chain = X509Chain.generateX509ChainFromDERList(derList) + creds = chain.getCredentials() + + + Delegating a proxy to a service:: + + # The server side generates a request + # Equivalent to ProxyManager.requestDelegationUpload + + x509Req = X509Request() + x509Req.generateProxyRequest() + + # This reqStr object is sent to the client + reqStr = x509Req.dumpRequest()['Value'] + + # This object contains both the public and private key + pkeyReq = x509Req.getPKey() + + ####################################################### + + # The client side signs the request, with its proxy + # Assume the proxy chain was already loaded one way or the otjer + + # The proxy will not contain a private key + res = proxyChain.generateChainFromRequestString(reqStr, lifetime=lifetime) + + # This is sent back to the server + delegatedProxyString = res['Value'] + + ###################################################### + # Equivalent to ProxyManager.completeDelegationUpload + + # Create the new chain + # the pkey was generated together with the Request + delegatedProxy = X509Chain(keyObj=pkeyReq) + delegatedProxy.loadChainFromString(delegatedProxyString) + + # make sure the public key match between Request and the new Chain + # (Stupid, of course it will ! But it is done in the ProxyManager...) + res = x509Req.checkChain(delegatedProxy) + + """ + + def __init__(self, certList=False, keyObj=False): + """ + C'tor + + :param certList: list of X509Certificate to constitute the chain + :param keyObj: cryptography private key object. The public or public/private key associated to + the last certificate of the chain + + """ + + # __isProxy is True if this chain represents a proxy + self.__isProxy = False + # Whether the proxy is limited or not + self.__isLimitedProxy = False + + # This is the position of the first proxy in the chain + self.__firstProxyStep = 0 + + # Cache for sha256 hash of the object + # This is just used as a unique identifier for + # indexing in the ProxyCache + self.__hash = False + + # List of X509Certificate constituing the chain + # The certificate in position N has been generated from the (N+1) + self._certList = [] + + # Place holder for the private key object + self._keyObj = None + + if certList: + # copy the content of the list, without copying the objects themselves + self._certList = copy.copy(certList) + # Immediately check if it is a proxy + self.__checkProxyness() + + if keyObj: + self._keyObj = keyObj + + @staticmethod + def generateX509ChainFromDERList(derList): + """Returns an instance of X509Chain from a list of DER encoded certificates, + as returned by ssl.SSLSocket.get_verified_chain() + + :param derList: iterable of DER encoded certificates, leaf first + + :returns: a X509Chain instance + """ + certList = [X509Certificate(x509Obj=x509.load_der_x509_certificate(der)) for der in derList] + return X509Chain(certList=certList) + + def loadChainFromFile(self, chainLocation): + """ + Load a x509 chain from a pem file + + :param chainLocation: path to the file + + :returns: S_OK/S_ERROR + """ + try: + with open(chainLocation) as fd: + pemData = fd.read() + except OSError as e: + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") + return self.loadChainFromString(pemData) + + def loadChainFromString(self, data): + """ + Load a x509 cert from a string containing the pem data + + :param data: data representing the chain of certificate in the + + Return : S_OK / S_ERROR + """ + try: + self._certList = self.__certListFromPemString(data) + except Exception as e: + return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')}") + + if not self._certList: + return S_ERROR(DErrno.EX509) + + # Update internals + self.__checkProxyness() + return S_OK() + + @staticmethod + def __certListFromPemString(certString): + """ + Create certificates list from string. String should contain certificates, just like plain text proxy file. + """ + # To get list of X509 certificates (not X509 Certificate Chain) from string it has to be parsed like that + # (constructors are not able to deal with big string) + certList = [] + pattern = r"(-----BEGIN CERTIFICATE-----((.|\n)*?)-----END CERTIFICATE-----)" + for cert in re.findall(pattern, certString): + certObj = X509Certificate() + result = certObj.loadFromString(cert[0]) + if not result["OK"]: + raise ValueError(result["Message"]) + certList.append(certObj) + return certList + + def loadKeyFromFile(self, chainLocation, password=False): + """ + Load a PKey from a pem file + + :param chainLocation: path to the file + :param password: password to decode the file. + + :returns: S_OK / S_ERROR + """ + try: + with open(chainLocation) as fd: + pemData = fd.read() + except Exception as e: + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") + return self.loadKeyFromString(pemData, password) + + def loadKeyFromString(self, pemData, password=False): + """ + Load a PKey from a string containing the pem data + + :param pemData: pem data of the key, potentially encoded with the password + :param password: password to decode the file. + + :returns: S_OK / S_ERROR + """ + self._keyObj = None + if isinstance(pemData, bytes): + pemData = pemData.decode("ascii") + + # The pem data may contain certificates as well (e.g. a proxy file), + # so extract the key block + keyBlocks = re.findall(_PEM_KEY_PATTERN, pemData) + if not keyBlocks: + return S_ERROR(DErrno.ECERTREAD, "No private key found in the pem data") + + if password: + password = password.encode() if isinstance(password, str) else password + else: + password = None + + try: + try: + self._keyObj = serialization.load_pem_private_key(keyBlocks[0].encode("ascii"), password=password) + except TypeError: + # A password was supplied but the key is not encrypted: + # be tolerant, like the previous implementations + if password is None: + raise + self._keyObj = serialization.load_pem_private_key(keyBlocks[0].encode("ascii"), password=None) + except Exception as e: + return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')} (Probably bad pass phrase?)") + + return S_OK() + + def setPKey(self, pkeyObj): + """ + Set the chain + Return : S_OK / S_ERROR + """ + self._keyObj = pkeyObj + return S_OK() + + def loadProxyFromFile(self, chainLocation): + """ + Load a Proxy from a pem file, that is both the Cert chain and the PKey + + :param chainLocation: path to the proxy file + + :returns: S_OK / S_ERROR + """ + try: + st = os.stat(chainLocation) + except OSError as e: + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") + + cacheKey = (chainLocation, st.st_mtime_ns, st.st_size, st.st_ino) + cached = _PROXY_LOAD_CACHE.get(cacheKey) + if cached is not None: + _PROXY_LOAD_CACHE.move_to_end(cacheKey) + # Shallow copy so callers mutating self._certList don't poison the cache + self._certList = list(cached["certList"]) + self._keyObj = cached["keyObj"] + self.__isProxy = cached["isProxy"] + self.__isLimitedProxy = cached["isLimitedProxy"] + self.__firstProxyStep = cached["firstProxyStep"] + self.__hash = False + return S_OK() + + try: + with open(chainLocation) as fd: + pemData = fd.read() + except Exception as e: + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") + result = self.loadProxyFromString(pemData) + if result["OK"]: + _PROXY_LOAD_CACHE[cacheKey] = { + "certList": list(self._certList), + "keyObj": self._keyObj, + "isProxy": self.__isProxy, + "isLimitedProxy": self.__isLimitedProxy, + "firstProxyStep": self.__firstProxyStep, + } + while len(_PROXY_LOAD_CACHE) > _PROXY_LOAD_CACHE_MAX: + _PROXY_LOAD_CACHE.popitem(last=False) + return result + + def loadProxyFromString(self, pemData): + """ + Load a Proxy from a pem buffer, that is both the Cert chain and the PKey + + :param pemData: PEM encoded cert chain and pkey + + :returns: S_OK / S_ERROR + """ + retVal = self.loadChainFromString(pemData) + if not retVal["OK"]: + return retVal + + return self.loadKeyFromString(pemData) + + @staticmethod + def __getProxyExtensionList(diracGroup=False, rfcLimited=False): + """ + Get the list of extensions necessary for a proxy. + Basically the keyUsage, the proxyCertInfo, and eventually the diracGroup + + :param diracGroup: name of the dirac group for the proxy + :param rfcLimited: boolean to generate for a limited proxy + + :returns: list of (cryptography.x509.ExtensionType, critical) tuples + """ + + extensions = [] + + # Standard certificate extensions + keyUsage = x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=True, + data_encipherment=True, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ) + extensions.append((keyUsage, True)) + + # Mandatory extension to be a proxy + policyOID = LIMITED_PROXY_OID if rfcLimited else PROXY_OID + proxyCertInfo = x509.UnrecognizedExtension( + x509.ObjectIdentifier(PROXY_CERT_INFO_EXTENSION_OID), asn1_utils.encodeProxyCertInfo(policyOID) + ) + extensions.append((proxyCertInfo, True)) + + # Add a dirac group + if diracGroup and isinstance(diracGroup, str): + diracGroupExt = x509.UnrecognizedExtension( + x509.ObjectIdentifier(DIRAC_GROUP_OID), asn1_utils.encodeDIRACGroup(diracGroup) + ) + extensions.append((diracGroupExt, False)) + + return extensions + + @needCertList + def getCertInChain(self, certPos=0): + """ + Get then a certificate in the chain + + :warning: Contrary to the pygsi version, this is not a copy! + + :param certPos: position of the certificate in the chain. Default: 0 + + :returns: S_OK(X509Certificate)/S_ERROR + """ + return S_OK(self._certList[certPos]) + + @needCertList + def getIssuerCert(self): + """ + Returns the issuer certificate of the last one if it is a proxy, otherwise + the last one in the chain + + :returns: S_OK(X509Certificate)/S_ERROR + """ + if self.__isProxy: + return S_OK(self._certList[self.__firstProxyStep + 1]) + return S_OK(self._certList[-1]) + + @needCertList + def getNumCertsInChain(self): + """ + length of the certificate chain + + :returns: length of the certificate chain + + + """ + return S_OK(len(self._certList)) + + # pylint: disable=unused-argument + @needCertList + @needPKey + def generateProxyToString( + self, lifetime, diracGroup=False, strength=DEFAULT_PROXY_STRENGTH, limited=False, proxyKey=False + ): + """ + Generate a proxy and get it as a string. + + Args: + lifetime (int): expected lifetime in seconds of proxy + diracGroup (str): diracGroup to add to the certificate + strength (int): length in bits of the pair if proxyKey not given (default 2048) + limited (bool): Create a limited proxy (default False) + proxyKey: cryptography key object (public, or private with public part) to certify. + If not given, generate one + + :returns: S_OK(PEM encoded string), S_ERROR. The PEM string contains all the certificates in the chain + and the private key associated to the last X509Certificate just generated. + """ + + issuerCert = self._certList[0] + + # If this is a certificate signing request then the private key will be + # appended by the server and we don't need to include it in the proxy + proxyPrivateKey = None + if not proxyKey: + proxyPrivateKey = rsa.generate_private_key(public_exponent=65537, key_size=strength) + proxyPublicKey = proxyPrivateKey.public_key() + elif hasattr(proxyKey, "public_key"): + # A private key object was given + proxyPublicKey = proxyKey.public_key() + else: + proxyPublicKey = proxyKey + + # Generate a new X509Certificate object, signed with one owns key + proxyExtensions = self.__getProxyExtensionList(diracGroup, limited) + res = X509Certificate.generateProxyCertFromIssuer( + issuerCert, proxyExtensions, proxyPublicKey, self._keyObj, lifetime=lifetime + ) + if not res["OK"]: + return res + proxyCert = res["Value"] + + # Generate the proxy string + proxyString = proxyCert.asPem() + if proxyPrivateKey: + proxyString += dumpPrivateKeyPEM(proxyPrivateKey) + for crt in self._certList: + proxyString += crt.asPem() + return S_OK(proxyString) + + # pylint: disable=unused-argument + def generateProxyToFile(self, filePath, lifetime, diracGroup=False, strength=DEFAULT_PROXY_STRENGTH, limited=False): + """ + Generate a proxy and put it into a file + + Args: + filePath: file to write + lifetime: expected lifetime in seconds of proxy + diracGroup: diracGroup to add to the certificate + strength: length in bits of the pair + limited: Create a limited proxy + """ + retVal = self.generateProxyToString(lifetime, diracGroup, strength, limited) + if not retVal["OK"]: + return retVal + try: + with secureOpenForWrite(filePath) as (fd, _filename): + fd.write(retVal["Value"]) + except Exception as e: + return S_ERROR(DErrno.EWF, f"{filePath} :{repr(e).replace(',)', ')')}") + return S_OK() + + @needCertList + def isProxy(self): + """ + Check whether this chain is a proxy + + :returns: S_OK(boolean) + """ + return S_OK(self.__isProxy) + + @needCertList + def isLimitedProxy(self): + """ + Check whether this chain is a limited proxy + + :returns: S_OK(boolean) + """ + return S_OK(self.__isProxy and self.__isLimitedProxy) + + @needCertList + def isValidProxy(self, ignoreDefault=False): + """ + Check whether this chain is a valid proxy, that is: + * a proxy + * still valid + * with a valid group + + :param ignoreDefault: (what a stupid name) if True, do not lookup the CS + + :returns: S_OK(True) if the proxy is valid, S_ERROR otherwise + + """ + if not self.__isProxy: + return S_ERROR(DErrno.ENOCHAIN, "Chain is not a proxy") + + if self.hasExpired()["Value"]: + return S_ERROR(DErrno.ENOCHAIN) + + if ignoreDefault: + groupRes = self.getDIRACGroup(ignoreDefault=True) + if not groupRes["OK"]: + return groupRes + if not groupRes["Value"]: + return S_ERROR(DErrno.ENOGROUP) + + return S_OK(True) + + def isVOMS(self): + """ + Check whether this proxy contains VOMS extensions. + It is enough for one of the certificate of the chain to have VOMS extension + + :returns: S_OK(boolean) + """ + + if not self.__isProxy: + return S_OK(False) + + for cert in self._certList: + if cert.hasVOMSExtensions()["Value"]: + return S_OK(True) + return S_OK(False) + + def isRFC(self): + """Check whether this is an RFC proxy. It can only be true, providing it is a proxy + + :returns: S_OK(boolean) + """ + + return self.isProxy() + + def getVOMSData(self): + """ + Returns the voms data. + + :returns: See :py:func:`~DIRAC.Core.Security.X509Certificate.X509Certificate.getVOMSData` + If no VOMS data is available, return DErrno.EVOMS + :warning: In case the chain is not a proxy, this method will return False. + Yes, it's stupid, but it is for compatibility... + + """ + if not self.__isProxy: + return S_OK(False) + + for cert in self._certList: + res = cert.getVOMSData() + if res["OK"]: + return res + return S_ERROR(DErrno.EVOMS) + + def __checkProxyness(self): + """This method is called upon initialization of a chain and fill in some internal attributes. + + Pure madness... + + To me, this method just seems to work by pure luck.. + """ + + self.__hash = False + self.__firstProxyStep = len(self._certList) - 2 # -1 is user cert by default, -2 is first proxy step + self.__isProxy = True + self.__isLimitedProxy = False + prevDNMatch = 2 + # If less than 2 steps in the chain is no proxy + if len(self._certList) < 2: + self.__isProxy = False + return + + # Here we make sure that each certificate in the chain was + # signed by the previous one + for step in range(len(self._certList) - 1): + # this is a cryptographic check with the keys + issuerMatch = self.__checkIssuer(step, step + 1) + if not issuerMatch: + self.__isProxy = False + return + + # Do we need to check the proxy DN? + if prevDNMatch: + dnMatch = self.__checkProxyDN(step, step + 1) + if dnMatch == 0: + # If we are not in the first step we've found the entity cert + if step > 0: + self.__firstProxyStep = step - 1 + # If we are in the first step this is not a proxy + else: + self.__isProxy = False + return + # Limited proxy DN match + elif dnMatch == 2: + self.__isLimitedProxy = True + if prevDNMatch != 2: + self.__isProxy = False + self.__isLimitedProxy = False + return + prevDNMatch = dnMatch + + def __checkProxyDN(self, certStep, issuerStep): + """ + Checks that the subject of the proxy is properly derived from the issuer subject. + + Args: + certStep: position of the certificate to check in self.__certList + issuerStep: position of the issuer certificate to check in self.__certList + + :returns: an int based on the match: + 0 = no match + 1 = proxy match + 2 = limited proxy match + """ + + issuerSubject = self._certList[issuerStep].getSubjectNameObject() + if not issuerSubject["OK"]: + return 0 + issuerSubject = issuerSubject["Value"] + + proxySubject = self._certList[certStep].getSubjectNameObject() + if not proxySubject["OK"]: + return 0 + proxySubject = proxySubject["Value"] + + lastEntry = str(proxySubject).split("/")[-1].split("=") + limited = False + if lastEntry[0] != "CN": + return 0 + + # For non-RFC proxy, the proxy always had these two strings in the CN + if lastEntry[1] not in ("proxy", "limited proxy"): + # for RFC proxy, one has to check the extension. + try: + proxyCertInfo = asn1_utils.decodeProxyCertInfo(self._certList[certStep]._certObj) + except (LookupError, ValueError): + return 0 + + if proxyCertInfo.proxyPolicy.policyLanguage.dotted_string == LIMITED_PROXY_OID: + limited = True + else: + if lastEntry[1] == "limited proxy": + limited = True + if not str(issuerSubject) == str(proxySubject)[: str(proxySubject).rfind("/")]: + return 0 + return 1 if not limited else 2 + + def __checkIssuer(self, certStep, issuerStep): + """ + Check that the issuer has signed the certificate with his private key + + :param certStep: position of the certificate in self.__certList + :param issuerStep: position of the issuer certificate + + :returns: boolean + + """ + issuerCert = self._certList[issuerStep] + cert = self._certList[certStep] + pubKey = issuerCert.getPublicKey()["Value"] + + return cert.verify(pubKey)["Value"] + + @needCertList + def getDIRACGroup(self, ignoreDefault=False): + """ + Retrieve the dirac group of the chain + + :param ignoreDefault: (default False) if True, do not lookup the CS for a group if it is not in the proxy + + :returns: S_OK(dirac group)/S_ERROR + """ + if not self.__isProxy: + return S_ERROR(DErrno.EX509, "Chain does not contain a valid proxy") + + # The code below will find the first match of the DIRAC group + for cert in reversed(self._certList): + # We specifically say we do not want the default to first check inside the proxy + retVal = cert.getDIRACGroup(ignoreDefault=True) + if retVal["OK"] and "Value" in retVal and retVal["Value"]: + return retVal + + # No DIRAC group found, try to get the default one + return self.getCertInChain(self.__firstProxyStep)["Value"].getDIRACGroup(ignoreDefault=ignoreDefault) + + @needCertList + def hasExpired(self): + """ + Check whether any element of the chain has expired + + :returns: S_OK(boolean) + """ + for cert in reversed(self._certList): + res = cert.hasExpired() + if not res["OK"]: + return res + # If True, it means the cert has expired + if res["Value"]: + return S_OK(True) + + return S_OK(False) + + @needCertList + def getNotAfterDate(self): + """ + Get the smallest not after date + + :returns: S_OK(datetime.datetime) + """ + notAfter = self._certList[0].getNotAfterDate() + if not notAfter["OK"]: + return notAfter + notAfter = notAfter["Value"] + + for cert in reversed(self._certList): + res = cert.getNotAfterDate() + if not res["OK"]: + return res + stepNotAfter = res["Value"] + + # If the current cert has already expired + # we return this as notAfter date + res = cert.hasExpired() + if not res["OK"]: + return res + if res["Value"]: + return S_OK(stepNotAfter) + + # if the current cert has a shorter lifetime + # as the current reference, take it as new reference + notAfter = min(notAfter, stepNotAfter) + + return S_OK(notAfter) + + @needCertList + def generateProxyRequest(self, bitStrength=DEFAULT_PROXY_STRENGTH, limited=False): + """ + Generate a proxy request. + See :py:meth:`DIRAC.Core.Security.X509Certificate.X509Certificate.generateProxyRequest` + + Return S_OK( X509Request ) / S_ERROR + """ + + # We use the first certificate of the chain to do the proxy request + x509Cert = self._certList[0] + return x509Cert.generateProxyRequest(bitStrength, limited) + + @needCertList + def getStrength(self): + """ + Returns the strength in bit of the key of the first certificate in the chain + """ + x509Cert = self._certList[0] + return x509Cert.getStrength() + + @needCertList + @needPKey + def generateChainFromRequestString(self, pemData, lifetime=86400, requireLimited=False, diracGroup=False): + """ + Generate a x509 chain from a request. + + :param pemData: PEM encoded request + :param lifetime: lifetime of the delegated proxy in seconds (default 1 day) + :param requireLimited: if True, requires a limited proxy + :param diracGroup: DIRAC group to put in the proxy + + :returns: S_OK( X509 chain pem encoded string ) / S_ERROR. The new chain will have been signed + with the public key included in the request + + """ + if not isinstance(pemData, bytes): + pemData = pemData.encode("ascii") + try: + req = x509.load_pem_x509_csr(pemData) + except Exception as e: + return S_ERROR(DErrno.ECERTREAD, f"Can't load request data: {repr(e).replace(',)', ')')}") + + # I am not sure this test makes sense. + # You can't request a limit proxy if you are yourself not limited ?! + # I think it should be a "or" instead of "and" + limited = requireLimited and self.isLimitedProxy().get("Value", False) + return self.generateProxyToString(lifetime, diracGroup, DEFAULT_PROXY_STRENGTH, limited, req.public_key()) + + @needCertList + def getRemainingSecs(self): + """ + Get remaining time (minimum of all cert in the chain) + + :returns: S_OK(time left in seconds) + """ + remainingSecs = self.getCertInChain(0)["Value"].getRemainingSecs()["Value"] + for cert in self._certList[1:]: + stepRS = cert.getRemainingSecs()["Value"] + remainingSecs = min(remainingSecs, stepRS) + + return S_OK(remainingSecs) + + @needCertList + def dumpAllToString(self): + """ + Dump the current chain as a PEM encoded string + The order would be: + + * first certificate + * private key (without passphrase) + * other certificates + + :returns: S_OK(PEM encoded chain with private key) + """ + data = self._certList[0].asPem() + if self._keyObj: + data += dumpPrivateKeyPEM(self._keyObj) + for cert in self._certList[1:]: + data += cert.asPem() + return S_OK(data) + + def dumpAllToFile(self, filename=False): + """ + Dump all to file. + + :param filename: If not specified, a temporary one will be created + + :returns: S_OK(filename)/S_ERROR + """ + retVal = self.dumpAllToString() + if not retVal["OK"]: + return retVal + pemData = retVal["Value"] + try: + with secureOpenForWrite(filename) as (fh, filename): + fh.write(pemData) + except Exception as e: + return S_ERROR(DErrno.EWF, f"{filename} :{repr(e).replace(',)', ')')}") + return S_OK(filename) + + @needCertList + def dumpChainToString(self): + """ + Dump only cert chain to string, without the PKey + + :returns: S_OK(pem chain) + """ + return S_OK("".join(cert.asPem() for cert in self._certList)) + + @needPKey + def dumpPKeyToString(self): + """ + Dump only the key to string, not encoded + + :returns: S_OK(PEM encoded key) + + """ + return S_OK(dumpPrivateKeyPEM(self._keyObj)) + + def __str__(self): + """String representation""" + repStr = " = ()" + Typically, the nickname will look like + 'nickname = chaen (lhcb)', + + * fqan: List of VOMS "position" (['/lhcb/Role=production/Capability=NULL', '/lhcb/Role=NULL/Capability=NULL']) + * vo: name of the VO, + * subject: subject DN to which the attributes were granted, + * issuer: typically the DN of the VOMS server (e.g '/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch') + + :raises: LookupError if the certificate has no VOMS extension + """ + vomsExtensionDict = {} + vomsExtensionData = retrieveExtension(cert, VOMS_EXTENSION_OID) + + # In principle, according to GFD 182, there could be more than one VO VOMS AC per proxy. + # The standard specifies that we have to accept at least the first one, which is what + # we do... + vomsCertAttribute = asn1.decode_der(_ACSequenceOfSequence, vomsExtensionData).acs[0] + + ###### + # TODO in principle, we should check the signature of the Attribute... + # (vomsCertAttribute.signatureAlgorithm / signatureValue) + ###### + + certAttrInfo = vomsCertAttribute.acinfo + + # The declarative API does things correctly by setting a timezone info in the datetime + # However, we do not in DIRAC, and so we can't compare the dates. + # We have to remove the timezone info from the datetime objects + validity = certAttrInfo.attrCertValidityPeriod + vomsExtensionDict["notBefore"] = validity.notBeforeTime.as_datetime().replace(tzinfo=None) + vomsExtensionDict["notAfter"] = validity.notAfterTime.as_datetime().replace(tzinfo=None) + + # The issuer and holder DNs have to be reconstructed from the rdnSequence + # of their directoryName + vomsExtensionDict["issuer"] = _dirNameToDN(certAttrInfo.issuer.issuerName[0]) + vomsExtensionDict["subject"] = _dirNameToDN(certAttrInfo.holder.baseCertificateID.issuer[0]) + + # ### Retrieving the FQAN #### + + # According to GFD182, there may be more attributes than just the FQAN, even though it + # does not seem to be the case in practice. So we make sure to have the good one + fqanOIDObj = x509.ObjectIdentifier(VOMS_FQANS_OID) + + # There shall be only one, hence the [0] + fqanAttrObj = [attrObj for attrObj in certAttrInfo.attributes if attrObj.type == fqanOIDObj][0] + + # According to GFD182 3.4.1, we decode the value as a IetfAttrSyntax. + # Since multiple values are not allowed, just take the first item + fqanObj = fqanAttrObj.values.as_list()[0].parse(IetfAttrSyntax) + + # We retrieve the VO and the VOMS server + voName, _, _ = fqanObj.policyAuthority[0].as_str().split(":") + vomsExtensionDict["vo"] = voName + + # Now retrieve the position of the holder (group, role) + vomsExtensionDict["fqan"] = [fqanPosition.decode() for fqanPosition in fqanObj.values] + + # ############ End of the FQAN ################ + + # Now the Tags, called attributes in the dict... + + tagDescriptions = [] + vomsTagsOIDObj = x509.ObjectIdentifier(VOMS_TAGS_EXT_OID) + + # First find the tag containers + tagExtensionObj = [extObj for extObj in (certAttrInfo.extensions or []) if extObj.extnID == vomsTagsOIDObj] + + # If we found tags + if tagExtensionObj: + # Multiple is forbidden, so only one tag container + tagContainers = asn1.decode_der(_TagContainers, tagExtensionObj[0].extnValue) + + # TODO in principle, we should check that the policyAuthority + # and the one of the fqan are the same + for tagList in tagContainers.tagLists: + for tag in tagList.tags: + # This gives a string like + # nickname = chaen (lhcb) + tagDescriptions.append(f"{tag.name.decode()} = {tag.value.decode()} ({tag.qualifier.decode()})") + + vomsExtensionDict["attribute"] = ",".join(tagDescriptions) + + # #### Tags are done ################ + + return vomsExtensionDict + + +# Mapping from OID to the OpenSSL short names, used to render DNs the same way +# X509_NAME_oneline (i.e. M2Crypto's str(X509_Name)) used to +_OID_TO_SHORT_NAME = { + "2.5.4.3": "CN", + "2.5.4.4": "SN", + "2.5.4.5": "serialNumber", + "2.5.4.6": "C", + "2.5.4.7": "L", + "2.5.4.8": "ST", + "2.5.4.9": "street", + "2.5.4.10": "O", + "2.5.4.11": "OU", + "2.5.4.12": "title", + "2.5.4.42": "GN", + "2.5.4.43": "initials", + "2.5.4.46": "dnQualifier", + "2.5.4.65": "pseudonym", + "0.9.2342.19200300.100.1.1": "UID", + "0.9.2342.19200300.100.1.25": "DC", + "1.2.840.113549.1.9.1": "emailAddress", +} + + +def _onelineEscape(value) -> str: + """Escape an attribute value the way X509_NAME_oneline does: + bytes outside the printable ASCII range are rendered as ``\\xHH``. + This keeps DNs of certificates with non-ASCII characters identical + to what they always were (e.g. in the Registry). + """ + raw = value.encode("utf-8") if isinstance(value, str) else bytes(value) + if all(0x20 <= char <= 0x7E for char in raw): + return raw.decode("ascii") + return "".join(chr(char) if 0x20 <= char <= 0x7E else f"\\x{char:02X}" for char in raw) + + +def nameToDN(name: x509.Name) -> str: + """Render a cryptography x509.Name in the OpenSSL oneline format, + e.g. '/O=Dirac Computing/O=CERN/CN=MrUser' + + :param name: cryptography.x509.Name object + + :returns: the DN as a string + """ + dn = "" + for rdn in name.rdns: + for attr in rdn: + shortName = _OID_TO_SHORT_NAME.get(attr.oid.dotted_string, attr.oid.dotted_string) + dn += f"/{shortName}={_onelineEscape(attr.value)}" + return dn diff --git a/src/DIRAC/Core/Security/test/Test_X509Certificate.py b/src/DIRAC/Core/Security/test/Test_X509Certificate.py index 2bac7d7792b..61090065862 100644 --- a/src/DIRAC/Core/Security/test/Test_X509Certificate.py +++ b/src/DIRAC/Core/Security/test/Test_X509Certificate.py @@ -21,15 +21,16 @@ CERTCONTENTS, getCertOption, HOSTCERT, + skipm2, VOMSPROXY, VOMS_PROXY_ATTR, ) -from pytest import mark, fixture, skip +from pytest import mark, fixture, param, skip parametrize = mark.parametrize -X509CERTTYPES = ("M2_X509Certificate",) +X509CERTTYPES = ("PYCA_X509Certificate", param("M2_X509Certificate", marks=skipm2)) # This fixture will return a X509Certificate class # https://docs.pytest.org/en/latest/fixture.html#automatic-grouping-of-tests-by-fixture-instances @@ -45,7 +46,9 @@ def get_X509Certificate_class(request): x509Class = request.param - if x509Class == "M2_X509Certificate": + if x509Class == "PYCA_X509Certificate": + from DIRAC.Core.Security.pyca.X509Certificate import X509Certificate + elif x509Class == "M2_X509Certificate": from DIRAC.Core.Security.m2crypto.X509Certificate import X509Certificate else: raise NotImplementedError() diff --git a/src/DIRAC/Core/Security/test/Test_X509Chain.py b/src/DIRAC/Core/Security/test/Test_X509Chain.py index bd30201182e..37e2c3abfaf 100644 --- a/src/DIRAC/Core/Security/test/Test_X509Chain.py +++ b/src/DIRAC/Core/Security/test/Test_X509Chain.py @@ -79,7 +79,9 @@ def get_proxy(request): deimportDIRAC() x509Class = request.param - if x509Class == "M2_X509Chain": + if x509Class == "PYCA_X509Chain": + from DIRAC.Core.Security.pyca.X509Chain import X509Chain + elif x509Class == "M2_X509Chain": from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain else: raise NotImplementedError() diff --git a/src/DIRAC/Core/Security/test/x509TestUtilities.py b/src/DIRAC/Core/Security/test/x509TestUtilities.py index 530223ca9be..cec8dd45dbb 100644 --- a/src/DIRAC/Core/Security/test/x509TestUtilities.py +++ b/src/DIRAC/Core/Security/test/x509TestUtilities.py @@ -3,7 +3,20 @@ import sys from datetime import datetime -from pytest import fixture +from pytest import fixture, mark, param + +# The M2Crypto based implementation is only tested if its (optional) +# dependencies are installed +try: + import M2Crypto # noqa pylint: disable=unused-import + import pyasn1 # noqa pylint: disable=unused-import + import pyasn1_modules # noqa pylint: disable=unused-import + + M2CRYPTO_AVAILABLE = True +except ImportError: + M2CRYPTO_AVAILABLE = False + +skipm2 = mark.skipif(not M2CRYPTO_AVAILABLE, reason="M2Crypto and/or pyasn1 are not installed") # We use certificates stored in the same folder as this test file CERTDIR = os.path.join(os.path.dirname(__file__), "certs") @@ -283,24 +296,20 @@ def getCertOption(cert, optionName): def deimportDIRAC(): """clean all what has already been imported from DIRAC. - This method is extremely fragile, but hopefully, we can get ride of all these - messy tests soon, when PyGSI has gone. + Nothing needs to be done anymore: the two X509 implementations are + imported through their fully qualified subpackage paths + (DIRAC.Core.Security.pyca and DIRAC.Core.Security.m2crypto), so they can + coexist in the same process, whatever DIRAC_USE_M2CRYPTO says. """ - if len(X509CHAINTYPES) != 1 or len(X509REQUESTTYPES) != 1: - raise NotImplementedError( - "This no longer de-imports DIRAC, if we want to test another SSL wrapper " - "we will have to find another way of doing this or run a separate pytest " - "process again" - ) # for mod in list(sys.modules): # # You should be careful with what you remove.... # if (mod == 'DIRAC' or mod.startswith('DIRAC.')) and not mod.startswith('DIRAC.Core.Security.test'): # sys.modules.pop(mod) -X509CHAINTYPES = ("M2_X509Chain",) +X509CHAINTYPES = ("PYCA_X509Chain", param("M2_X509Chain", marks=skipm2)) -# This fixture will return a pyGSI or M2Crypto X509Chain class +# This fixture will return a pyca/cryptography or M2Crypto X509Chain class # https://docs.pytest.org/en/latest/fixture.html#automatic-grouping-of-tests-by-fixture-instances @@ -314,7 +323,9 @@ def get_X509Chain_class(request): x509Class = request.param - if x509Class == "M2_X509Chain": + if x509Class == "PYCA_X509Chain": + from DIRAC.Core.Security.pyca.X509Chain import X509Chain + elif x509Class == "M2_X509Chain": from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain else: raise NotImplementedError() @@ -325,7 +336,7 @@ def get_X509Chain_class(request): deimportDIRAC() -X509REQUESTTYPES = ("M2_X509Request",) +X509REQUESTTYPES = ("PYCA_X509Request", param("M2_X509Request", marks=skipm2)) # This fixture will return a X509Request class # https://docs.pytest.org/en/latest/fixture.html#automatic-grouping-of-tests-by-fixture-instances @@ -341,7 +352,9 @@ def get_X509Request(request): x509Class = request.param - if x509Class == "M2_X509Request": + if x509Class == "PYCA_X509Request": + from DIRAC.Core.Security.pyca.X509Request import X509Request + elif x509Class == "M2_X509Request": from DIRAC.Core.Security.m2crypto.X509Request import X509Request else: raise NotImplementedError() @@ -370,6 +383,8 @@ def get_X509Chain_from_X509Request(x509ReqObj): # In principle, we should deimport Dirac everywhere, but I am not even sure it makes any difference if "m2crypto" in x509ReqObj.__class__.__module__: from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain + elif "pyca" in x509ReqObj.__class__.__module__: + from DIRAC.Core.Security.pyca.X509Chain import X509Chain else: raise NotImplementedError() diff --git a/src/DIRAC/Core/Tornado/Server/TornadoServer.py b/src/DIRAC/Core/Tornado/Server/TornadoServer.py index f10eccd5742..8361de02651 100644 --- a/src/DIRAC/Core/Tornado/Server/TornadoServer.py +++ b/src/DIRAC/Core/Tornado/Server/TornadoServer.py @@ -9,13 +9,21 @@ import psutil import secrets -import M2Crypto.SSL +# With the legacy M2Crypto implementation (DIRAC_USE_M2CRYPTO=Yes), tornado +# must be configured to use the M2Crypto based iostream (from +# tornado_m2crypto, which requires the patched DIRACGrid tornado fork) before +# anything instantiates an SSLIOStream. With the default implementation +# neither M2Crypto nor tornado_m2crypto may be imported. +DIRAC_USE_M2CRYPTO = os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true") -import tornado.iostream +if DIRAC_USE_M2CRYPTO: + import M2Crypto.SSL -tornado.iostream.SSLIOStream.configure( # pylint: disable=no-member - "tornado_m2crypto.m2iostream.M2IOStream" -) # pylint: disable=wrong-import-position + import tornado.iostream + + tornado.iostream.SSLIOStream.configure( # pylint: disable=no-member + "tornado_m2crypto.m2iostream.M2IOStream" + ) # pylint: disable=wrong-import-position import tornado.platform.asyncio import tornado.ioloop @@ -23,6 +31,7 @@ from tornado.web import Application, RequestHandler from DIRAC import gConfig, gLogger, S_OK +from DIRAC.Core.DISET.private.Transports.SSLTransport import getSSLContext from DIRAC.Core.Security import Locations from DIRAC.Core.Utilities import Network, TimeUtilities from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager @@ -189,18 +198,29 @@ def startTornado(self): sLog.debug("Starting Tornado") # Prepare SSL settings - certs = Locations.getHostCertificateAndKeyLocation() - if certs is False: - sLog.fatal("Host certificates not found ! Can't start the Server") - raise ImportError("Unable to load certificates") - ca = Locations.getCAsLocation() - ssl_options = { - "certfile": certs[0], - "keyfile": certs[1], - "cert_reqs": M2Crypto.SSL.verify_peer, - "ca_certs": ca, - "sslDebug": DEBUG_M2CRYPTO, # Set to true if you want to see the TLS debug messages - } + if DIRAC_USE_M2CRYPTO: + certs = Locations.getHostCertificateAndKeyLocation() + if certs is False: + sLog.fatal("Host certificates not found ! Can't start the Server") + raise ImportError("Unable to load certificates") + ca = Locations.getCAsLocation() + ssl_options = { + "certfile": certs[0], + "keyfile": certs[1], + "cert_reqs": M2Crypto.SSL.verify_peer, + "ca_certs": ca, + "sslDebug": DEBUG_M2CRYPTO, # Set to true if you want to see the TLS debug messages + } + else: + # The client certificate is optional: some requests are authenticated + # with a token or as visitor instead (see BaseRequestHandler) + try: + ssl_options = getSSLContext(bServerMode=True, optionalClientCert=True) + except RuntimeError as e: + sLog.fatal("Unable to prepare the TLS settings ! Can't start the Server", repr(e)) + raise ImportError(f"Unable to prepare the TLS settings: {e}") from e + certs = Locations.getHostCertificateAndKeyLocation() + ca = Locations.getCAsLocation() # Init monitoring if self.activityMonitoring: @@ -214,7 +234,7 @@ def startTornado(self): tornado.ioloop.PeriodicCallback(self.__reportToMonitoring, self.__monitoringLoopDelay * 1000).start() for port, app in self.__appsSettings.items(): - sLog.debug(" - %s" % "\n - ".join([f"{k} = {ssl_options[k]}" for k in ssl_options])) + sLog.debug(f" - certfile = {certs[0]}\n - keyfile = {certs[1]}\n - ca_certs = {ca}") # Default server configuration settings = dict(compress_response=True, cookie_secret=secrets.token_hex(32)) diff --git a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py index bb57e93f2df..d08cbac8298 100644 --- a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py +++ b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py @@ -4,6 +4,7 @@ This module is basic for each of these components and describes the basic concept of access to them. """ +import os import time import inspect import threading @@ -28,6 +29,10 @@ from DIRAC.Resources.IdProvider.Utilities import getIdProviderIdentifiers from DIRAC.Resources.IdProvider.IdProviderFactory import IdProviderFactory +# Whether the legacy M2Crypto based TLS implementation is used instead of the +# default one, based on the standard library ssl module (see TornadoServer) +DIRAC_USE_M2CRYPTO = os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true") + def set_attribute(attr, val): """Decorator to determine target method settings. Set method attribute. @@ -733,6 +738,57 @@ def _gatherPeerCredentials(self, grants: list = None) -> dict: def _authzSSL(self): """Load client certchain in DIRAC and extract information. + :return: S_OK(dict)/S_ERROR() + """ + if DIRAC_USE_M2CRYPTO: + return self._authzSSLM2() + + try: + # The peer chain, as sent by the client (leaf first, DER encoded). + # It was validated by OpenSSL during the TLS handshake + derChain = self.request.connection.stream.socket.get_unverified_chain() + except Exception: + # Plain HTTP connection, or no client certificate presented + derChain = None + + # Boolean whether we are behind a balancer and can trust headers + balancer = gConfig.getValue("/WebApp/Balancer", "none") != "none" + + # Get the client certificate chain + if derChain: + peerChain = X509Chain.generateX509ChainFromDERList(derChain) + elif balancer: + if self.request.headers.get("X-Ssl_client_verify") == "SUCCESS" and self.request.headers.get("X-SSL-CERT"): + chainAsText = unquote(self.request.headers.get("X-SSL-CERT")) + peerChain = X509Chain() + peerChain.loadChainFromString(chainAsText) + else: + return S_ERROR(DErrno.ECERTFIND, "Valid certificate not found.") + else: + return S_ERROR(DErrno.ECERTFIND, "Valid certificate not found.") + + # Retrieve the credentials + res = peerChain.getCredentials(withRegistryInfo=False) + if not res["OK"]: + return res + + credDict = res["Value"] + + # getCredentials already resolves DN (identity for proxies, subject + # otherwise), isProxy and isLimitedProxy + credDict["x509Chain"] = peerChain + + # We check if client sends extra credentials... + if "extraCredentials" in self.request.arguments: + extraCred = self.get_argument("extraCredentials") + if extraCred: + credDict["extraCredentials"] = self.decode(extraCred)[0] + return S_OK(credDict) + + def _authzSSLM2(self): + """Load client certchain in DIRAC and extract information, using the + legacy M2Crypto based TLS implementation (DIRAC_USE_M2CRYPTO=Yes). + :return: S_OK(dict)/S_ERROR() """ try: diff --git a/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py b/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py index 3329e90718e..cf7f4cfcd32 100644 --- a/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py +++ b/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py @@ -6,7 +6,7 @@ from prompt_toolkit import prompt from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.Core.Base.Script import Script -from DIRAC.Core.Security.m2crypto import DEFAULT_PROXY_STRENGTH +from DIRAC.Core.Security import DEFAULT_PROXY_STRENGTH class CLIParams: diff --git a/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py b/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py index 37f75140487..76e59ffa265 100644 --- a/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py +++ b/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py @@ -30,408 +30,855 @@ SERIALNUMBER(serialNumber) """ +import os import re -import time import secrets import datetime import collections -from M2Crypto import m2, util, X509, ASN1, EVP, RSA - from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Resources.ProxyProvider.ProxyProvider import ProxyProvider - -class DIRACCAProxyProvider(ProxyProvider): - def __init__(self, parameters=None): - """Constructor""" - super().__init__(parameters) - self.log = gLogger.getSubLogger(__name__) - # Initialize - self.maxDict = {} - self.minDict = {} - self.bits = 2048 - self.algoritm = "sha256" - self.match = [] - self.supplied = ["CN"] - self.optional = ["C", "O", "OU", "emailAddress"] - self.dnList = ["C", "O", "OU", "CN", "emailAddress"] - # Distinguished names - self.fields2nid = X509.X509_Name.nid.copy() - self.fields2nid["DC"] = -1 # Add DN that is not liested in X509.X509_Name - self.fields2nid["domainComponent"] = -1 # Add DN description that is not liested in X509.X509_Name - self.fields2nid["organizationalUnitName"] = 18 # Add 'OU' description - self.fields2nid["countryName"] = 14 # Add 'C' description - self.fields2nid["SERIALNUMBER"] = 105 # Add 'SERIALNUMBER' distinguished name - self.nid2fields = {} # nid: list of distinguished names - # Specify standart fields - for field, nid in self.fields2nid.items(): - self.nid2fields.setdefault(nid, []).append(field) - self.dnInfoDictCA = {} - - def setParameters(self, parameters): - """Set new parameters - - :param dict parameters: provider parameters - - :return: S_OK()/S_ERROR() - """ - for k, v in parameters.items(): - if not isinstance(v, list) and k in ["Match", "Supplied", "Optional", "DNOrder"] + list(self.fields2nid): - parameters[k] = v.replace(", ", ",").split(",") - self.parameters = parameters - # If CA configuration file exist - if parameters.get("CAConfigFile"): - self.__parseCACFG() - if "Bits" in parameters: - self.bits = int(parameters["Bits"]) - if "Algoritm" in parameters: - self.algoritm = parameters["Algoritm"] - if "Match" in parameters: - self.match = [self.fields2nid[f] for f in parameters["Match"]] - if "Supplied" in parameters: - self.supplied = [self.fields2nid[f] for f in parameters["Supplied"]] - if "Optional" in parameters: - self.optional = [self.fields2nid[f] for f in parameters["Optional"]] - allFields = self.optional + self.supplied + self.match - if "DNOrder" in parameters: - self.dnList = [] - if not any([any([f in parameters["DNOrder"] for f in self.nid2fields[n]]) for n in allFields]): - return S_ERROR("DNOrder must contain all configured fields.") - for field in parameters["DNOrder"]: - if self.fields2nid[field] in allFields: - self.dnList.append(field) - - # Set defaults for distridutes names - self.nid2defField = {} - for field, value in list(self.parameters.items()): - if field in self.fields2nid and self.fields2nid[field] in allFields: - self.parameters[self.fields2nid[field]] = value - self.nid2defField[self.fields2nid[field]] = field - - # Read CA certificate - chain = X509Chain() - result = chain.loadChainFromFile(self.parameters["CertFile"]) - if result["OK"]: - result = chain.getCredentials() +# Two implementations are available: the default one, based on +# pyca/cryptography, and the legacy one, based on M2Crypto. The same public +# name (DIRACCAProxyProvider) is defined once, at import time, based on the +# DIRAC_USE_M2CRYPTO environment variable, matching the implementation used +# for the rest of the X509 layer (see DIRAC.Core.Security). M2Crypto may not +# be imported unless the fallback is enabled. +if os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true"): + import time + + from M2Crypto import m2, util, X509, ASN1, EVP, RSA + + class DIRACCAProxyProvider(ProxyProvider): + def __init__(self, parameters=None): + """Constructor""" + super().__init__(parameters) + self.log = gLogger.getSubLogger(__name__) + # Initialize + self.maxDict = {} + self.minDict = {} + self.bits = 2048 + self.algoritm = "sha256" + self.match = [] + self.supplied = ["CN"] + self.optional = ["C", "O", "OU", "emailAddress"] + self.dnList = ["C", "O", "OU", "CN", "emailAddress"] + # Distinguished names + self.fields2nid = X509.X509_Name.nid.copy() + self.fields2nid["DC"] = -1 # Add DN that is not liested in X509.X509_Name + self.fields2nid["domainComponent"] = -1 # Add DN description that is not liested in X509.X509_Name + self.fields2nid["organizationalUnitName"] = 18 # Add 'OU' description + self.fields2nid["countryName"] = 14 # Add 'C' description + self.fields2nid["SERIALNUMBER"] = 105 # Add 'SERIALNUMBER' distinguished name + self.nid2fields = {} # nid: list of distinguished names + # Specify standart fields + for field, nid in self.fields2nid.items(): + self.nid2fields.setdefault(nid, []).append(field) + self.dnInfoDictCA = {} + + def setParameters(self, parameters): + """Set new parameters + + :param dict parameters: provider parameters + + :return: S_OK()/S_ERROR() + """ + for k, v in parameters.items(): + if not isinstance(v, list) and k in ["Match", "Supplied", "Optional", "DNOrder"] + list( + self.fields2nid + ): + parameters[k] = v.replace(", ", ",").split(",") + self.parameters = parameters + # If CA configuration file exist + if parameters.get("CAConfigFile"): + self.__parseCACFG() + if "Bits" in parameters: + self.bits = int(parameters["Bits"]) + if "Algoritm" in parameters: + self.algoritm = parameters["Algoritm"] + if "Match" in parameters: + self.match = [self.fields2nid[f] for f in parameters["Match"]] + if "Supplied" in parameters: + self.supplied = [self.fields2nid[f] for f in parameters["Supplied"]] + if "Optional" in parameters: + self.optional = [self.fields2nid[f] for f in parameters["Optional"]] + allFields = self.optional + self.supplied + self.match + if "DNOrder" in parameters: + self.dnList = [] + if not any([any([f in parameters["DNOrder"] for f in self.nid2fields[n]]) for n in allFields]): + return S_ERROR("DNOrder must contain all configured fields.") + for field in parameters["DNOrder"]: + if self.fields2nid[field] in allFields: + self.dnList.append(field) + + # Set defaults for distridutes names + self.nid2defField = {} + for field, value in list(self.parameters.items()): + if field in self.fields2nid and self.fields2nid[field] in allFields: + self.parameters[self.fields2nid[field]] = value + self.nid2defField[self.fields2nid[field]] = field + + # Read CA certificate + chain = X509Chain() + result = chain.loadChainFromFile(self.parameters["CertFile"]) if result["OK"]: - result = self.__parseDN(result["Value"]["subject"]) - if not result["OK"]: - return result - self.dnInfoDictCA = result["Value"] - return S_OK() + result = chain.getCredentials() + if result["OK"]: + result = self.__parseDN(result["Value"]["subject"]) + if not result["OK"]: + return result + self.dnInfoDictCA = result["Value"] + return S_OK() - def checkStatus(self, userDN): - """Read ready to work status of proxy provider + def checkStatus(self, userDN): + """Read ready to work status of proxy provider - :param str userDN: user DN + :param str userDN: user DN - :return: S_OK()/S_ERROR() - """ - self.log.debug("Ckecking work status of", self.parameters["ProviderName"]) - result = self.__parseDN(userDN) - if not result["OK"]: - return result - dnInfoDict = result["Value"] - - try: - userNIDs = [self.fields2nid[f.split("=")[0]] for f in userDN.lstrip("/").split("/")] - except (ValueError, KeyError) as e: - return S_ERROR(f"Unknown DN field in used DN: {e}") - nidOrder = [self.fields2nid[f] for f in self.dnList] - for index, nid in enumerate(userNIDs): - if nid not in nidOrder: - return S_ERROR( - f'"{self.nid2defField.get(nid, min(self.nid2fields[nid], key=len))}" field not found in order.' - ) - if index > nidOrder.index(nid): - return S_ERROR("Bad DNs order") - for i in range(nidOrder.index(nid) - 1): - try: - if userNIDs.index(nidOrder[i]) > index: - return S_ERROR("Bad DNs order") - except (ValueError, KeyError): - continue - for i in range(nidOrder.index(nid) + 1, len(nidOrder)): - try: - if userNIDs.index(nidOrder[i]) < index: - return S_ERROR("Bad DNs order") - except (ValueError, KeyError): - continue - - for nid in self.supplied: - if nid not in [self.fields2nid[f] for f in dnInfoDict]: - return S_ERROR( - 'Current DN is invalid, "%s" field must be set.' - % self.nid2defField.get(nid, min(self.nid2fields[nid], key=len)) - ) - - for field, values in dnInfoDict.items(): - nid = self.fields2nid[field] - err = f'Current DN is invalid, "{field}" field' - if nid not in self.supplied + self.match + self.optional: - return S_ERROR(f"{err} is not found for current CA.") - if nid in self.match and not self.dnInfoDictCA[field] == values: - return S_ERROR(f"{err} must be /{field}={('/%s=' % field).joing(self.dnInfoDictCA[field])}.") - if nid in self.maxDict: - rangeMax = list(range(min(len(values), len(self.maxDict[nid])))) - if any([True if len(values[i]) > self.maxDict[nid][i] else False for i in rangeMax]): - return S_ERROR(f"{err} values must be less then {', '.join(self.maxDict[nid])}.") - if nid in self.minDict: - rangeMin = list(range(min(len(values), len(self.minDict[nid])))) - if any([True if len(values[i]) < self.minDict[nid][i] else False for i in rangeMin]): - return S_ERROR(f"{err} values must be more then {', '.join(self.minDict[nid])}.") - - result = self.__fillX509Name(field, values) + :return: S_OK()/S_ERROR() + """ + self.log.debug("Ckecking work status of", self.parameters["ProviderName"]) + result = self.__parseDN(userDN) if not result["OK"]: return result + dnInfoDict = result["Value"] + + try: + userNIDs = [self.fields2nid[f.split("=")[0]] for f in userDN.lstrip("/").split("/")] + except (ValueError, KeyError) as e: + return S_ERROR(f"Unknown DN field in used DN: {e}") + nidOrder = [self.fields2nid[f] for f in self.dnList] + for index, nid in enumerate(userNIDs): + if nid not in nidOrder: + return S_ERROR( + f'"{self.nid2defField.get(nid, min(self.nid2fields[nid], key=len))}" field not found in order.' + ) + if index > nidOrder.index(nid): + return S_ERROR("Bad DNs order") + for i in range(nidOrder.index(nid) - 1): + try: + if userNIDs.index(nidOrder[i]) > index: + return S_ERROR("Bad DNs order") + except (ValueError, KeyError): + continue + for i in range(nidOrder.index(nid) + 1, len(nidOrder)): + try: + if userNIDs.index(nidOrder[i]) < index: + return S_ERROR("Bad DNs order") + except (ValueError, KeyError): + continue + + for nid in self.supplied: + if nid not in [self.fields2nid[f] for f in dnInfoDict]: + return S_ERROR( + 'Current DN is invalid, "%s" field must be set.' + % self.nid2defField.get(nid, min(self.nid2fields[nid], key=len)) + ) + + for field, values in dnInfoDict.items(): + nid = self.fields2nid[field] + err = f'Current DN is invalid, "{field}" field' + if nid not in self.supplied + self.match + self.optional: + return S_ERROR(f"{err} is not found for current CA.") + if nid in self.match and not self.dnInfoDictCA[field] == values: + return S_ERROR(f"{err} must be /{field}={('/%s=' % field).joing(self.dnInfoDictCA[field])}.") + if nid in self.maxDict: + rangeMax = list(range(min(len(values), len(self.maxDict[nid])))) + if any([True if len(values[i]) > self.maxDict[nid][i] else False for i in rangeMax]): + return S_ERROR(f"{err} values must be less then {', '.join(self.maxDict[nid])}.") + if nid in self.minDict: + rangeMin = list(range(min(len(values), len(self.minDict[nid])))) + if any([True if len(values[i]) < self.minDict[nid][i] else False for i in rangeMin]): + return S_ERROR(f"{err} values must be more then {', '.join(self.minDict[nid])}.") + + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + return S_OK() + + def getProxy(self, userDN): + """Generate user proxy + + :param str userDN: user DN + + :return: S_OK(str)/S_ERROR() -- contain a proxy string + """ + self.__X509Name = X509.X509_Name() + result = self.checkStatus(userDN) + if result["OK"]: + result = self.__createCertM2Crypto() + if result["OK"]: + certStr, keyStr = result["Value"] + + chain = X509Chain() + result = chain.loadChainFromString(certStr) + if result["OK"]: + result = chain.loadKeyFromString(keyStr) + if result["OK"]: + result = chain.generateProxyToString(365 * 24 * 3600) + + return result + + def generateDN(self, **kwargs): + """Get DN of the user certificate that will be created - return S_OK() + :param dict kwargs: user description dictionary with possible fields: + - FullName or CN + - Email or emailAddress - def getProxy(self, userDN): - """Generate user proxy + :return: S_OK(str)/S_ERROR() -- contain DN + """ + if kwargs.get("FullName"): + kwargs["CN"] = [kwargs["FullName"]] + if kwargs.get("Email"): + kwargs["emailAddress"] = [kwargs["Email"]] - :param str userDN: user DN + self.__X509Name = X509.X509_Name() + self.log.info("Creating distinguished names chain") + + for nid in self.supplied: + if nid not in [self.fields2nid[f] for f in self.dnList]: + return S_ERROR( + 'DNs order list does not contain supplied DN "%s"' + % self.nid2defField.get(nid, min(self.nid2fields[nid], key=len)) + ) + + for field in self.dnList: + values = [] + nid = self.fields2nid[field] + if nid in self.match: + for field in self.nid2fields[nid]: + if field in self.dnInfoDictCA: + values = self.dnInfoDictCA[field] + if not values: + return S_ERROR(f'Not found "{field}" match DN in CA') + for field in self.nid2fields[nid]: + if kwargs.get(field): + values = kwargs[field] if isinstance(kwargs[field], list) else [kwargs[field]] + if not values and nid in self.supplied: + # Search default value + if nid not in self.nid2defField: + return S_ERROR(f'No values set for "{min(self.nid2fields[nid], key=len)}" DN') + values = self.parameters[nid] + + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + # WARN: This logic not support list of distribtes name elements + resDN = m2.x509_name_oneline(self.__X509Name.x509_name) # pylint: disable=no-member + + result = self.checkStatus(resDN) + if not result["OK"]: + return result + return S_OK(resDN) + + def __parseCACFG(self): + """Parse CA configuration file""" + block = "" + self.cfg = {} + self.supplied, self.optional, self.match, self.dnList = [], [], [], [] + with open(self.parameters["CAConfigFile"]) as caCFG: + for line in caCFG: + # Ignore comments + line = re.sub(r"#.*", "", line) + if re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", "")): + block = "".join(re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", ""))) + if block not in self.cfg: + self.cfg[block] = {} + if not block: + continue + if len(re.findall("=", line)) == 1: + field, val = line.split("=") + field = field.strip() + variables = re.findall(r"[$]([A-Za-z0-9_]+)", val) + for v in variables: + for b in self.cfg: + if v in self.cfg[b]: + val = val.replace("$" + v, self.cfg[b][v]) + if "default_ca" in self.cfg.get("ca", {}): + if "policy" in self.cfg.get(self.cfg["ca"]["default_ca"], {}): + if block == self.cfg[self.cfg["ca"]["default_ca"]]["policy"]: + self.dnList.append(field) + self.cfg[block][field] = val.strip() + + self.bits = int(self.cfg["req"].get("default_bits") or self.bits) + self.algoritm = self.cfg[self.cfg["ca"]["default_ca"]].get("default_md") or self.algoritm + if not self.parameters.get("CertFile"): + self.parameters["CertFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["certificate"] + self.parameters["KeyFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["private_key"] + # Read distinguished names + for k, v in self.cfg[self.cfg[self.cfg["ca"]["default_ca"]]["policy"]].items(): + nid = self.fields2nid[k] + self.parameters[nid], self.minDict[nid], self.maxDict[nid] = [], [], [] + for k in [f"{i}.{k}" for i in range(0, 5)] + [k]: + if k + "_default" in self.cfg["req"]["distinguished_name"]: + self.parameters[nid].append(self.cfg["req"]["distinguished_name"][k + "_default"]) + if k + "_min" in self.cfg["req"]["distinguished_name"]: + self.minDict[nid].append(self.cfg["req"]["distinguished_name"][k + "_min"]) + if k + "_max" in self.cfg["req"]["distinguished_name"]: + self.maxDict[nid].append(self.cfg["req"]["distinguished_name"][k + "_max"]) + if v == "supplied": + self.supplied.append(nid) + elif v == "optional": + self.optional.append(nid) + elif v == "match": + self.match.append(nid) + + def __parseDN(self, dn): + """Return DN fields + + :param str dn: DN + + :return: list -- contain tuple with positionOfField.fieldName, fieldNID, fieldValue + """ + dnInfoDict = collections.OrderedDict() + for f, v in [f.split("=") for f in dn.lstrip("/").split("/")]: + if not v: + return S_ERROR(f'No value set for "{f}"') + if f not in dnInfoDict: + dnInfoDict[f] = [v] + else: + dnInfoDict[f].append(v) + return S_OK(dnInfoDict) + + def __fillX509Name(self, field, values): + """Fill x509_Name object by M2Crypto + + :param str field: DN field name + :param list values: values of field, order important + + :return: S_OK()/S_ERROR() + """ + for value in values: + if ( + value + and m2.x509_name_set_by_nid( # pylint: disable=no-member + self.__X509Name.x509_name, self.fields2nid[field], value.encode() + ) + == 0 + ): + if ( + not self.__X509Name.add_entry_by_txt( + field=field, type=ASN1.MBSTRING_ASC, entry=value, len=-1, loc=-1, set=0 + ) + == 1 + ): + return S_ERROR(f'Cannot set "{field}" field.') + return S_OK() + + def __createCertM2Crypto(self): + """Create new certificate for user + + :return: S_OK(tuple)/S_ERROR() -- tuple contain certificate and pulic key as strings + """ + # Create public key + userPubKey = EVP.PKey() + userPubKey.assign_rsa(RSA.gen_key(self.bits, 65537, util.quiet_genparam_callback)) + # Create certificate + userCert = X509.X509() + userCert.set_pubkey(userPubKey) + userCert.set_version(2) + userCert.set_subject(self.__X509Name) + userCert.set_serial_number(secrets.randbits(64)) + # Add extentionals + userCert.add_ext(X509.new_extension("basicConstraints", "CA:" + str(False).upper())) + userCert.add_ext(X509.new_extension("extendedKeyUsage", "clientAuth", critical=1)) + # Set livetime + validityTime = datetime.timedelta(days=400) + notBefore = ASN1.ASN1_UTCTIME() + notBefore.set_time(int(time.time())) + notAfter = ASN1.ASN1_UTCTIME() + notAfter.set_time(int(time.time()) + int(validityTime.total_seconds())) + userCert.set_not_before(notBefore) + userCert.set_not_after(notAfter) + # Add subject from CA + with open(self.parameters["CertFile"]) as cf: + caCertStr = cf.read() + caCert = X509.load_cert_string(caCertStr) + userCert.set_issuer(caCert.get_subject()) + # Use CA key + with open(self.parameters["KeyFile"], "rb") as cf: + caKeyStr = cf.read() + pkey = EVP.PKey() + pkey.assign_rsa(RSA.load_key_string(caKeyStr, callback=util.no_passphrase_callback)) + # Sign + userCert.sign(pkey, self.algoritm) + + userCertStr = userCert.as_pem().decode("ascii") + userPubKeyStr = userPubKey.as_pem(cipher=None, callback=util.no_passphrase_callback).decode("ascii") + return S_OK((userCertStr, userPubKeyStr)) + + def _forceGenerateProxyForDN(self, dn, time, group=None): + """An additional helper method for creating a proxy without any substantial validation, + it can be used for a specific case(such as testing) where just need to generate a proxy + with specific DN on the fly. + + :param str dn: requested proxy DN + :param int time: expired time in a seconds + :param str group: if need to add DIRAC group + + :return: S_OK(tuple)/S_ERROR() -- contain proxy as chain and as string + """ + self.__X509Name = X509.X509_Name() + result = self.__parseDN(dn) + if not result["OK"]: + return result + dnInfoDict = result["Value"] + + for field, values in dnInfoDict.items(): + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result - :return: S_OK(str)/S_ERROR() -- contain a proxy string - """ - self.__X509Name = X509.X509_Name() - result = self.checkStatus(userDN) - if result["OK"]: result = self.__createCertM2Crypto() if result["OK"]: certStr, keyStr = result["Value"] - chain = X509Chain() - result = chain.loadChainFromString(certStr) + if chain.loadChainFromString(certStr)["OK"] and chain.loadKeyFromString(keyStr)["OK"]: + result = chain.generateProxyToString(time, diracGroup=group) + if not result["OK"]: + return result + chain = X509Chain() + chain.loadProxyFromString(result["Value"]) + return S_OK((chain, result["Value"])) + +else: + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + + from DIRAC.Core.Security import asn1_utils + + # Mapping between the distinguished name aliases and their OID, replacing + # the M2Crypto X509_Name.nid lookup table + _FIELDS_TO_OID = { + "C": "2.5.4.6", + "countryName": "2.5.4.6", + "SP": "2.5.4.8", + "ST": "2.5.4.8", + "stateOrProvinceName": "2.5.4.8", + "L": "2.5.4.7", + "localityName": "2.5.4.7", + "O": "2.5.4.10", + "organizationName": "2.5.4.10", + "OU": "2.5.4.11", + "organizationalUnitName": "2.5.4.11", + "CN": "2.5.4.3", + "commonName": "2.5.4.3", + "Email": "1.2.840.113549.1.9.1", + "emailAddress": "1.2.840.113549.1.9.1", + "SN": "2.5.4.4", + "surname": "2.5.4.4", + "GN": "2.5.4.42", + "givenName": "2.5.4.42", + "serialNumber": "2.5.4.5", + "SERIALNUMBER": "2.5.4.5", + "DC": "0.9.2342.19200300.100.1.25", + "domainComponent": "0.9.2342.19200300.100.1.25", + } + + _ALGORITHMS = { + "sha1": hashes.SHA1, + "sha224": hashes.SHA224, + "sha256": hashes.SHA256, + "sha384": hashes.SHA384, + "sha512": hashes.SHA512, + } + + class DIRACCAProxyProvider(ProxyProvider): + def __init__(self, parameters=None): + """Constructor""" + super().__init__(parameters) + self.log = gLogger.getSubLogger(__name__) + # Initialize + self.maxDict = {} + self.minDict = {} + self.bits = 2048 + self.algoritm = "sha256" + self.match = [] + self.supplied = [_FIELDS_TO_OID["CN"]] + self.optional = [_FIELDS_TO_OID[f] for f in ("C", "O", "OU", "emailAddress")] + self.dnList = ["C", "O", "OU", "CN", "emailAddress"] + # Distinguished names + self.fields2oid = dict(_FIELDS_TO_OID) + self.oid2fields = {} # oid: list of distinguished names + # Specify standart fields + for field, oid in self.fields2oid.items(): + self.oid2fields.setdefault(oid, []).append(field) + self.dnInfoDictCA = {} + # List of x509.NameAttribute of the DN being built + self.__nameAttributes = [] + + def setParameters(self, parameters): + """Set new parameters + + :param dict parameters: provider parameters + + :return: S_OK()/S_ERROR() + """ + for k, v in parameters.items(): + if not isinstance(v, list) and k in ["Match", "Supplied", "Optional", "DNOrder"] + list( + self.fields2oid + ): + parameters[k] = v.replace(", ", ",").split(",") + self.parameters = parameters + # If CA configuration file exist + if parameters.get("CAConfigFile"): + self.__parseCACFG() + if "Bits" in parameters: + self.bits = int(parameters["Bits"]) + if "Algoritm" in parameters: + self.algoritm = parameters["Algoritm"] + if "Match" in parameters: + self.match = [self.fields2oid[f] for f in parameters["Match"]] + if "Supplied" in parameters: + self.supplied = [self.fields2oid[f] for f in parameters["Supplied"]] + if "Optional" in parameters: + self.optional = [self.fields2oid[f] for f in parameters["Optional"]] + allFields = self.optional + self.supplied + self.match + if "DNOrder" in parameters: + self.dnList = [] + if not any([any([f in parameters["DNOrder"] for f in self.oid2fields[n]]) for n in allFields]): + return S_ERROR("DNOrder must contain all configured fields.") + for field in parameters["DNOrder"]: + if self.fields2oid[field] in allFields: + self.dnList.append(field) + + # Set defaults for distridutes names + self.oid2defField = {} + for field, value in list(self.parameters.items()): + if field in self.fields2oid and self.fields2oid[field] in allFields: + self.parameters[self.fields2oid[field]] = value + self.oid2defField[self.fields2oid[field]] = field + + # Read CA certificate + chain = X509Chain() + result = chain.loadChainFromFile(self.parameters["CertFile"]) + if result["OK"]: + result = chain.getCredentials() if result["OK"]: - result = chain.loadKeyFromString(keyStr) - if result["OK"]: - result = chain.generateProxyToString(365 * 24 * 3600) - - return result - - def generateDN(self, **kwargs): - """Get DN of the user certificate that will be created - - :param dict kwargs: user description dictionary with possible fields: - - FullName or CN - - Email or emailAddress - - :return: S_OK(str)/S_ERROR() -- contain DN - """ - if kwargs.get("FullName"): - kwargs["CN"] = [kwargs["FullName"]] - if kwargs.get("Email"): - kwargs["emailAddress"] = [kwargs["Email"]] - - self.__X509Name = X509.X509_Name() - self.log.info("Creating distinguished names chain") - - for nid in self.supplied: - if nid not in [self.fields2nid[f] for f in self.dnList]: - return S_ERROR( - 'DNs order list does not contain supplied DN "%s"' - % self.nid2defField.get(nid, min(self.nid2fields[nid], key=len)) - ) - - for field in self.dnList: - values = [] - nid = self.fields2nid[field] - if nid in self.match: - for field in self.nid2fields[nid]: - if field in self.dnInfoDictCA: - values = self.dnInfoDictCA[field] - if not values: - return S_ERROR(f'Not found "{field}" match DN in CA') - for field in self.nid2fields[nid]: - if kwargs.get(field): - values = kwargs[field] if isinstance(kwargs[field], list) else [kwargs[field]] - if not values and nid in self.supplied: - # Search default value - if nid not in self.nid2defField: - return S_ERROR(f'No values set for "{min(self.nid2fields[nid], key=len)}" DN') - values = self.parameters[nid] - - result = self.__fillX509Name(field, values) + result = self.__parseDN(result["Value"]["subject"]) if not result["OK"]: return result + self.dnInfoDictCA = result["Value"] + return S_OK() - # WARN: This logic not support list of distribtes name elements - resDN = m2.x509_name_oneline(self.__X509Name.x509_name) # pylint: disable=no-member + def checkStatus(self, userDN): + """Read ready to work status of proxy provider - result = self.checkStatus(resDN) - if not result["OK"]: - return result - return S_OK(resDN) - - def __parseCACFG(self): - """Parse CA configuration file""" - block = "" - self.cfg = {} - self.supplied, self.optional, self.match, self.dnList = [], [], [], [] - with open(self.parameters["CAConfigFile"]) as caCFG: - for line in caCFG: - # Ignore comments - line = re.sub(r"#.*", "", line) - if re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", "")): - block = "".join(re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", ""))) - if block not in self.cfg: - self.cfg[block] = {} - if not block: - continue - if len(re.findall("=", line)) == 1: - field, val = line.split("=") - field = field.strip() - variables = re.findall(r"[$]([A-Za-z0-9_]+)", val) - for v in variables: - for b in self.cfg: - if v in self.cfg[b]: - val = val.replace("$" + v, self.cfg[b][v]) - if "default_ca" in self.cfg.get("ca", {}): - if "policy" in self.cfg.get(self.cfg["ca"]["default_ca"], {}): - if block == self.cfg[self.cfg["ca"]["default_ca"]]["policy"]: - self.dnList.append(field) - self.cfg[block][field] = val.strip() - - self.bits = int(self.cfg["req"].get("default_bits") or self.bits) - self.algoritm = self.cfg[self.cfg["ca"]["default_ca"]].get("default_md") or self.algoritm - if not self.parameters.get("CertFile"): - self.parameters["CertFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["certificate"] - self.parameters["KeyFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["private_key"] - # Read distinguished names - for k, v in self.cfg[self.cfg[self.cfg["ca"]["default_ca"]]["policy"]].items(): - nid = self.fields2nid[k] - self.parameters[nid], self.minDict[nid], self.maxDict[nid] = [], [], [] - for k in [f"{i}.{k}" for i in range(0, 5)] + [k]: - if k + "_default" in self.cfg["req"]["distinguished_name"]: - self.parameters[nid].append(self.cfg["req"]["distinguished_name"][k + "_default"]) - if k + "_min" in self.cfg["req"]["distinguished_name"]: - self.minDict[nid].append(self.cfg["req"]["distinguished_name"][k + "_min"]) - if k + "_max" in self.cfg["req"]["distinguished_name"]: - self.maxDict[nid].append(self.cfg["req"]["distinguished_name"][k + "_max"]) - if v == "supplied": - self.supplied.append(nid) - elif v == "optional": - self.optional.append(nid) - elif v == "match": - self.match.append(nid) - - def __parseDN(self, dn): - """Return DN fields - - :param str dn: DN - - :return: list -- contain tuple with positionOfField.fieldName, fieldNID, fieldValue - """ - dnInfoDict = collections.OrderedDict() - for f, v in [f.split("=") for f in dn.lstrip("/").split("/")]: - if not v: - return S_ERROR(f'No value set for "{f}"') - if f not in dnInfoDict: - dnInfoDict[f] = [v] - else: - dnInfoDict[f].append(v) - return S_OK(dnInfoDict) - - def __fillX509Name(self, field, values): - """Fill x509_Name object by M2Crypto - - :param str field: DN field name - :param list values: values of field, order important - - :return: S_OK()/S_ERROR() - """ - for value in values: - if ( - value - and m2.x509_name_set_by_nid( # pylint: disable=no-member - self.__X509Name.x509_name, self.fields2nid[field], value.encode() - ) - == 0 - ): - if ( - not self.__X509Name.add_entry_by_txt( - field=field, type=ASN1.MBSTRING_ASC, entry=value, len=-1, loc=-1, set=0 + :param str userDN: user DN + + :return: S_OK()/S_ERROR() + """ + self.log.debug("Ckecking work status of", self.parameters["ProviderName"]) + result = self.__parseDN(userDN) + if not result["OK"]: + return result + dnInfoDict = result["Value"] + + try: + userOIDs = [self.fields2oid[f.split("=")[0]] for f in userDN.lstrip("/").split("/")] + except (ValueError, KeyError) as e: + return S_ERROR(f"Unknown DN field in used DN: {e}") + oidOrder = [self.fields2oid[f] for f in self.dnList] + for index, oid in enumerate(userOIDs): + if oid not in oidOrder: + return S_ERROR( + f'"{self.oid2defField.get(oid, min(self.oid2fields[oid], key=len))}" field not found in order.' ) - == 1 - ): - return S_ERROR(f'Cannot set "{field}" field.') - return S_OK() - - def __createCertM2Crypto(self): - """Create new certificate for user - - :return: S_OK(tuple)/S_ERROR() -- tuple contain certificate and pulic key as strings - """ - # Create public key - userPubKey = EVP.PKey() - userPubKey.assign_rsa(RSA.gen_key(self.bits, 65537, util.quiet_genparam_callback)) - # Create certificate - userCert = X509.X509() - userCert.set_pubkey(userPubKey) - userCert.set_version(2) - userCert.set_subject(self.__X509Name) - userCert.set_serial_number(secrets.randbits(64)) - # Add extentionals - userCert.add_ext(X509.new_extension("basicConstraints", "CA:" + str(False).upper())) - userCert.add_ext(X509.new_extension("extendedKeyUsage", "clientAuth", critical=1)) - # Set livetime - validityTime = datetime.timedelta(days=400) - notBefore = ASN1.ASN1_UTCTIME() - notBefore.set_time(int(time.time())) - notAfter = ASN1.ASN1_UTCTIME() - notAfter.set_time(int(time.time()) + int(validityTime.total_seconds())) - userCert.set_not_before(notBefore) - userCert.set_not_after(notAfter) - # Add subject from CA - with open(self.parameters["CertFile"]) as cf: - caCertStr = cf.read() - caCert = X509.load_cert_string(caCertStr) - userCert.set_issuer(caCert.get_subject()) - # Use CA key - with open(self.parameters["KeyFile"], "rb") as cf: - caKeyStr = cf.read() - pkey = EVP.PKey() - pkey.assign_rsa(RSA.load_key_string(caKeyStr, callback=util.no_passphrase_callback)) - # Sign - userCert.sign(pkey, self.algoritm) - - userCertStr = userCert.as_pem().decode("ascii") - userPubKeyStr = userPubKey.as_pem(cipher=None, callback=util.no_passphrase_callback).decode("ascii") - return S_OK((userCertStr, userPubKeyStr)) - - def _forceGenerateProxyForDN(self, dn, time, group=None): - """An additional helper method for creating a proxy without any substantial validation, - it can be used for a specific case(such as testing) where just need to generate a proxy - with specific DN on the fly. - - :param str dn: requested proxy DN - :param int time: expired time in a seconds - :param str group: if need to add DIRAC group - - :return: S_OK(tuple)/S_ERROR() -- contain proxy as chain and as string - """ - self.__X509Name = X509.X509_Name() - result = self.__parseDN(dn) - if not result["OK"]: + if index > oidOrder.index(oid): + return S_ERROR("Bad DNs order") + for i in range(oidOrder.index(oid) - 1): + try: + if userOIDs.index(oidOrder[i]) > index: + return S_ERROR("Bad DNs order") + except (ValueError, KeyError): + continue + for i in range(oidOrder.index(oid) + 1, len(oidOrder)): + try: + if userOIDs.index(oidOrder[i]) < index: + return S_ERROR("Bad DNs order") + except (ValueError, KeyError): + continue + + for oid in self.supplied: + if oid not in [self.fields2oid[f] for f in dnInfoDict]: + return S_ERROR( + 'Current DN is invalid, "%s" field must be set.' + % self.oid2defField.get(oid, min(self.oid2fields[oid], key=len)) + ) + + for field, values in dnInfoDict.items(): + oid = self.fields2oid[field] + err = f'Current DN is invalid, "{field}" field' + if oid not in self.supplied + self.match + self.optional: + return S_ERROR(f"{err} is not found for current CA.") + if oid in self.match and not self.dnInfoDictCA[field] == values: + return S_ERROR(f"{err} must be /{field}={('/%s=' % field).joing(self.dnInfoDictCA[field])}.") + if oid in self.maxDict: + rangeMax = list(range(min(len(values), len(self.maxDict[oid])))) + if any([True if len(values[i]) > self.maxDict[oid][i] else False for i in rangeMax]): + return S_ERROR(f"{err} values must be less then {', '.join(self.maxDict[oid])}.") + if oid in self.minDict: + rangeMin = list(range(min(len(values), len(self.minDict[oid])))) + if any([True if len(values[i]) < self.minDict[oid][i] else False for i in rangeMin]): + return S_ERROR(f"{err} values must be more then {', '.join(self.minDict[oid])}.") + + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + return S_OK() + + def getProxy(self, userDN): + """Generate user proxy + + :param str userDN: user DN + + :return: S_OK(str)/S_ERROR() -- contain a proxy string + """ + self.__nameAttributes = [] + result = self.checkStatus(userDN) + if result["OK"]: + result = self.__createCertificate() + if result["OK"]: + certStr, keyStr = result["Value"] + + chain = X509Chain() + result = chain.loadChainFromString(certStr) + if result["OK"]: + result = chain.loadKeyFromString(keyStr) + if result["OK"]: + result = chain.generateProxyToString(365 * 24 * 3600) + return result - dnInfoDict = result["Value"] - for field, values in dnInfoDict.items(): - result = self.__fillX509Name(field, values) + def generateDN(self, **kwargs): + """Get DN of the user certificate that will be created + + :param dict kwargs: user description dictionary with possible fields: + - FullName or CN + - Email or emailAddress + + :return: S_OK(str)/S_ERROR() -- contain DN + """ + if kwargs.get("FullName"): + kwargs["CN"] = [kwargs["FullName"]] + if kwargs.get("Email"): + kwargs["emailAddress"] = [kwargs["Email"]] + + self.__nameAttributes = [] + self.log.info("Creating distinguished names chain") + + for oid in self.supplied: + if oid not in [self.fields2oid[f] for f in self.dnList]: + return S_ERROR( + 'DNs order list does not contain supplied DN "%s"' + % self.oid2defField.get(oid, min(self.oid2fields[oid], key=len)) + ) + + for field in self.dnList: + values = [] + oid = self.fields2oid[field] + if oid in self.match: + for field in self.oid2fields[oid]: + if field in self.dnInfoDictCA: + values = self.dnInfoDictCA[field] + if not values: + return S_ERROR(f'Not found "{field}" match DN in CA') + for field in self.oid2fields[oid]: + if kwargs.get(field): + values = kwargs[field] if isinstance(kwargs[field], list) else [kwargs[field]] + if not values and oid in self.supplied: + # Search default value + if oid not in self.oid2defField: + return S_ERROR(f'No values set for "{min(self.oid2fields[oid], key=len)}" DN') + values = self.parameters[oid] + + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + # WARN: This logic not support list of distribtes name elements + resDN = asn1_utils.nameToDN(x509.Name(self.__nameAttributes)) + + result = self.checkStatus(resDN) if not result["OK"]: return result + return S_OK(resDN) + + def __parseCACFG(self): + """Parse CA configuration file""" + block = "" + self.cfg = {} + self.supplied, self.optional, self.match, self.dnList = [], [], [], [] + with open(self.parameters["CAConfigFile"]) as caCFG: + for line in caCFG: + # Ignore comments + line = re.sub(r"#.*", "", line) + if re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", "")): + block = "".join(re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", ""))) + if block not in self.cfg: + self.cfg[block] = {} + if not block: + continue + if len(re.findall("=", line)) == 1: + field, val = line.split("=") + field = field.strip() + variables = re.findall(r"[$]([A-Za-z0-9_]+)", val) + for v in variables: + for b in self.cfg: + if v in self.cfg[b]: + val = val.replace("$" + v, self.cfg[b][v]) + if "default_ca" in self.cfg.get("ca", {}): + if "policy" in self.cfg.get(self.cfg["ca"]["default_ca"], {}): + if block == self.cfg[self.cfg["ca"]["default_ca"]]["policy"]: + self.dnList.append(field) + self.cfg[block][field] = val.strip() + + self.bits = int(self.cfg["req"].get("default_bits") or self.bits) + self.algoritm = self.cfg[self.cfg["ca"]["default_ca"]].get("default_md") or self.algoritm + if not self.parameters.get("CertFile"): + self.parameters["CertFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["certificate"] + self.parameters["KeyFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["private_key"] + # Read distinguished names + for k, v in self.cfg[self.cfg[self.cfg["ca"]["default_ca"]]["policy"]].items(): + oid = self.fields2oid[k] + self.parameters[oid], self.minDict[oid], self.maxDict[oid] = [], [], [] + for k in [f"{i}.{k}" for i in range(0, 5)] + [k]: + if k + "_default" in self.cfg["req"]["distinguished_name"]: + self.parameters[oid].append(self.cfg["req"]["distinguished_name"][k + "_default"]) + if k + "_min" in self.cfg["req"]["distinguished_name"]: + self.minDict[oid].append(self.cfg["req"]["distinguished_name"][k + "_min"]) + if k + "_max" in self.cfg["req"]["distinguished_name"]: + self.maxDict[oid].append(self.cfg["req"]["distinguished_name"][k + "_max"]) + if v == "supplied": + self.supplied.append(oid) + elif v == "optional": + self.optional.append(oid) + elif v == "match": + self.match.append(oid) + + def __parseDN(self, dn): + """Return DN fields + + :param str dn: DN + + :return: list -- contain tuple with positionOfField.fieldName, fieldOID, fieldValue + """ + dnInfoDict = collections.OrderedDict() + for f, v in [f.split("=") for f in dn.lstrip("/").split("/")]: + if not v: + return S_ERROR(f'No value set for "{f}"') + if f not in dnInfoDict: + dnInfoDict[f] = [v] + else: + dnInfoDict[f].append(v) + return S_OK(dnInfoDict) + + def __fillX509Name(self, field, values): + """Collect the DN attributes for the certificate subject + + :param str field: DN field name + :param list values: values of field, order important + + :return: S_OK()/S_ERROR() + """ + for value in values: + if value: + try: + self.__nameAttributes.append( + x509.NameAttribute(x509.ObjectIdentifier(self.fields2oid[field]), value) + ) + except (KeyError, TypeError, ValueError) as e: + return S_ERROR(f'Cannot set "{field}" field: {e!r}.') + return S_OK() + + def __createCertificate(self): + """Create new certificate for user + + :return: S_OK(tuple)/S_ERROR() -- tuple contain certificate and private key as strings + """ + if self.algoritm not in _ALGORITHMS: + return S_ERROR(f'Unsupported signing algorithm "{self.algoritm}"') + hashAlgo = _ALGORITHMS[self.algoritm]() + + # Create user key pair + userKey = rsa.generate_private_key(public_exponent=65537, key_size=self.bits) + + # Read CA certificate and key + try: + with open(self.parameters["CertFile"], "rb") as cf: + caCert = x509.load_pem_x509_certificate(cf.read()) + with open(self.parameters["KeyFile"], "rb") as cf: + caKey = serialization.load_pem_private_key(cf.read(), password=None) + except Exception as e: + return S_ERROR(f"Cannot load CA credentials: {e!r}") + + serial = 0 + while not serial: + serial = secrets.randbits(64) + + now = datetime.datetime.now(datetime.timezone.utc) + builder = ( + x509.CertificateBuilder() + .subject_name(x509.Name(self.__nameAttributes)) + .issuer_name(caCert.subject) + .public_key(userKey.public_key()) + .serial_number(serial) + .not_valid_before(now) + .not_valid_after(now + datetime.timedelta(days=400)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=False) + .add_extension(x509.ExtendedKeyUsage([x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH]), critical=True) + ) + + try: + userCert = builder.sign(caKey, hashAlgo) + except Exception as e: + return S_ERROR(f"Cannot sign the user certificate: {e!r}") + + userCertStr = userCert.public_bytes(serialization.Encoding.PEM).decode("ascii") + userKeyStr = userKey.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode("ascii") + return S_OK((userCertStr, userKeyStr)) + + def _forceGenerateProxyForDN(self, dn, time, group=None): + """An additional helper method for creating a proxy without any substantial validation, + it can be used for a specific case(such as testing) where just need to generate a proxy + with specific DN on the fly. + + :param str dn: requested proxy DN + :param int time: expired time in a seconds + :param str group: if need to add DIRAC group + + :return: S_OK(tuple)/S_ERROR() -- contain proxy as chain and as string + """ + self.__nameAttributes = [] + result = self.__parseDN(dn) + if not result["OK"]: + return result + dnInfoDict = result["Value"] + + for field, values in dnInfoDict.items(): + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result - result = self.__createCertM2Crypto() - if result["OK"]: - certStr, keyStr = result["Value"] + result = self.__createCertificate() + if result["OK"]: + certStr, keyStr = result["Value"] + chain = X509Chain() + if chain.loadChainFromString(certStr)["OK"] and chain.loadKeyFromString(keyStr)["OK"]: + result = chain.generateProxyToString(time, diracGroup=group) + if not result["OK"]: + return result chain = X509Chain() - if chain.loadChainFromString(certStr)["OK"] and chain.loadKeyFromString(keyStr)["OK"]: - result = chain.generateProxyToString(time, diracGroup=group) - if not result["OK"]: - return result - chain = X509Chain() - chain.loadProxyFromString(result["Value"]) - return S_OK((chain, result["Value"])) + chain.loadProxyFromString(result["Value"]) + return S_OK((chain, result["Value"])) diff --git a/src/DIRAC/__init__.py b/src/DIRAC/__init__.py index a408647cc85..9282a0fc841 100755 --- a/src/DIRAC/__init__.py +++ b/src/DIRAC/__init__.py @@ -65,13 +65,6 @@ __path__ = extend_path(__path__, __name__) -# Set the environment variable such that openssl accepts proxy cert -# Sadly, this trick was removed in openssl >= 1.1.0 -# https://github.com/openssl/openssl/commit/8e21938ce3a5306df753eb40a20fe30d17cf4a68 -# Lets see if they would accept to put it back -# https://github.com/openssl/openssl/issues/8177 -os.environ["OPENSSL_ALLOW_PROXY_CERTS"] = "True" - # Now that's one hell of a hack :) # _strptime is not thread safe, resulting in obscure callstack # whenever you would have multiple threads and calling datetime.datetime.strptime diff --git a/tests/py3CheckDirs.txt b/tests/py3CheckDirs.txt index 1a3ad63d306..00ab41c4414 100644 --- a/tests/py3CheckDirs.txt +++ b/tests/py3CheckDirs.txt @@ -9,7 +9,7 @@ src/DIRAC/Core/Base/Client.py src/DIRAC/Core/DISET/private/BaseClient.py src/DIRAC/Core/DISET/private/Service.py src/DIRAC/Core/DISET/RequestHandler.py -src/DIRAC/Core/Security/m2crypto/X509Chain.py +src/DIRAC/Core/Security/pyca/X509Chain.py src/DIRAC/Core/Tornado src/DIRAC/DataManagementSystem/Service/TornadoFileCatalogHandler.py src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py