From 5e28d5e2e7f37ed48c894d0455bc7130f20d10a8 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 9 Jul 2026 06:33:22 +0200 Subject: [PATCH 1/7] feat: replace M2Crypto and pyasn1 with cryptography and stdlib ssl Reimplement the X509 layer (X509Certificate, X509Chain, X509Request, X509CRL) with pyca/cryptography, moving the modules to the top of DIRAC.Core.Security and dropping the m2crypto subpackage and its extend_path import hack. Decode the VOMS and diracGroup extensions with the declarative ASN.1 API of cryptography (>= 47) instead of pyasn1, which is 40-80x faster. Reimplement the DISET SSLTransport with ssl.SSLContext: RFC 3820 proxies are accepted via ssl.VERIFY_ALLOW_PROXY_CERTS and the peer credentials are extracted from SSLSocket.get_unverified_chain(), which requires python >= 3.13 on the server side (clients work on 3.11+). Run the Tornado HTTPS server with a standard SSLContext and read the peer chain directly as DER in BaseRequestHandler, removing the need for the patched tornado fork and tornado-m2crypto. Rewrite DIRACCAProxyProvider with cryptography, replacing the M2Crypto NID lookups with OID-based tables. The TLS floor is now 1.2 and the legacy default cipher list is no longer applied; ciphers can be pinned with DIRAC_SSL_CIPHERS (the old DIRAC_M2CRYPTO_SSL_CIPHERS is still honoured). --- .../environment_variable_configuration.rst | 10 +- .../DeveloperGuide/TornadoServices/index.rst | 7 +- environment.yml | 11 +- setup.cfg | 5 +- src/DIRAC/Core/DISET/__init__.py | 15 - .../private/Transports/M2SSLTransport.py | 505 ----------------- .../DISET/private/Transports/SSL/M2Utils.py | 219 -------- .../DISET/private/Transports/SSLTransport.py | 447 ++++++++++++++- .../Transports/test/Test_SSLTransport.py | 8 +- .../Core/Security/{m2crypto => }/X509CRL.py | 30 +- src/DIRAC/Core/Security/X509Certificate.py | 523 ++++++++++++++++++ .../Core/Security/{m2crypto => }/X509Chain.py | 221 ++++---- .../Security/{m2crypto => }/X509Request.py | 96 ++-- src/DIRAC/Core/Security/__init__.py | 70 ++- src/DIRAC/Core/Security/asn1_utils.py | 423 ++++++++++++++ .../Core/Security/m2crypto/X509Certificate.py | 500 ----------------- src/DIRAC/Core/Security/m2crypto/__init__.py | 47 -- .../Core/Security/m2crypto/asn1_utils.py | 368 ------------ .../Security/test/Test_X509Certificate.py | 6 +- .../Core/Security/test/Test_X509Chain.py | 4 +- .../Core/Security/test/x509TestUtilities.py | 16 +- .../Core/Tornado/Server/TornadoServer.py | 31 +- .../Server/private/BaseRequestHandler.py | 22 +- .../FrameworkSystem/Client/ProxyGeneration.py | 2 +- .../ProxyProvider/DIRACCAProxyProvider.py | 287 +++++----- src/DIRAC/__init__.py | 7 - tests/py3CheckDirs.txt | 2 +- 27 files changed, 1817 insertions(+), 2065 deletions(-) delete mode 100755 src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py delete mode 100644 src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py rename src/DIRAC/Core/Security/{m2crypto => }/X509CRL.py (71%) create mode 100644 src/DIRAC/Core/Security/X509Certificate.py rename src/DIRAC/Core/Security/{m2crypto => }/X509Chain.py (85%) rename src/DIRAC/Core/Security/{m2crypto => }/X509Request.py (60%) create mode 100644 src/DIRAC/Core/Security/asn1_utils.py delete mode 100644 src/DIRAC/Core/Security/m2crypto/X509Certificate.py delete mode 100644 src/DIRAC/Core/Security/m2crypto/__init__.py delete mode 100644 src/DIRAC/Core/Security/m2crypto/asn1_utils.py diff --git a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst index 9f35dc57bcb..2f20c6aecf4 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst @@ -42,14 +42,8 @@ DIRAC_HTTPS_SSL_METHOD_MAX DIRAC_HTTPS_SSL_METHOD_MIN If set, overrides the lowest supported TLS version when using HTTPS. It should be a valid value of :py:class:`ssl.TLSVersion`. -DIRAC_M2CRYPTO_SPLIT_HANDSHAKE - If ``true`` or ``yes`` the SSL handshake is done in a new thread (default Yes) - -DIRAC_M2CRYPTO_SSL_CIPHERS - If set, overwrites the default SSL ciphers accepted. It should be a colon separated list. See :py:mod:`DIRAC.Core.DISET` - -DIRAC_M2CRYPTO_SSL_METHODS - If set, overwrites the default SSL methods accepted. It should be a colon separated list. See :py:mod:`DIRAC.Core.DISET` +DIRAC_SSL_CIPHERS + If set, overwrites the default SSL ciphers accepted by the DISET protocol. It should be an OpenSSL style colon separated list. DIRAC_MYSQL_OPTIMIZER_TRACES_PATH If set, it should point to an existing directory, where MySQL Optimizer traces will be stored. See :py:func:`DIRAC.Core.Utilities.MySQL.captureOptimizerTraces` diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index 2246fcd5285..870c44a9356 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -313,10 +313,9 @@ How to install Tornado Requirements ************ -Two special python packages are needed: - -* git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable : in place of the standard tornado. This adds configurable feature to tornado -* git+https://github.com/DIRACGrid/tornado_m2crypto.git: this allows to use tornado with M2Crypto +The standard tornado package is used: the TLS layer (including grid proxy +support) relies on the ``ssl`` module of the python standard library +(python >= 3.13 is required on the server side). Install a service diff --git a/environment.yml b/environment.yml index b6c6a16716e..e512a024928 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - fts3 - gitpython >=2.1.0 - invoke - - m2crypto >=0.38.0 + - cryptography >=47.0.0 - matplotlib - numpy - paramiko @@ -30,8 +30,6 @@ dependencies: - pillow - prompt-toolkit >=3,<4 - psutil >=4.2.0 - - pyasn1 >0.4.1 - - pyasn1-modules - pydantic >=2 - python-json-logger >=0.1.8 - pytz >=2015.7 @@ -89,7 +87,7 @@ dependencies: - uritemplate # - readline >=6.2.4 in the standard library - simplejson >=3.8.1 - #- tornado >=5.0.0,<6.0.0 + - tornado >=5.1.1,<6.0.0 - typing >=3.6.6 - rucio-clients >=34.4.2 # For mypy @@ -105,11 +103,6 @@ dependencies: - Authlib >=1.0.0 - dominate - pyjwt - # This is a fork of tornado with a patch to allow for configurable iostream - - git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable - # This is an extension of Tornado to use M2Crypto - # It should eventually be part of DIRACGrid - - git+https://github.com/DIRACGrid/tornado_m2crypto - -e .[server] - -e ./dirac-common/ # Add diracdoctools diff --git a/setup.cfg b/setup.cfg index 6c9e1c1b2bd..a56e9a47bed 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ install_requires = cachetools certifi cwltool + cryptography >=47.0.0 diraccfg DIRACCommon diracx-client >=v0.0.1 @@ -41,14 +42,11 @@ install_requires = importlib_metadata >=4.4 importlib_resources invoke - M2Crypto >=0.36 packaging paramiko pexpect prompt-toolkit >=3 psutil - pyasn1 - pyasn1-modules pydantic >=2.4 pyparsing python-dateutil @@ -93,7 +91,6 @@ server = stomp.py suds tornado ~=5.1.1 - tornado-m2crypto importlib_resources testing = hypothesis diff --git a/src/DIRAC/Core/DISET/__init__.py b/src/DIRAC/Core/DISET/__init__.py index 4f7bc852cf8..6aa018fca56 100755 --- a/src/DIRAC/Core/DISET/__init__.py +++ b/src/DIRAC/Core/DISET/__init__.py @@ -4,18 +4,3 @@ DEFAULT_RPC_TIMEOUT = 600 #: Default timeout to establish a connection DEFAULT_CONNECTION_TIMEOUT = 10 - -#: Default SSL Ciher accepted. Current default is for pyGSI/M2crypto compatibility -#: Can be changed with DIRAC_M2CRYPTO_SSL_CIPHERS -#: Recommandation (incompatible with pyGSI) -# pylint: disable=line-too-long -#: AES256-GCM-SHA384:AES256-SHA256:AES128-GCM-SHA256:AES128-SHA256:HIGH:MEDIUM:RSA:!3DES:!RC4:!aNULL:!eNULL:!MD5:!SEED:!IDEA:!SHA # noqa -# Cipher line should be as readable as possible, sorry pylint -# pylint: disable=line-too-long -DEFAULT_SSL_CIPHERS = "AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:CAMELLIA256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:HIGH:MEDIUM:RSA:!3DES:!RC4:!aNULL:!eNULL:!MD5:!SEED:!IDEA" # noqa - -#: Default SSL methods accepted. Current default accepts TLSv1 for pyGSI/M2crypto compatibility -#: Can be changed with DIRAC_M2CRYPTO_SSL_METHODS -#: Recommandation (incompatible with pyGSI) -# TLSv2:TLSv3 -DEFAULT_SSL_METHODS = "TLSv1:TLSv2:TLSv3" diff --git a/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py deleted file mode 100755 index a7685e010d6..00000000000 --- a/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +++ /dev/null @@ -1,505 +0,0 @@ -#!/usr/bin/env python -""" -M2Crypto SSLTransport Library -""" -import os -import socket -from M2Crypto import SSL, threading as M2Threading -from M2Crypto.SSL.Checker import SSLVerificationError - -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR -from DIRAC.Core.DISET.private.Transports.BaseTransport import BaseTransport -from DIRAC.Core.DISET.private.Transports.SSL.M2Utils import getM2SSLContext, getM2PeerInfo - -from DIRAC.Core.DISET import DEFAULT_CONNECTION_TIMEOUT, DEFAULT_RPC_TIMEOUT - -# TODO: For now we have to set an environment variable for proxy support in OpenSSL -# Eventually we may need to add API support for this to M2Crypto... -os.environ["OPENSSL_ALLOW_PROXY_CERTS"] = "1" -M2Threading.init() - -# TODO: CRL checking should be implemented but this will require support adding -# to M2Crypto: Quite a few functions will need mapping through from OpenSSL to -# allow the CRL stack to be set on the X509 CTX used for verification. - -# TODO: Log useful messages to the logger - - -class SSLTransport(BaseTransport): - """SSL Transport implementation using the M2Crypto library.""" - - # This name is the same as BaseClient, - # and is used a bit everywhere, so it should be factorized out - # eventually - KW_TIMEOUT = "timeout" - - def __getConnection(self): - """Helper function to get a connection object, - Tries IPv6 (AF_INET6) first, then falls back to IPv4 (AF_INET). - """ - try: - conn = SSL.Connection(self.__ctx, family=socket.AF_INET6) - except OSError: - # Maybe no IPv6 support? Try IPv4 only socket. - conn = SSL.Connection(self.__ctx, family=socket.AF_INET) - return conn - - def __init__(self, *args, **kwargs): - """Create an SSLTransport object, parameters are the same - as for other transports. If ctx is specified (as an instance of - SSL.Context) then use that rather than creating a new context. - - kwargs can contain all the parameters defined in BaseClient, - in particular timeout - """ - # The thread init of M2Crypto is not really thread safe. - # So we put it a second time - M2Threading.init() - self.remoteAddress = None - self.peerCredentials = {} - - # The timeout used here is different from what it was in pyGSI. - # It is to be understood here as the timeout for socket operations - # involved in the RPC call, but NOT the establishment of the connection, - # for which there is a different timeout. - # - # The timeout management of pyGSI was a bit off. - # This is proven by that type of trace (look at the timestamp): - # - # 2020-07-16 09:48:55 UTC dirac-proxy-init [140013698656064] DEBUG: Connection timeout set to: 1 - # 2020-07-16 09:58:55 UTC dirac-proxy-init [140013698656064] WARN: Issue getting socket: - # - - self.__timeout = kwargs.get(SSLTransport.KW_TIMEOUT, DEFAULT_RPC_TIMEOUT) - - self.__locked = False # We don't support locking, so this is always false. - - # If not specified in the arguments (never is in DIRAC code...) - # and we are setting up a server listing connection, set the accepted - # ssl methods and ciphers - if kwargs.get("bServerMode"): - if "sslMethods" not in kwargs: - kwargs["sslMethods"] = os.environ.get("DIRAC_M2CRYPTO_SSL_METHODS") - if "sslCiphers" not in kwargs: - kwargs["sslCiphers"] = os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") - - self.__ctx = kwargs.pop("ctx", None) - if not self.__ctx: - self.__ctx = getM2SSLContext(**kwargs) - - # Note that kwargs is already kept in BaseTransport - # as self.extraArgsDict, but at least I am sure that - # self.__kwargs will never be modified - self.__kwargs = kwargs - - BaseTransport.__init__(self, *args, **kwargs) - - def setSocketTimeout(self, timeout): - """Set the timeout for RPC calls. - - .. warning: This needs to be called before initAsClient. - It is used as a timeout for RPC calls, not connection. - - :param timeout: timeout for socket operation in seconds - - """ - self.__timeout = timeout - - def initAsClient(self): - """Prepare this client socket for use.""" - if self.serverMode(): - raise RuntimeError("SSLTransport is in server mode.") - - errors = [] - host, port = self.stServerAddress - - # The following piece of code was inspired by the python socket documentation - # as well as the implementation of M2Crypto.httpslib.HTTPSConnection - - # Get all available addresses (IPv6 and IPv4) and try them in order - try: - addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) - except OSError as e: - return S_ERROR(f"DNS lookup failed {e!r}") - for family, _socketType, _proto, _canonname, socketAddress in addrInfoList: - try: - self.oSocket = SSL.Connection(self.__ctx, family=family) - - # First set a short connection timeout, that will trigger - # during blocking operations (read/write) use to - # establish the SSL connection - self.oSocket.settimeout(DEFAULT_CONNECTION_TIMEOUT) - - # Enable keepAlive, with default options - # (see more comments about keepalive in :py:meth:`.acceptConnection`) - self.oSocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - - # set SNI server name since we know it at this point - self.oSocket.set_tlsext_host_name(host) - - # tell the connection which host we are connecting to so we can - # use the address we obtained from DNS - self.oSocket.set1_host(host) - self.oSocket.connect(socketAddress) - - # Once the connection is established, we can use the timeout - # asked for RPC - self.oSocket.settimeout(self.__timeout) - - self.remoteAddress = self.oSocket.getpeername() - - return S_OK() - # warning: do NOT catch SSL related error here - # They should be propagated upwards and caught by the BaseClient - # not to enter the retry loop - except OSError as e: - errors.append(f"{socketAddress} {e}:{repr(e)}") - - if self.oSocket is not None: - self.close() - - return S_ERROR("; ".join(errors)) - - def initAsServer(self): - """Prepare this server socket for use.""" - if not self.serverMode(): - raise RuntimeError("SSLTransport is in client mode.") - # Before getting the connection object, we need to set - # a server session ID in the context - host = self.stServerAddress[0] - port = self.stServerAddress[1] - self.__ctx.set_session_id_ctx((f"DIRAC-{host}-{port}").encode()) - self.oSocket = self.__getConnection() - # Make sure reuse address is set correctly - if self.bAllowReuseAddress: - param = 1 - else: - param = 0 - - self.oSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, param) - - self.oSocket.bind(self.stServerAddress) - self.oSocket.listen(self.iListenQueueSize) - return S_OK() - - def close(self): - # pylint: disable=line-too-long - """Close this socket.""" - - if self.oSocket: - # TL;DR: - # Do NOT touch that method - # - # Surprisingly (to me at least), M2Crypto does not close - # the underlying socket when calling SSL.Connection.close - # It only does it when the garbage collector kicks in (see ~M2Crypto.SSL.Connection.Connection.__del__) - # If the socket is not closed, the connection may hang forever. - # - # Thus, we are setting self.oSocket to None to allow the GC to do the work, but since we are not sure - # that it will run, we anyway force the connection to be closed - # - # However, we should close the underlying socket only after SSL was shutdown properly. - # This is because OpenSSL `ssl3_shutdown` (see callstack below) may still read some data - # (see https://github.com/openssl/openssl/blob/master/ssl/s3_lib.c#L4509):: - # - # - # 1 0x00007fffe9d48fc0 in sock_read () from /lib/libcrypto.so.1.0.0 - # 2 0x00007fffe9d46e83 in BIO_read () from /lib/libcrypto.so.1.0.0 - # 3 0x00007fffe9eab9dd in ssl3_read_n () from /lib/libssl.so.1.0.0 - # 4 0x00007fffe9ead216 in ssl3_read_bytes () from /lib/libssl.so.1.0.0 - # 5 0x00007fffe9ea999c in ssl3_shutdown () from /lib/libssl.so.1.0.0 - # 6 0x00007fffe9ed4f93 in ssl_free () from /lib/libssl.so.1.0.0 - # 7 0x00007fffe9d46d5b in BIO_free () from /lib/libcrypto.so.1.0.0 - # 8 0x00007fffe9f30a96 in bio_free (bio=0x5555556f3200) at SWIG/_m2crypto_wrap.c:5008 - # 9 0x00007fffe9f30b1e in _wrap_bio_free (self=, args=) at SWIG/_m2crypto_wrap.c - # - # We unfortunately have no way to force that order, and there is a risk of deadlock - # when running in a multi threaded environment like the agents:: - # - # Thread A opens socket, gets FD = 111 - # Thread A works on it - # Thread A closes FD 111 (underlying socket.close()) - # Thread B opens socket, gets FD = 111 - # Thread A calls read on FD=111 from ssl3_shutdown - # - # This is illustrated on the strace below:: - # - # 26461 14:25:15.266692 write(111]:42688->[]:9140]>, - # "blabla", 37 - # 26464 14:25:15.266857 <... connect resumed>) = 0 <0.000195> - # 26464 14:25:15.267023 getsockname(120:44252->188.185.84.86:9140]>, - # 26461 14:25:15.267176 <... write resumed>) = 37 <0.000453> - # 26464 14:25:15.267425 <... getsockname resumed>{sa_family=AF_INET, sin_port=htons(44252), - # sin_addr=inet_addr("")}, [28->16]) = 0 <0.000292> - # 26461 14:25:15.267466 close(111]:42688->[]:9140]> - # 26464 14:25:15.267637 close(120:44252->188.185.84.86:9140]> - # 26464 14:25:15.267738 <... close resumed>) = 0 <0.000086> - # 26461 14:25:15.267768 <... close resumed>) = 0 <0.000285> - # 26464 14:25:15.267827 socket(AF_INET6, SOCK_DGRAM|SOCK_CLOEXEC, IPPROTO_IP - # 26461 14:25:15.267888 futex(0x21f8620, FUTEX_WAKE_PRIVATE, 1 - # 26464 14:25:15.267976 <... socket resumed>) = 111 <0.000138> - # 26461 14:25:15.268092 <... futex resumed>) = 1 <0.000196> - # 26464 14:25:15.268195 connect(111, - # {sa_family=AF_INET6, sin6_port=htons(9140), - # inet_pton(AF_INET6, "", &sin6_addr), - # sin6_flowinfo=htonl(0), sin6_scope_id=0 - # }, 28 - # 26461 14:25:15.268294 read(111]:42480->[]:9140]>, - # 26464 14:25:15.268503 <... connect resumed>) = 0 <0.000217> - # 26464 14:25:15.268673 getsockname(111]:42480->[]:9140]>, - # 26464 14:25:15.268862 <... getsockname resumed>{sa_family=AF_INET6, sin6_port=htons(42480), - # inet_pton(AF_INET6, "", &sin6_addr), sin6_flowinfo=htonl(0), sin6_scope_id= - # 0}, [28]) = 0 <0.000168> - # 26464 14:25:15.269048 - # close(111]:42480->[]:9140]> - # - # - # - # Update 16.07.20: - # M2Crypto 0.36 contains the bug fix https://gitlab.com/m2crypto/m2crypto/-/merge_requests/247 - # that allows proper closing. So manual closing of the underlying socket should not be needed anymore - - # Update 16.07.20 - # I add this shutdown call without being 100% sure - # it solves some hanging connections issues, but it seems - # to work. it does not appear in any M2Crypto doc, but comparing - # some internals of M2Crypto and official python SSL library, - # it seems to make sense - self.oSocket.shutdown(socket.SHUT_RDWR) - - # Update 16.07.20 - # With freeBio=True, we force the - # closing of the socket before the GC runs - self.oSocket.close(freeBio=True) - # underlyingSocket = self.oSocket.socket - self.oSocket = None - # underlyingSocket.close() - return S_OK() - - def renewServerContext(self): - # pylint: disable=line-too-long - """Renews the server context. - This reloads the certificates and re-initialises the SSL context. - - NOTE: Chris 15.05.20 - I noticed python segfault on a regular time interval. The stack trace always looks like that:: - - #0 0x00007fdb5bbe2388 in ?? () from /opt/dirac/pro/diracos/usr/lib64/python2.7/lib-dynload/../../libcrypto.so.10 - #1 0x00007fdb5bbd8742 in X509_STORE_load_locations () from /opt/dirac/pro/diracos/usr/lib64/python2.7/lib-dynload/../../libcrypto.so.10 - #2 0x00007fdb57edcc9d in _wrap_ssl_ctx_load_verify_locations (self=, args=) at SWIG/_m2crypto_wrap.c:20602 - #3 0x00007fdb644ec484 in PyEval_EvalFrameEx () from /opt/dirac/versions/v10r0_1587978031/diracos/usr/bin/../lib64/libpython2.7.so.1.0 - - I could not find anything fundamentaly wrong, and the context renewal is the only place I could think of. - - GSI based SSLTransport did the following: renew the context, and renew the Connection object using the same raw socket - This still seems very fishy to me though, especially that the ServiceReactor still has the old object in self.__listeningConnections[svcName]['socket']] - - Here, we were are refreshing the CA store. What was missing was the call to the parent class, thus entering some sort of infinite loop. - The parent's call seems to have fixed it. - """ # noqa # pylint: disable=line-too-long - if not self.serverMode(): - raise RuntimeError("SSLTransport is in client mode.") - super().renewServerContext() - self.__ctx = getM2SSLContext(self.__ctx, **self.__kwargs) - - return S_OK() - - def handshake_singleStep(self): - """Used to perform SSL handshakes. - These are now done automatically. - """ - # This isn't used any more, the handshake is done inside the M2Crypto library - return S_OK() - - def handshake_multipleSteps(self): - """Perform SSL handshakes. - This has to be called after the connection was accepted (acceptConnection_multipleSteps) - - The remote credentials are gathered here - """ - try: - # M2Crypto does not provide public method to - # accept and handshake in two steps. - # So we have to do it manually - # The following lines are basically a copy/paste - # of the end of SSL.Connection.accept method - self.oSocket.setup_ssl() - self.oSocket.set_accept_state() - self.oSocket.accept_ssl() - check = getattr(self.oSocket, "postConnectionCheck", self.oSocket.serverPostConnectionCheck) - if check is not None: - if not check(self.oSocket.get_peer_cert(), self.oSocket.addr[0]): - raise SSL.Checker.SSLVerificationError("post connection check failed") - - self.peerCredentials = getM2PeerInfo(self.oSocket) - - # Now that the handshake has been performed on the server - # we can set the timeout for the RPC operations. - # In practice, since we are on the server side, the - # timeout we set here represents the timeout for receiving - # the arguments and sending back the response. This should - # in principle be reasonably quick, but just to be sure - # we can set it to the DEFAULT_RPC_TIMEOUT - self.oSocket.settimeout(DEFAULT_RPC_TIMEOUT) - - return S_OK() - except (OSError, SSL.SSLError, SSLVerificationError) as e: - return S_ERROR(f"Error in handhsake: {e} {repr(e)}") - - def setClientSocket_singleStep(self, oSocket): - """Set the inner socket (i.e. SSL.Connection object) of this instance - to the value of oSocket. - We also gather the remote peer credentials - This method is intended to be used to create client connection objects - from a server and should be considered to be an internal function. - - :param oSocket: client socket SSL.Connection object - - """ - - # TODO: The calling method (ServiceReactor.__acceptIncomingConnection) expects - # socket.error to be thrown in case of issue. Maybe we should catch the M2Crypto - # errors here and raise socket.error instead - - self.oSocket = oSocket - self.remoteAddress = self.oSocket.getpeername() - self.peerCredentials = getM2PeerInfo(self.oSocket) - - def setClientSocket_multipleSteps(self, oSocket): - """Set the inner socket (i.e. SSL.Connection object) of this instance - to the value of oSocket. - This method is intended to be used to create client connection objects - from a server and should be considered to be an internal function. - - :param oSocket: client socket SSL.Connection object - - """ - # warning: do NOT catch socket.error here, because for who knows what reason - # exceptions are actually properly used for once, and the calling method - # relies on it (ServiceReactor.__acceptIncomingConnection) - self.oSocket = oSocket - self.remoteAddress = self.oSocket.getpeername() - - def acceptConnection_multipleSteps(self): - """Accept a new client, returns a new SSLTransport object representing - the client connection. - - The connection is accepted, but no SSL handshake is performed - - :returns: S_OK(SSLTransport object) - """ - # M2Crypto does not provide public method to - # accept and handshake in two steps. - # So we have to do it manually - # The following lines are basically a copy/paste - # of the begining of SSL.Connection.accept method - # with added options and timeout - try: - sock, addr = self.oSocket.socket.accept() - oClient = SSL.Connection(self.oSocket.ctx, sock) - - # Set the keep alive to true. This keepalive will ensure that we - # detect remote peer crashing or network interruption - # Note that this is ineffective if we are in the middle of blocking - # operations. - oClient.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - - # I am adding here for reference the code that would allow to change - # the keepalive settings, although we should be fine with the default - # (connection would be closed after 7200 + 9 * 75 ~= 2h and 10mn ) - - # Duration between two keepalive probes, in seconds - # oClient.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 75) - # Number of consecutive bad probes to cut the connection - # oClient.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 9) - # Delay before the keepalive starts ticking, in seconds - # oClient.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 7200) - - # Here we set the timeout server side. - # We first set the connection timeout, which will - # be effective for TLS handshake - oClient.settimeout(DEFAULT_CONNECTION_TIMEOUT) - - oClient.addr = addr - oClientTrans = SSLTransport(self.stServerAddress, ctx=self.__ctx) - oClientTrans.setClientSocket(oClient) - return S_OK(oClientTrans) - except (OSError, SSL.SSLError, SSLVerificationError) as e: - return S_ERROR(f"Error in acceptConnection: {e} {repr(e)}") - - def acceptConnection_singleStep(self): - """Accept a new client, returns a new SSLTransport object representing - the client connection. - - The SSL handshake is performed here. - - :returns: S_OK(SSLTransport object) - """ - try: - oClient, _ = self.oSocket.accept() - oClientTrans = SSLTransport(self.stServerAddress, ctx=self.__ctx) - oClientTrans.setClientSocket(oClient) - return S_OK(oClientTrans) - except (OSError, SSL.SSLError, SSLVerificationError) as e: - return S_ERROR(f"Error in acceptConnection: {e} {repr(e)}") - - # Depending on the DIRAC_M2CRYPTO_SPLIT_HANDSHAKE we either do the - # handshake separately or not - if os.getenv("DIRAC_M2CRYPTO_SPLIT_HANDSHAKE", "Yes").lower() in ("yes", "true"): - acceptConnection = acceptConnection_multipleSteps - handshake = handshake_multipleSteps - setClientSocket = setClientSocket_multipleSteps - else: - acceptConnection = acceptConnection_singleStep - handshake = handshake_singleStep - setClientSocket = setClientSocket_singleStep - - def _read(self, bufSize=4096, skipReadyCheck=False): - """Read bufSize bytes from the buffer. - - :param bufSize: size of the buffer to read - :param skipReadyCheck: ignored. - - - :returns: S_OK(number of byte read) - """ - try: - read = self.oSocket.read(bufSize) - return S_OK(read) - except (OSError, SSL.SSLError, SSLVerificationError) as e: - return S_ERROR(f"Error in _read: {e} {repr(e)}") - - def isLocked(self): - """Returns if this instance is locked. - Always returns false. - - :returns: False - """ - return self.__locked - - def _write(self, buf): - """Write all bytes contained within iterable "buf" to the - connected peer. - - :param buf: iterable buffer - - :returns: S_OK(number of bytes written) - """ - try: - # If the client application has abruptly terminated - # the connection will be in CLOSE_WAIT on the server side. - # And when the server side will anyway try to send back its data. - # The first call to this _write method will succeed, - # and we will never know that the connection was broken. - # However, the client would answer with an RST packet. - # And writting on a socket that received an RST packet - # triggers a SIGPIPE. - # In practice, this means that if the server replies to a - # dead client with less that 16384 bytes - # (see https://datatracker.ietf.org/doc/html/rfc8446#section-5.1), - # we will never notice that we sent the answer to the vacuum. - # And don't look for a fix, there just isn't. - wrote = self.oSocket.write(buf) - return S_OK(wrote) - except (OSError, SSL.SSLError, SSLVerificationError) as e: - return S_ERROR(f"Error in _write: {e} {repr(e)}") diff --git a/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py b/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py deleted file mode 100644 index 79d4b1da003..00000000000 --- a/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python -""" -Utilities for using M2Crypto SSL with DIRAC. -""" -import os -import tempfile -import M2Crypto -from packaging.version import Version -from M2Crypto import SSL, m2, X509 - - -from DIRAC.Core.DISET import DEFAULT_SSL_CIPHERS, DEFAULT_SSL_METHODS -from DIRAC.Core.Security import Locations -from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain - -# Verify depth of peer certs -VERIFY_DEPTH = 50 -DEBUG_M2CRYPTO = os.getenv("DIRAC_DEBUG_M2CRYPTO", "No").lower() in ("yes", "true") - - -VERIFY_ALLOW_PROXY_CERTS = 0 - -# If the version of M2Crypto is recent enough, there is an API -# to accept proxy certificate, and we do not need to rely on -# OPENSSL_ALLOW_PROXY_CERT environment variable -# which was removed as of openssl 1.1 -# We need this to be merged in M2Crypto: https://gitlab.com/m2crypto/m2crypto/merge_requests/236 -# We set the proper verify flag to the X509Store of the context -# as described here https://www.openssl.org/docs/man1.1.1/man7/proxy-certificates.html -if hasattr(SSL, "verify_allow_proxy_certs"): - VERIFY_ALLOW_PROXY_CERTS = SSL.verify_allow_proxy_certs # pylint: disable=no-member -# As of M2Crypto 0.37, the `verify_allow_proxy_certs` flag was moved -# to X509 (https://gitlab.com/m2crypto/m2crypto/-/merge_requests/238) -# It is more consistent with all the other flags, -# but pySSL had it in SSL. Well... -elif hasattr(X509, "verify_allow_proxy_certs"): - VERIFY_ALLOW_PROXY_CERTS = X509.verify_allow_proxy_certs # pylint: disable=no-member -# As of M2Crypto 0.38, M2Crypto did not export the symbol correctly -# Anymore -# https://gitlab.com/m2crypto/m2crypto/-/issues/298 -elif Version(M2Crypto.__version__) >= Version("0.38.0"): - VERIFY_ALLOW_PROXY_CERTS = 64 - - -def __loadM2SSLCTXHostcert(ctx): - """Load hostcert & key from the default location and set them as the - credentials for SSL context ctx. - Returns None. - """ - certKeyTuple = Locations.getHostCertificateAndKeyLocation() - if not certKeyTuple: - raise RuntimeError("Hostcert/key location not set") - hostcert, hostkey = certKeyTuple - if not os.path.isfile(hostcert): - raise RuntimeError(f"Hostcert file ({hostcert}) is missing") - if not os.path.isfile(hostkey): - raise RuntimeError(f"Hostkey file ({hostkey}) is missing") - # Make sure we never stall on a password prompt if the hostkey has a password - # by specifying a blank string. - ctx.load_cert(hostcert, hostkey, callback=lambda: "") - - -def __loadM2SSLCTXProxy(ctx, proxyPath=None): - """Load proxy from proxyPath (or default location if not specified) and - set it as the certificate & key to use for this SSL context. - Returns None. - """ - if not proxyPath: - proxyPath = Locations.getProxyLocation() - if not proxyPath: - raise RuntimeError("Proxy location not set") - if not os.path.isfile(proxyPath): - raise RuntimeError(f"Proxy file ({proxyPath}) is missing") - # See __loadM2SSLCTXHostcert for description of why lambda is needed. - ctx.load_cert_chain(proxyPath, proxyPath, callback=lambda: "") - - -def ssl_verify_callback_print_error(ok, store): - """This callback method does nothing but printing the error. - It prints a few more useful info than the exception - - :param ok: current validation status - :param store: pointer to the X509_CONTEXT_STORE - """ - errnum = store.get_error() - if errnum: - print(f"SSL DEBUG ERRNUM {errnum} ERRMSG {m2.x509_get_verify_error(errnum)}") # pylint: disable=no-member - return ok - - -def getM2SSLContext(ctx=None, **kwargs): - """Gets an M2Crypto.SSL.Context configured using the standard - DIRAC connection keywords from kwargs. The keywords are: - - - clientMode: Boolean, if False hostcerts are always used. If True - a proxy is used unless other flags are set. - - useCertificates: Boolean, Set to true to use hostcerts in client - mode. - - proxyString: String, allow a literal proxy string to be provided. - - proxyLocation: String, Path to file to use as proxy, defaults to - usual location(s) if not set. - - skipCACheck: Boolean, if True, don't verify peer certificates. - - sslMethods: String, List of SSL algorithms to enable in OpenSSL style - cipher format, e.g. "SSLv3:TLSv1". - - sslCiphers: String, OpenSSL style cipher string of ciphers to allow - on this connection. - - If an existing context "ctx" is provided, it is just reconfigured with - the selected arguments. - - Returns the new or updated context. - """ - if not ctx: - ctx = SSL.Context() - # Set certificates for connection - # CHRIS: I think clientMode was just an internal of pyGSI implementation - # if kwargs.get('clientMode', False) and not kwargs.get('useCertificates', False): - # if not kwargs.get('useCertificates', False): - if kwargs.get("bServerMode", False) or ( - kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False) - ): - # Server mode always uses hostcert - __loadM2SSLCTXHostcert(ctx) - - else: - # Client mode has a choice of possible options - if kwargs.get("proxyString", None): - # M2Crypto cannot take an inmemory location or a string, so - # so write it to a temp file and use proxyLocation - with tempfile.NamedTemporaryFile(mode="w") as tmpFile: - tmpFilePath = tmpFile.name - tmpFile.write(kwargs["proxyString"]) - # Flush, otherwise the file is empty in the subsequent call - tmpFile.flush() - __loadM2SSLCTXProxy(ctx, proxyPath=tmpFilePath) - else: - # Use normal proxy - __loadM2SSLCTXProxy(ctx, proxyPath=kwargs.get("proxyLocation", None)) - - verify_callback = ssl_verify_callback_print_error if DEBUG_M2CRYPTO else None - - # Set peer verification - if kwargs.get("skipCACheck", False): - # Don't validate peer, but still request creds - ctx.set_verify(SSL.verify_none, VERIFY_DEPTH, callback=verify_callback) - else: - # Do validate peer - ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, VERIFY_DEPTH, callback=verify_callback) - # Set CA location - caPath = Locations.getCAsLocation() - if not caPath: - raise RuntimeError("Failed to find CA location") - if not os.path.isdir(caPath): - raise RuntimeError(f"CA path ({caPath}) is not a valid directory") - ctx.load_verify_locations(capath=caPath) - - # Allow proxy certificates to be used - if VERIFY_ALLOW_PROXY_CERTS: - ctx.get_cert_store().set_flags(VERIFY_ALLOW_PROXY_CERTS) - - # Other parameters - sslMethods = kwargs.get("sslMethods", DEFAULT_SSL_METHODS) - if sslMethods: - # Pylint can't see the m2 constants due to the way the library is loaded - # We just have to disable that warning for the next bit... - # pylint: disable=no-member - methods = [("SSLv2", m2.SSL_OP_NO_SSLv2), ("SSLv3", m2.SSL_OP_NO_SSLv3), ("TLSv1", m2.SSL_OP_NO_TLSv1)] - allowed_methods = sslMethods.split(":") - # If a method isn't explicitly allowed, set the flag to disable it... - for method, method_flag in methods: - if method not in allowed_methods: - ctx.set_options(method_flag) - # SSL_OP_NO_SSLv2, SSL_OP_NO_SSLv3, SSL_OP_NO_TLSv1 - ciphers = kwargs.get("sslCiphers", DEFAULT_SSL_CIPHERS) - ctx.set_cipher_list(ciphers) - - # log the debug messages - if DEBUG_M2CRYPTO: - ctx.set_info_callback() - - return ctx - - -def getM2PeerInfo(conn): - """Gets the details of the current peer as a standard dict. The peer - details are obtained from the supplied M2 SSL Connection obj "conn". - The details returned are those from ~X509Chain.getCredentials, without Registry info: - - DN - Full peer DN as string - x509Chain - Full chain of peer - isProxy - Boolean, True if chain ends with proxy - isLimitedProxy - Boolean, True if chain ends with limited proxy - group - String, DIRAC group for this peer, if known - - Returns a dict of details. - """ - chain = X509Chain.generateX509ChainFromSSLConnection(conn) - creds = chain.getCredentials(withRegistryInfo=False) - if not creds["OK"]: - raise RuntimeError(f"Failed to get SSL peer info ({creds['Message']}).") - peer = creds["Value"] - - peer["x509Chain"] = chain - isProxy = chain.isProxy() - if not isProxy["OK"]: - raise RuntimeError(f"Failed to get SSL peer isProxy ({isProxy['Message']}).") - peer["isProxy"] = isProxy["Value"] - - if peer["isProxy"]: - peer["DN"] = creds["Value"]["identity"] - else: - peer["DN"] = creds["Value"]["subject"] - - isLimited = chain.isLimitedProxy() - if not isLimited["OK"]: - raise RuntimeError(f"Failed to get SSL peer isProxy ({isLimited['Message']}).") - peer["isLimitedProxy"] = isLimited["Value"] - - return peer diff --git a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py index f5b42f03393..1626abff7d6 100755 --- a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py @@ -1,15 +1,452 @@ +"""SSL Transport implementation using the standard library ssl module. + +The peer chain validation (including RFC 3820 proxy support) is delegated to +OpenSSL through ``ssl.SSLContext``: + +* proxy certificates are accepted by setting the ``X509_V_FLAG_ALLOW_PROXY_CERTS`` + verify flag (``ssl.VERIFY_ALLOW_PROXY_CERTS``) +* the peer chain is retrieved after the handshake with + ``ssl.SSLSocket.get_unverified_chain`` (python >= 3.13), and the DIRAC + credentials (DN, group, ...) are extracted from it with + :py:class:`DIRAC.Core.Security.X509Chain.X509Chain` +""" import os +import socket +import ssl +import tempfile from DIRAC import gLogger +from DIRAC.Core.DISET import DEFAULT_CONNECTION_TIMEOUT, DEFAULT_RPC_TIMEOUT +from DIRAC.Core.DISET.private.Transports.BaseTransport import BaseTransport from DIRAC.Core.Security import Locations +from DIRAC.Core.Security.X509Certificate import X509Certificate +from DIRAC.Core.Security.X509Chain import X509Chain from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK -from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error -from DIRAC.Core.Security.X509Certificate import X509Certificate # pylint: disable=import-error +# Verify depth of peer certs +VERIFY_DEPTH = 50 + +# Re-exported for convenience (exists since python 3.10) +VERIFY_ALLOW_PROXY_CERTS = ssl.VERIFY_ALLOW_PROXY_CERTS + + +def __loadHostCertificate(ctx): + """Load hostcert & key from the default location and set them as the + credentials for SSL context ctx. + Returns None. + """ + certKeyTuple = Locations.getHostCertificateAndKeyLocation() + if not certKeyTuple: + raise RuntimeError("Hostcert/key location not set") + hostcert, hostkey = certKeyTuple + if not os.path.isfile(hostcert): + raise RuntimeError(f"Hostcert file ({hostcert}) is missing") + if not os.path.isfile(hostkey): + raise RuntimeError(f"Hostkey file ({hostkey}) is missing") + ctx.load_cert_chain(hostcert, keyfile=hostkey) + + +def __loadProxy(ctx, proxyPath=None): + """Load proxy from proxyPath (or default location if not specified) and + set it as the certificate & key to use for this SSL context. + Returns None. + """ + if not proxyPath: + proxyPath = Locations.getProxyLocation() + if not proxyPath: + raise RuntimeError("Proxy location not set") + if not os.path.isfile(proxyPath): + raise RuntimeError(f"Proxy file ({proxyPath}) is missing") + # A proxy file contains the leaf certificate, its key, and the rest + # of the chain in a single file, which load_cert_chain handles natively + ctx.load_cert_chain(proxyPath) + + +def getSSLContext(**kwargs): + """Gets an ssl.SSLContext configured using the standard + DIRAC connection keywords from kwargs. The keywords are: + + - bServerMode: Boolean, if True the context is setup for a server + (hostcert is always used). + - useCertificates: Boolean, Set to true to use hostcerts in client + mode. + - proxyString: String, allow a literal proxy string to be provided. + - proxyLocation: String, Path to file to use as proxy, defaults to + usual location(s) if not set. + - skipCACheck: Boolean, if True, don't verify peer certificates. + - optionalClientCert: Boolean, in server mode, request but do not require + the client certificate (used by the HTTPS services, + where token/visitor authentication also exists). + - sslCiphers: String, OpenSSL style cipher string of ciphers to allow + on this connection. + + Returns the new context. + """ + serverMode = kwargs.get("bServerMode", False) + + if serverMode: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + else: + # PROTOCOL_TLS_CLIENT enables check_hostname and CERT_REQUIRED by default + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + + # Set certificates for connection + if serverMode or (kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False)): + # Server mode always uses hostcert + __loadHostCertificate(ctx) + + else: + # Client mode has a choice of possible options + if kwargs.get("proxyString", None): + # ssl cannot take an in-memory location or a string, + # so write it to a temp file and use proxyLocation + with tempfile.NamedTemporaryFile(mode="w") as tmpFile: + tmpFile.write(kwargs["proxyString"]) + # Flush, otherwise the file is empty in the subsequent call + tmpFile.flush() + __loadProxy(ctx, proxyPath=tmpFile.name) + else: + # Use normal proxy + __loadProxy(ctx, proxyPath=kwargs.get("proxyLocation", None)) + + # Set peer verification + if kwargs.get("skipCACheck", False): + # Don't validate peer + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + else: + # Do validate peer + if serverMode and kwargs.get("optionalClientCert", False): + ctx.verify_mode = ssl.CERT_OPTIONAL + else: + ctx.verify_mode = ssl.CERT_REQUIRED + # Allow proxy certificates to be used + ctx.verify_flags |= VERIFY_ALLOW_PROXY_CERTS + # Set CA location + caPath = Locations.getCAsLocation() + if not caPath: + raise RuntimeError("Failed to find CA location") + if not os.path.isdir(caPath): + raise RuntimeError(f"CA path ({caPath}) is not a valid directory") + ctx.load_verify_locations(capath=caPath) + + # DIRAC_M2CRYPTO_SSL_CIPHERS is accepted for backward compatibility + ciphers = ( + kwargs.get("sslCiphers") or os.environ.get("DIRAC_SSL_CIPHERS") or os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") + ) + if ciphers: + ctx.set_ciphers(ciphers) + + return ctx + + +def getPeerInfo(sslSocket): + """Gets the details of the current peer as a standard dict. The peer + details are obtained from the supplied ssl.SSLSocket. + The details returned are those from ~X509Chain.getCredentials, without Registry info: + + DN - Full peer DN as string + x509Chain - Full chain of peer + isProxy - Boolean, True if chain ends with proxy + isLimitedProxy - Boolean, True if chain ends with limited proxy + group - String, DIRAC group for this peer, if known + + Returns a dict of details. + """ + if not hasattr(sslSocket, "get_unverified_chain"): + raise RuntimeError("Extracting the peer certificate chain requires python >= 3.13 (ssl get_unverified_chain)") + # The chain, as sent by the peer (leaf first). It was already validated + # by OpenSSL during the handshake, and the proxy-ness of it is + # anyway cryptographically re-checked by X509Chain + derChain = sslSocket.get_unverified_chain() + if not derChain: + return {} + chain = X509Chain.generateX509ChainFromDERList(derChain) + creds = chain.getCredentials(withRegistryInfo=False) + if not creds["OK"]: + raise RuntimeError(f"Failed to get SSL peer info ({creds['Message']}).") + peer = creds["Value"] + + peer["x509Chain"] = chain + isProxy = chain.isProxy() + if not isProxy["OK"]: + raise RuntimeError(f"Failed to get SSL peer isProxy ({isProxy['Message']}).") + peer["isProxy"] = isProxy["Value"] + + if peer["isProxy"]: + peer["DN"] = creds["Value"]["identity"] + else: + peer["DN"] = creds["Value"]["subject"] + + isLimited = chain.isLimitedProxy() + if not isLimited["OK"]: + raise RuntimeError(f"Failed to get SSL peer isLimitedProxy ({isLimited['Message']}).") + peer["isLimitedProxy"] = isLimited["Value"] + + return peer + + +class SSLTransport(BaseTransport): + """SSL Transport implementation based on the standard library ssl module.""" + + # This name is the same as BaseClient, + # and is used a bit everywhere, so it should be factorized out + # eventually + KW_TIMEOUT = "timeout" + + def __init__(self, *args, **kwargs): + """Create an SSLTransport object, parameters are the same + as for other transports. If ctx is specified (as an instance of + ssl.SSLContext) then use that rather than creating a new context. + + kwargs can contain all the parameters defined in BaseClient, + in particular timeout + """ + self.remoteAddress = None + self.peerCredentials = {} + + # The timeout is to be understood here as the timeout for socket + # operations involved in the RPC call, but NOT the establishment + # of the connection, for which there is a different timeout + # (DEFAULT_CONNECTION_TIMEOUT) + self.__timeout = kwargs.get(SSLTransport.KW_TIMEOUT, DEFAULT_RPC_TIMEOUT) + + self.__locked = False # We don't support locking, so this is always false. + + self.__ctx = kwargs.pop("ctx", None) + if not self.__ctx: + self.__ctx = getSSLContext(**kwargs) + + # Note that kwargs is already kept in BaseTransport + # as self.extraArgsDict, but at least I am sure that + # self.__kwargs will never be modified + self.__kwargs = kwargs + + BaseTransport.__init__(self, *args, **kwargs) + + def setSocketTimeout(self, timeout): + """Set the timeout for RPC calls. + + .. warning: This needs to be called before initAsClient. + It is used as a timeout for RPC calls, not connection. + + :param timeout: timeout for socket operation in seconds + + """ + self.__timeout = timeout + + def initAsClient(self): + """Prepare this client socket for use.""" + if self.serverMode(): + raise RuntimeError("SSLTransport is in server mode.") + + errors = [] + host, port = self.stServerAddress + + # Get all available addresses (IPv6 and IPv4) and try them in order + try: + addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except OSError as e: + return S_ERROR(f"DNS lookup failed {e!r}") + for family, socketType, proto, _canonname, socketAddress in addrInfoList: + rawSocket = None + try: + rawSocket = socket.socket(family, socketType, proto) + + # First set a short connection timeout, that will be used + # for the TCP connection and the TLS handshake + rawSocket.settimeout(DEFAULT_CONNECTION_TIMEOUT) + + # Enable keepAlive, with default options + # (see more comments about keepalive in :py:meth:`.acceptConnection`) + rawSocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + + rawSocket.connect(socketAddress) + + # server_hostname takes care of both SNI and the hostname check + self.oSocket = self.__ctx.wrap_socket(rawSocket, server_hostname=host) + + # Once the connection is established, we can use the timeout + # asked for RPC + self.oSocket.settimeout(self.__timeout) + + self.remoteAddress = self.oSocket.getpeername() + + return S_OK() + # warning: do NOT catch SSL related errors here + # They should be propagated upwards and caught by the BaseClient + # not to enter the retry loop + except ssl.SSLError: + if rawSocket is not None: + rawSocket.close() + raise + except OSError as e: + errors.append(f"{socketAddress} {e}:{repr(e)}") + if self.oSocket is not None: + self.close() + elif rawSocket is not None: + rawSocket.close() + + return S_ERROR("; ".join(errors)) + + def initAsServer(self): + """Prepare this server socket for use. + + Contrary to the client mode, the listening socket is a plain TCP + socket: the TLS layer is only added on the accepted connections, + in :py:meth:`.handshake` + """ + if not self.serverMode(): + raise RuntimeError("SSLTransport is in client mode.") + + try: + self.oSocket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + except OSError: + # Maybe no IPv6 support? Try IPv4 only socket. + self.oSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + # Make sure reuse address is set correctly + param = 1 if self.bAllowReuseAddress else 0 + self.oSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, param) + + self.oSocket.bind(self.stServerAddress) + self.oSocket.listen(self.iListenQueueSize) + return S_OK() + + def close(self): + """Close this socket.""" + if self.oSocket: + try: + self.oSocket.shutdown(socket.SHUT_RDWR) + except OSError: + # The socket may already be in a closed state + pass + self.oSocket.close() + self.oSocket = None + return S_OK() + + def renewServerContext(self): + """Renews the server context. + This reloads the certificates and the CA store by re-creating + the SSL context, which will be used for the subsequent connections. + """ + if not self.serverMode(): + raise RuntimeError("SSLTransport is in client mode.") + super().renewServerContext() + try: + self.__ctx = getSSLContext(**self.__kwargs) + except Exception as e: + gLogger.error("Failed to renew the server context", repr(e)) + return S_ERROR(f"Failed to renew the server context: {e!r}") + + return S_OK() + + def handshake(self): + """Perform the SSL handshake. + This has to be called after the connection was accepted (acceptConnection) + + The remote credentials are gathered here + """ + try: + # Warning: this method is called on the object returned + # by acceptConnection, which shares the context of the + # listening object + self.oSocket = self.__ctx.wrap_socket(self.oSocket, server_side=True) + + self.peerCredentials = getPeerInfo(self.oSocket) + + # Now that the handshake has been performed on the server + # we can set the timeout for the RPC operations. + # In practice, since we are on the server side, the + # timeout we set here represents the timeout for receiving + # the arguments and sending back the response. This should + # in principle be reasonably quick, but just to be sure + # we can set it to the DEFAULT_RPC_TIMEOUT + self.oSocket.settimeout(DEFAULT_RPC_TIMEOUT) + + return S_OK() + except (OSError, RuntimeError) as e: + return S_ERROR(f"Error in handshake: {e} {repr(e)}") + + def setClientSocket(self, oSocket): + """Set the inner socket of this instance to the value of oSocket. + This method is intended to be used to create client connection objects + from a server and should be considered to be an internal function. + + :param oSocket: client socket (plain, the handshake is done in :py:meth:`.handshake`) + + """ + # warning: do NOT catch socket.error here, because for who knows what reason + # exceptions are actually properly used for once, and the calling method + # relies on it (ServiceReactor.__acceptIncomingConnection) + self.oSocket = oSocket + self.remoteAddress = self.oSocket.getpeername() + + def acceptConnection(self): + """Accept a new client, returns a new SSLTransport object representing + the client connection. + + The connection is accepted, but no SSL handshake is performed + + :returns: S_OK(SSLTransport object) + """ + try: + oClient, _ = self.oSocket.accept() + + # Set the keep alive to true. This keepalive will ensure that we + # detect remote peer crashing or network interruption + # Note that this is ineffective if we are in the middle of blocking + # operations. + oClient.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + + # Here we set the timeout server side. + # We first set the connection timeout, which will + # be effective for the TLS handshake + oClient.settimeout(DEFAULT_CONNECTION_TIMEOUT) + + oClientTrans = SSLTransport(self.stServerAddress, ctx=self.__ctx) + oClientTrans.setClientSocket(oClient) + return S_OK(oClientTrans) + except (OSError, RuntimeError) as e: + return S_ERROR(f"Error in acceptConnection: {e} {repr(e)}") + + def _read(self, bufSize=4096, skipReadyCheck=False): + """Read bufSize bytes from the buffer. + + :param bufSize: size of the buffer to read + :param skipReadyCheck: ignored. + + + :returns: S_OK(data read) + """ + try: + read = self.oSocket.recv(bufSize) + return S_OK(read) + except OSError as e: + return S_ERROR(f"Error in _read: {e} {repr(e)}") + + def isLocked(self): + """Returns if this instance is locked. + Always returns false. + + :returns: False + """ + return self.__locked + + def _write(self, buf): + """Write all bytes contained within iterable "buf" to the + connected peer. + + :param buf: iterable buffer -# Eventhough SSLTransport is not used in this file, it is imported in other module from there, -# so do not remove these imports ! -from DIRAC.Core.DISET.private.Transports.M2SSLTransport import SSLTransport + :returns: S_OK(number of bytes written) + """ + try: + wrote = self.oSocket.send(buf) + return S_OK(wrote) + except OSError as e: + return S_ERROR(f"Error in _write: {e} {repr(e)}") def delegate(delegationRequest, kwargs): diff --git a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py index 0e8134652db..98ee3ac045c 100644 --- a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py @@ -7,7 +7,7 @@ from pytest import fixture from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.Core.DISET.private.Transports import M2SSLTransport, PlainTransport +from DIRAC.Core.DISET.private.Transports import PlainTransport, SSLTransport from DIRAC.Core.Security.test.x509TestUtilities import CERTDIR, USERCERT, getCertOption # TODO: Expired hostcert @@ -38,7 +38,7 @@ # Transports are now tested in pairs: # "Server-Client" # Each pair is defined as a string. -TRANSPORTTESTS = ("Plain-Plain", "M2-M2") +TRANSPORTTESTS = ("Plain-Plain", "SSL-SSL") # https://www.ibm.com/developerworks/linux/library/l-openssl/index.html @@ -121,8 +121,8 @@ def transportByName(transport): """helper function to get a transport class by 'friendly' name.""" if transport.lower() == "plain": return PlainTransport.PlainTransport - elif transport.lower() == "m2": - return M2SSLTransport.SSLTransport + elif transport.lower() == "ssl": + return SSLTransport.SSLTransport raise RuntimeError(f"Unknown Transport Name: {transport}") diff --git a/src/DIRAC/Core/Security/m2crypto/X509CRL.py b/src/DIRAC/Core/Security/X509CRL.py similarity index 71% rename from src/DIRAC/Core/Security/m2crypto/X509CRL.py rename to src/DIRAC/Core/Security/X509CRL.py index f942c2c41a6..fafc0550498 100644 --- a/src/DIRAC/Core/Security/m2crypto/X509CRL.py +++ b/src/DIRAC/Core/Security/X509CRL.py @@ -1,11 +1,12 @@ """ X509CRL is a class for managing X509CRL This class is used to manage the revoked certificates.... """ -import re import datetime -import M2Crypto.X509 +from cryptography import x509 + from DIRAC import S_OK, S_ERROR +from DIRAC.Core.Security import asn1_utils from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.File import secureOpenForWrite @@ -38,13 +39,13 @@ def loadCRLFromFile(self, crlLocation): """ self.__loadedCert = False try: - self.__revokedCert = M2Crypto.X509.load_crl(crlLocation) + with open(crlLocation, "rb") as crlFile: + pemData = crlFile.read() + self.__revokedCert = x509.load_pem_x509_crl(pemData) except Exception as e: return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')}") self.__loadedCert = True - with open(crlLocation) as crlFile: - pemData = crlFile.read() - self.__pemData = pemData + self.__pemData = pemData.decode("ascii") return S_OK() def __bytes__(self): @@ -79,24 +80,19 @@ def dumpAllToFile(self, filename=False): def hasExpired(self): if not self.__loadedCert: return S_ERROR("No certificate loaded") - # XXX It should be done better, for now M2Crypto doesn't offer access to fields like Next Update - txt = self.__revokedCert.as_text() - pattern = r"Next Update: (?P.*)\n" - dateStr = re.search(pattern, txt).group("nextUpdate") - nextUpdate = datetime.datetime.strptime(dateStr, "%b %d %H:%M:%S %Y GMT") - return S_OK(datetime.datetime.now() > nextUpdate) + nextUpdate = self.__revokedCert.next_update_utc + if nextUpdate is None: + return S_OK(False) + return S_OK(datetime.datetime.now(datetime.timezone.utc) > nextUpdate) def getIssuer(self): if not self.__loadedCert: return S_ERROR("No certificate loaded") - # XXX It should be done better, for now M2Crypto doesn't offer access to fields like Issuer - txt = self.__revokedCert.as_text() - pattern = r"Issuer: (?P.*)\n" - return S_OK(re.search(pattern, txt).group("issuer")) + return S_OK(asn1_utils.nameToDN(self.__revokedCert.issuer)) def __repr__(self): repStr = " X509Request -> X509Chain -> X509Certificate + from DIRAC.Core.Security.X509Request import X509Request + + req = X509Request() + req.generateProxyRequest(bitStrength=bitStrength, limited=limited) + + return S_OK(req) + + @executeOnlyIfCertLoaded + def getRemainingSecs(self): + """ + Get remaining lifetime in secs + + :returns: S_OK(remaining seconds) + """ + notAfter = self.getNotAfterDate()["Value"] + now = datetime.datetime.utcnow() + remainingSeconds = max(0, int((notAfter - now).total_seconds())) + + return S_OK(remainingSeconds) + + @executeOnlyIfCertLoaded + def getExtensions(self): + """ + Get a decoded list of extensions + + :returns: S_OK( list of tuple (extensionName, extensionValue)) + """ + extList = [] + for extension in self._certObj.extensions: + name = _EXT_OID_TO_NAME.get(extension.oid.dotted_string, "UNDEF") + try: + value = _formatExtensionValue(extension.value) + except Exception: + value = "Cannot decode value" + extList.append((name, value)) + + return S_OK(sorted(extList)) + + @executeOnlyIfCertLoaded + def verify(self, pkey): + """ + Verify the signature of the certificate using the public key provided + + :param pkey: cryptography public key object + + :returns: S_OK(bool) where the boolean shows the success of the verification + """ + try: + # The padding (PKCS1v15/PSS for RSA, ECDSA for EC) is derived from + # the signature algorithm of the certificate itself + signatureParameters = self._certObj.signature_algorithm_parameters + if isinstance(pkey, rsa.RSAPublicKey): + pkey.verify( + self._certObj.signature, + self._certObj.tbs_certificate_bytes, + signatureParameters, + self._certObj.signature_hash_algorithm, + ) + elif isinstance(pkey, ec.EllipticCurvePublicKey): + pkey.verify( + self._certObj.signature, + self._certObj.tbs_certificate_bytes, + signatureParameters, + ) + else: + pkey.verify(self._certObj.signature, self._certObj.tbs_certificate_bytes) + except Exception: + return S_OK(False) + return S_OK(True) + + @executeOnlyIfCertLoaded + def asPem(self): + """ + Return certificate as PEM string + + :returns: pem string + """ + return self._certObj.public_bytes(serialization.Encoding.PEM).decode("ascii") diff --git a/src/DIRAC/Core/Security/m2crypto/X509Chain.py b/src/DIRAC/Core/Security/X509Chain.py similarity index 85% rename from src/DIRAC/Core/Security/m2crypto/X509Chain.py rename to src/DIRAC/Core/Security/X509Chain.py index 1ebe73be38b..061ec4ae1e0 100644 --- a/src/DIRAC/Core/Security/m2crypto/X509Chain.py +++ b/src/DIRAC/Core/Security/X509Chain.py @@ -12,12 +12,21 @@ import re from collections import OrderedDict -import M2Crypto.X509 +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa from DIRAC import S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client.Helpers import Registry -from DIRAC.Core.Security.m2crypto import DEFAULT_PROXY_STRENGTH, DIRAC_GROUP_OID, LIMITED_PROXY_OID, PROXY_OID -from DIRAC.Core.Security.m2crypto.X509Certificate import X509Certificate +from DIRAC.Core.Security import ( + DEFAULT_PROXY_STRENGTH, + DIRAC_GROUP_OID, + LIMITED_PROXY_OID, + PROXY_CERT_INFO_EXTENSION_OID, + PROXY_OID, +) +from DIRAC.Core.Security import asn1_utils +from DIRAC.Core.Security.X509Certificate import X509Certificate from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.Decorators import executeOnlyIf from DIRAC.Core.Utilities.File import secureOpenForWrite @@ -31,18 +40,31 @@ _PROXY_LOAD_CACHE: OrderedDict[tuple, dict] = OrderedDict() _PROXY_LOAD_CACHE_MAX = 8 +_PEM_KEY_PATTERN = r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----(?:.|\n)*?-----END [A-Z0-9 ]*PRIVATE KEY-----" + + +def dumpPrivateKeyPEM(keyObj): + """Serialize a private key as an unencrypted PKCS8 PEM string + + :param keyObj: cryptography private key object + + :returns: PEM encoded string + """ + return keyObj.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode("ascii") + class X509Chain: """ - An X509Chain is basically a list of X509Certificate object, as well as a PKey object, + An X509Chain is basically a list of X509Certificate object, as well as a private key object, which is associated to the X509Certificate the lowest in the chain. This is what you will want to use for user certificate (because they will turn into proxy....), and for proxy. - A priori, once we get rid of pyGSI, we could even meld the X509Certificate into this one, and use the X509Chain - for host certificates. After all, a certificate is nothing but a chain of length 1... - There are normally 4 ways you would instanciate an X509Chain object: * You are loading a proxy from a file @@ -82,8 +104,8 @@ class X509Chain: Getting information from a peer in an SSL Connection:: - # conn is an M2Crypto.SSL.Connection instance - chain = X509Chain.generateX509ChainFromSSLConnection(conn) + # derList is the verified peer chain from ssl.SSLSocket.get_verified_chain() + chain = X509Chain.generateX509ChainFromDERList(derList) creds = chain.getCredentials() @@ -131,7 +153,7 @@ def __init__(self, certList=False, keyObj=False): C'tor :param certList: list of X509Certificate to constitute the chain - :param keyObj: ~M2Crypto.EVP.PKey object. The public or public/private key associated to + :param keyObj: cryptography private key object. The public or public/private key associated to the last certificate of the chain """ @@ -153,7 +175,7 @@ def __init__(self, certList=False, keyObj=False): # The certificate in position N has been generated from the (N+1) self._certList = [] - # Place holder for the EVP.PKey object + # Place holder for the private key object self._keyObj = None if certList: @@ -166,25 +188,16 @@ def __init__(self, certList=False, keyObj=False): self._keyObj = keyObj @staticmethod - def generateX509ChainFromSSLConnection(sslConnection): - """Returns an instance of X509Chain from the SSL connection + def generateX509ChainFromDERList(derList): + """Returns an instance of X509Chain from a list of DER encoded certificates, + as returned by ssl.SSLSocket.get_verified_chain() - :param sslConnection: ~M2Crypto.SSl.Connection instance + :param derList: iterable of DER encoded certificates, leaf first :returns: a X509Chain instance """ - certList = [] - - certStack = sslConnection.get_peer_cert_chain() - for cert in certStack: - certList.append(X509Certificate(x509Obj=cert)) - - # Servers don't receive the whole chain, the last cert comes alone - # if not self.infoDict['clientMode']: - certList.insert(0, X509Certificate(x509Obj=sslConnection.get_peer_cert())) - peerChain = X509Chain(certList=certList) - - return peerChain + certList = [X509Certificate(x509Obj=x509.load_der_x509_certificate(der)) for der in derList] + return X509Chain(certList=certList) def loadChainFromFile(self, chainLocation): """ @@ -231,19 +244,13 @@ def __certListFromPemString(certString): certList = [] pattern = r"(-----BEGIN CERTIFICATE-----((.|\n)*?)-----END CERTIFICATE-----)" for cert in re.findall(pattern, certString): - certList.append(X509Certificate(certString=cert[0])) + certObj = X509Certificate() + result = certObj.loadFromString(cert[0]) + if not result["OK"]: + raise ValueError(result["Message"]) + certList.append(certObj) return certList - # Not used in m2crypto version - # def setChain(self, certList): - # """ - # Set the chain - # Return : S_OK / S_ERROR - # """ - # self._certList = certList - # self.__loadedChain = True - # return S_OK() - def loadKeyFromFile(self, chainLocation, password=False): """ Load a PKey from a pem file @@ -270,12 +277,29 @@ def loadKeyFromString(self, pemData, password=False): :returns: S_OK / S_ERROR """ self._keyObj = None - if not isinstance(pemData, bytes): - pemData = pemData.encode("ascii") + if isinstance(pemData, bytes): + pemData = pemData.decode("ascii") + + # The pem data may contain certificates as well (e.g. a proxy file), + # so extract the key block + keyBlocks = re.findall(_PEM_KEY_PATTERN, pemData) + if not keyBlocks: + return S_ERROR(DErrno.ECERTREAD, "No private key found in the pem data") + if password: - password = password.encode() + password = password.encode() if isinstance(password, str) else password + else: + password = None + try: - self._keyObj = M2Crypto.EVP.load_key_string(pemData, lambda x: password) + try: + self._keyObj = serialization.load_pem_private_key(keyBlocks[0].encode("ascii"), password=password) + except TypeError: + # A password was supplied but the key is not encrypted: + # be tolerant, like the previous implementations + if password is None: + raise + self._keyObj = serialization.load_pem_private_key(keyBlocks[0].encode("ascii"), password=None) except Exception as e: return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')} (Probably bad pass phrase?)") @@ -350,36 +374,46 @@ def loadProxyFromString(self, pemData): @staticmethod def __getProxyExtensionList(diracGroup=False, rfcLimited=False): """ - Get an extension stack containing the necessary extension for a proxy. + Get the list of extensions necessary for a proxy. Basically the keyUsage, the proxyCertInfo, and eventually the diracGroup :param diracGroup: name of the dirac group for the proxy :param rfcLimited: boolean to generate for a limited proxy - :returns: M2Crypto.X509.X509_Extension_Stack object. + :returns: list of (cryptography.x509.ExtensionType, critical) tuples """ - extStack = M2Crypto.X509.X509_Extension_Stack() + extensions = [] # Standard certificate extensions - kUext = M2Crypto.X509.new_extension( - "keyUsage", "digitalSignature, keyEncipherment, dataEncipherment", critical=1 + keyUsage = x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=True, + data_encipherment=True, + key_agreement=False, + key_cert_sign=False, + crl_sign=False, + encipher_only=False, + decipher_only=False, ) - extStack.push(kUext) + extensions.append((keyUsage, True)) # Mandatory extension to be a proxy policyOID = LIMITED_PROXY_OID if rfcLimited else PROXY_OID - ext = M2Crypto.X509.new_extension("proxyCertInfo", f"critical, language:{policyOID}", critical=1) - extStack.push(ext) + proxyCertInfo = x509.UnrecognizedExtension( + x509.ObjectIdentifier(PROXY_CERT_INFO_EXTENSION_OID), asn1_utils.encodeProxyCertInfo(policyOID) + ) + extensions.append((proxyCertInfo, True)) # Add a dirac group if diracGroup and isinstance(diracGroup, str): - # the str cast is needed because M2Crypto does not play it cool with unicode here it seems - # Also one needs to specify the ASN1 type. That's what it is... - dGext = M2Crypto.X509.new_extension(DIRAC_GROUP_OID, str(f"ASN1:IA5:{diracGroup}")) - extStack.push(dGext) + diracGroupExt = x509.UnrecognizedExtension( + x509.ObjectIdentifier(DIRAC_GROUP_OID), asn1_utils.encodeDIRACGroup(diracGroup) + ) + extensions.append((diracGroupExt, False)) - return extStack + return extensions @needCertList def getCertInChain(self, certPos=0): @@ -426,15 +460,13 @@ def generateProxyToString( """ Generate a proxy and get it as a string. - Check here: https://github.com/eventbrite/m2crypto/blob/master/demo/x509/ca.py#L45 - Args: lifetime (int): expected lifetime in seconds of proxy diracGroup (str): diracGroup to add to the certificate strength (int): length in bits of the pair if proxyKey not given (default 2048) limited (bool): Create a limited proxy (default False) - proxyKey: M2Crypto.EVP.PKey instance with private and public key. If not given, generate one - rfc: placeholder for backward compatibility and ignored + proxyKey: cryptography key object (public, or private with public part) to certify. + If not given, generate one :returns: S_OK(PEM encoded string), S_ERROR. The PEM string contains all the certificates in the chain and the private key associated to the last X509Certificate just generated. @@ -444,29 +476,30 @@ def generateProxyToString( # If this is a certificate signing request then the private key will be # appended by the server and we don't need to include it in the proxy - include_private_key = not proxyKey + proxyPrivateKey = None if not proxyKey: - # Generating key is a two step process: create key object and then assign RSA key. - # This contains both the private and public key - proxyKey = M2Crypto.EVP.PKey() - proxyKey.assign_rsa(M2Crypto.RSA.gen_key(strength, 65537, callback=M2Crypto.util.quiet_genparam_callback)) + proxyPrivateKey = rsa.generate_private_key(public_exponent=65537, key_size=strength) + proxyPublicKey = proxyPrivateKey.public_key() + elif hasattr(proxyKey, "public_key"): + # A private key object was given + proxyPublicKey = proxyKey.public_key() + else: + proxyPublicKey = proxyKey - # Generate a new X509Certificate object + # Generate a new X509Certificate object, signed with one owns key proxyExtensions = self.__getProxyExtensionList(diracGroup, limited) - res = X509Certificate.generateProxyCertFromIssuer(issuerCert, proxyExtensions, proxyKey, lifetime=lifetime) + res = X509Certificate.generateProxyCertFromIssuer( + issuerCert, proxyExtensions, proxyPublicKey, self._keyObj, lifetime=lifetime + ) if not res["OK"]: return res proxyCert = res["Value"] - # Sign it with one owns key - proxyCert.sign(self._keyObj, "sha256") - # Generate the proxy string proxyString = proxyCert.asPem() - if include_private_key: - proxyString += proxyKey.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode("ascii") - for i in range(len(self._certList)): - crt = self._certList[i] + if proxyPrivateKey: + proxyString += dumpPrivateKeyPEM(proxyPrivateKey) + for crt in self._certList: proxyString += crt.asPem() return S_OK(proxyString) @@ -481,7 +514,6 @@ def generateProxyToFile(self, filePath, lifetime, diracGroup=False, strength=DEF diracGroup: diracGroup to add to the certificate strength: length in bits of the pair limited: Create a limited proxy - rfc: placeholder and ignored """ retVal = self.generateProxyToString(lifetime, diracGroup, strength, limited) if not retVal["OK"]: @@ -567,7 +599,7 @@ def getVOMSData(self): """ Returns the voms data. - :returns: See :py:func:`~DIRAC.Core.Security.m2crypto.X509Certificate.getVOMSData` + :returns: See :py:func:`~DIRAC.Core.Security.X509Certificate.X509Certificate.getVOMSData` If no VOMS data is available, return DErrno.EVOMS :warning: In case the chain is not a proxy, this method will return False. Yes, it's stupid, but it is for compatibility... @@ -661,21 +693,12 @@ def __checkProxyDN(self, certStep, issuerStep): # For non-RFC proxy, the proxy always had these two strings in the CN if lastEntry[1] not in ("proxy", "limited proxy"): # for RFC proxy, one has to check the extension. - ext = self._certList[certStep].getExtension("proxyCertInfo") - if not ext["OK"]: + try: + proxyCertInfo = asn1_utils.decodeProxyCertInfo(self._certList[certStep]._certObj) + except (LookupError, ValueError): return 0 - ext = ext["Value"] - - # Check the RFC - contraint = [ - line.split(":")[1].strip() - for line in ext.get_value().split("\n") - if line.split(":")[0] == "Policy Language" - ] - if not contraint: - return 0 - if contraint[0] == LIMITED_PROXY_OID: + if proxyCertInfo.proxyPolicy.policyLanguage.dotted_string == LIMITED_PROXY_OID: limited = True else: if lastEntry[1] == "limited proxy": @@ -691,7 +714,7 @@ def __checkIssuer(self, certStep, issuerStep): :param certStep: position of the certificate in self.__certList :param issuerStep: position of the issuer certificate - :returns: S_OK(boolean) + :returns: boolean """ issuerCert = self._certList[issuerStep] @@ -775,22 +798,22 @@ def getNotAfterDate(self): def generateProxyRequest(self, bitStrength=DEFAULT_PROXY_STRENGTH, limited=False): """ Generate a proxy request. - See :py:meth:`DIRAC.Core.Security.m2crypto.X509Certificate.X509Certificate.generateProxyRequest` + See :py:meth:`DIRAC.Core.Security.X509Certificate.X509Certificate.generateProxyRequest` Return S_OK( X509Request ) / S_ERROR """ # We use the first certificate of the chain to do the proxy request - x509 = self._certList[0] - return x509.generateProxyRequest(bitStrength, limited) + x509Cert = self._certList[0] + return x509Cert.generateProxyRequest(bitStrength, limited) @needCertList def getStrength(self): """ Returns the strength in bit of the key of the first certificate in the chain """ - x509 = self._certList[0] - return x509.getStrength() + x509Cert = self._certList[0] + return x509Cert.getStrength() @needCertList @needPKey @@ -802,15 +825,15 @@ def generateChainFromRequestString(self, pemData, lifetime=86400, requireLimited :param lifetime: lifetime of the delegated proxy in seconds (default 1 day) :param requireLimited: if True, requires a limited proxy :param diracGroup: DIRAC group to put in the proxy - :param rfc: placeholder for compatibility, ignored :returns: S_OK( X509 chain pem encoded string ) / S_ERROR. The new chain will have been signed with the public key included in the request """ + if not isinstance(pemData, bytes): + pemData = pemData.encode("ascii") try: - req = M2Crypto.X509.load_request_string(pemData, format=M2Crypto.X509.FORMAT_PEM) - + req = x509.load_pem_x509_csr(pemData) except Exception as e: return S_ERROR(DErrno.ECERTREAD, f"Can't load request data: {repr(e).replace(',)', ')')}") @@ -818,7 +841,7 @@ def generateChainFromRequestString(self, pemData, lifetime=86400, requireLimited # You can't request a limit proxy if you are yourself not limited ?! # I think it should be a "or" instead of "and" limited = requireLimited and self.isLimitedProxy().get("Value", False) - return self.generateProxyToString(lifetime, diracGroup, DEFAULT_PROXY_STRENGTH, limited, req.get_pubkey()) + return self.generateProxyToString(lifetime, diracGroup, DEFAULT_PROXY_STRENGTH, limited, req.public_key()) @needCertList def getRemainingSecs(self): @@ -848,7 +871,7 @@ def dumpAllToString(self): """ data = self._certList[0].asPem() if self._keyObj: - data += self._keyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode("ascii") + data += dumpPrivateKeyPEM(self._keyObj) for cert in self._certList[1:]: data += cert.asPem() return S_OK(data) @@ -889,7 +912,7 @@ def dumpPKeyToString(self): :returns: S_OK(PEM encoded key) """ - return S_OK(self._keyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode("ascii")) + return S_OK(dumpPrivateKeyPEM(self._keyObj)) def __str__(self): """String representation""" diff --git a/src/DIRAC/Core/Security/m2crypto/X509Request.py b/src/DIRAC/Core/Security/X509Request.py similarity index 60% rename from src/DIRAC/Core/Security/m2crypto/X509Request.py rename to src/DIRAC/Core/Security/X509Request.py index 917e3577586..0bdf9130922 100644 --- a/src/DIRAC/Core/Security/m2crypto/X509Request.py +++ b/src/DIRAC/Core/Security/X509Request.py @@ -1,13 +1,19 @@ """ X509Request is a class for managing X509 requests with their Pkeys. It's main use is for proxy delegation. """ -import M2Crypto.X509 +import re + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID + from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Security.m2crypto import DEFAULT_PROXY_STRENGTH +from DIRAC.Core.Security import DEFAULT_PROXY_STRENGTH +from DIRAC.Core.Security import asn1_utils +from DIRAC.Core.Security.X509Chain import _PEM_KEY_PATTERN, dumpPrivateKeyPEM from DIRAC.Core.Utilities import DErrno -# from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain # pylint: disable=import-error - # pylint: disable=broad-except @@ -15,15 +21,15 @@ class X509Request: """ Class representing X509 Certificate Request. it is used for delegation. Please see :ref:`about_proxies` for detailed explanations on delegation, - and :py:class:`DIRAC.Core.Security.m2crypto.X509Chain` for code examples. + and :py:class:`DIRAC.Core.Security.X509Chain` for code examples. """ def __init__(self, reqObj=None, pkeyObj=None): """C'tor - :param reqObj: M2Crypto.X509.Request object. Never used. Shall be removed - :param pkeyObj: M2Crypto.EVP.PKey() object. Never used. Shall be removed + :param reqObj: cryptography.x509.CertificateSigningRequest object. Never used. Shall be removed + :param pkeyObj: cryptography private key object. Never used. Shall be removed """ self.__valid = False self.__reqObj = reqObj @@ -39,22 +45,12 @@ def generateProxyRequest(self, bitStrength=DEFAULT_PROXY_STRENGTH, limited=False :param limited: (default False) If True, request is done for a limited proxy """ # self.__pkeyObj is both the public and private key - self.__pkeyObj = M2Crypto.EVP.PKey() - self.__pkeyObj.assign_rsa( - M2Crypto.RSA.gen_key(bitStrength, 65537, callback=M2Crypto.util.quiet_genparam_callback) + self.__pkeyObj = rsa.generate_private_key(public_exponent=65537, key_size=bitStrength) + + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "limited proxy" if limited else "proxy")]) + self.__reqObj = ( + x509.CertificateSigningRequestBuilder().subject_name(subject).sign(self.__pkeyObj, hashes.SHA256()) ) - self.__reqObj = M2Crypto.X509.Request() - self.__reqObj.set_pubkey(self.__pkeyObj) - - if limited: - self.__reqObj.get_subject().add_entry_by_txt( - field="CN", type=M2Crypto.ASN1.MBSTRING_ASC, entry="limited proxy", len=-1, loc=-1, set=0 - ) - else: - self.__reqObj.get_subject().add_entry_by_txt( - field="CN", type=M2Crypto.ASN1.MBSTRING_ASC, entry="proxy", len=-1, loc=-1, set=0 - ) - self.__reqObj.sign(self.__pkeyObj, "sha256") self.__valid = True def dumpRequest(self): @@ -66,23 +62,16 @@ def dumpRequest(self): if not self.__valid: return S_ERROR(DErrno.ENOCERT) try: - reqStr = self.__reqObj.as_pem().decode("ascii") + reqStr = self.__reqObj.public_bytes(serialization.Encoding.PEM).decode("ascii") except Exception as e: return S_ERROR(DErrno.EX509, f"Can't serialize request: {e}") return S_OK(reqStr) - # def getRequestObject(self): - # """ - # Get internal X509Request object - # Not used - # """ - # return S_OK(self.__reqObj) - def getPKey(self): """ Get PKey Internal - :returns: M2Crypto.EVP.PKEY object + :returns: cryptography private key object """ return self.__pkeyObj @@ -95,7 +84,7 @@ def dumpPKey(self): if not self.__valid: return S_ERROR(DErrno.ENOCERT) try: - pkeyStr = self.__pkeyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode("ascii") + pkeyStr = dumpPrivateKeyPEM(self.__pkeyObj) except Exception as e: return S_ERROR(DErrno.EX509, f"Can't serialize pkey: {e}") return S_OK(pkeyStr) @@ -121,39 +110,26 @@ def dumpAll(self): def loadAllFromString(self, pemData): """load the Request and key argument from a PEM encoded string. - :param pemData: PEN encoded string containing Request and PKey + :param pemData: PEM encoded string containing Request and PKey :returns: S_OK() """ - if not isinstance(pemData, bytes): - pemData = pemData.encode("ascii") + if isinstance(pemData, bytes): + pemData = pemData.decode("ascii") try: - self.__reqObj = M2Crypto.X509.load_request_string(pemData) + self.__reqObj = x509.load_pem_x509_csr(pemData.encode("ascii")) except Exception as e: return S_ERROR(DErrno.ENOCERT, str(e)) try: - self.__pkeyObj = M2Crypto.EVP.load_key_string(pemData) + keyBlocks = re.findall(_PEM_KEY_PATTERN, pemData) + if not keyBlocks: + raise ValueError("No private key found in the pem data") + self.__pkeyObj = serialization.load_pem_private_key(keyBlocks[0].encode("ascii"), password=None) except Exception as e: return S_ERROR(DErrno.ENOPKEY, str(e)) self.__valid = True return S_OK() - # def generateChainFromResponse(self, pemData): - # """ - # Generate a X509 Chain from the pkey and the pem data passed as the argument - # Return : S_OK( X509Chain ) / S_ERROR - # """ - # if not self.__valid: - # return S_ERROR(DErrno.ENOCERT) - # chain = X509Chain() - # ret = chain.loadChainFromString(pemData) - # if not ret['OK']: - # return ret - # ret = chain.setPKey(self.__pkeyObj) - # if not ret['OK']: - # return ret - # return chain - def getSubjectDN(self): """ Get subject DN of the request as a string @@ -162,7 +138,7 @@ def getSubjectDN(self): """ if not self.__valid: return S_ERROR(DErrno.ENOCERT) - return S_OK(str(self.__reqObj.get_subject())) + return S_OK(asn1_utils.nameToDN(self.__reqObj.subject)) def checkChain(self, chain): """ @@ -181,10 +157,12 @@ def checkChain(self, chain): if not chainPubKey["OK"]: return chainPubKey - # as_der will dump public key info, while as_pem - # dumps private key. - chainPubKey = chainPubKey["Value"].as_der() - reqPubKey = self.__reqObj.get_pubkey().as_der() + chainPubKey = chainPubKey["Value"].public_bytes( + serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo + ) + reqPubKey = self.__reqObj.public_key().public_bytes( + serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo + ) if not chainPubKey == reqPubKey: return S_ERROR(DErrno.EX509, "Public keys do not match") @@ -198,6 +176,6 @@ def getStrength(self): """ try: - return S_OK(self.__pkeyObj.size() * 8) + return S_OK(self.__pkeyObj.key_size) except Exception as e: return S_ERROR(f"Cannot get request strength: {e}") diff --git a/src/DIRAC/Core/Security/__init__.py b/src/DIRAC/Core/Security/__init__.py index 2f2209f7de1..f85e887a41a 100644 --- a/src/DIRAC/Core/Security/__init__.py +++ b/src/DIRAC/Core/Security/__init__.py @@ -1,30 +1,56 @@ """ -Just a selector for X509Certificate +DIRAC X509 security infrastructure, based on pyca/cryptography. + +This package provides the X509Certificate, X509Chain, X509Request and X509CRL +classes as submodules, as well as a set of OID constants used for handling +proxies and VOMS extensions. """ -from pkgutil import extend_path +# List of OIDs used in handling VOMS extension. +# VOMS extension is encoded in ASN.1 format and it's surprisingly hard to decode. OIDs describe content of sections +# of the data. There is no "official list of OIDs", ones used here are sourced from analyzing VOMS extensions itself +# and different pieces of code and presentations in subject of X509 certificates, certificate extensions and VOMS. +# Googling names or values of those OIDs, especially VOMS related, usually result in up to three pages of results, +# mainly Java code defining those values like code below. +# This is literally lookup table, so I know WTH is this, when I read value and see '1.3.6.1.4.1.8005.100.100.4'. + +DOMAIN_COMPONENT_OID = "0.9.2342.19200300.100.1.25" +DIRAC_GROUP_OID = "1.2.42.42" +VOMS_FQANS_OID = "1.3.6.1.4.1.8005.100.100.4" +VOMS_EXTENSION_OID = "1.3.6.1.4.1.8005.100.100.5" +VOMS_TAGS_EXT_OID = "1.3.6.1.4.1.8005.100.100.11" +COMMON_NAME_OID = "2.5.4.3" +SURNAME_OID = "2.5.4.4" +SERIALNUMBER_OID = "2.5.4.5" +COUNTRY_NAME = "2.5.4.6" +LOCALITY_NAME = "2.5.4.7" +STATE_OR_PROVINCE_NAME = "2.5.4.8" +ORGANIZATION_NAME = "2.5.4.10" +ORGANIZATIONAL_UNIT_NAME_OID = "2.5.4.11" +TITLE_OID = "2.5.4.12" +GIVEN_NAME_OID = "2.5.4.42" -##### -# SUPER DISGUSTING HACK -# We define these variables, and then remove them immediately. -# it is to allow something like 'from DIRAC.Core.Security import X509Chain' -# But pylint would complain just like that -# I've spent a lot of time trying to get pylint to work, but... -# https://github.com/PyCQA/pylint/issues/2474 +# See https://tools.ietf.org/html/rfc3820#appendix-A +PROXY_CERT_INFO_EXTENSION_OID = "1.3.6.1.5.5.7.1.14" +PROXY_OID = "1.3.6.1.5.5.7.21.1" +LIMITED_PROXY_OID = "1.3.6.1.4.1.3536.1.1.1.9" -X509Chain = None -X509CRL = None -X509Certificate = None -X509Request = None +# Some specific distinguished names: https://www.cryptosys.net/pki/manpki/pki_distnames.html -locals().pop("X509Chain") -locals().pop("X509CRL") -locals().pop("X509Certificate") -locals().pop("X509Request") -#### +DN_MAPPING = { + COMMON_NAME_OID: "/CN=", + COUNTRY_NAME: "/C=", + DOMAIN_COMPONENT_OID: "/DC=", + GIVEN_NAME_OID: "/G=", + LOCALITY_NAME: "/L=", + ORGANIZATION_NAME: "/O=", + ORGANIZATIONAL_UNIT_NAME_OID: "/OU=", + SERIALNUMBER_OID: "/SERIALNUMBER=", + STATE_OR_PROVINCE_NAME: "/ST=", + SURNAME_OID: "/SN=", + TITLE_OID: "/T=", +} -# Add the m2crypto subpackage to the search path -# This allows imports like 'from DIRAC.Core.Security.X509Chian...' to work transparently -# Nice kind of tricks you find in libraries like xml... -__path__ = extend_path(__path__, __name__ + ".m2crypto") +#: Default strength of the proxy in bit +DEFAULT_PROXY_STRENGTH = 2048 diff --git a/src/DIRAC/Core/Security/asn1_utils.py b/src/DIRAC/Core/Security/asn1_utils.py new file mode 100644 index 00000000000..46dde69ab40 --- /dev/null +++ b/src/DIRAC/Core/Security/asn1_utils.py @@ -0,0 +1,423 @@ +""" This module contains utilities for parsing X509 extensions, mostly the VOMS extensions. +It has been done based on the reading of the VOMS standard (https://www.ogf.org/documents/GFD.182.pdf) +and on the RFC 5755 (http://www.ietf.org/rfc/rfc5755.txt) + +This module relies on definition of the RFC 3281, which is the predecessor of 5755, but it still +seems to work for what we are interested in. + +To summarize, the attributes we are interested in are called CertificateAttributes, and are stored in proxy extensions. +The VOMS extension in a proxy is a Sequence of Sequence (??) of CertificateAttribute. One Sequence is due to the fact +that you can embed more than one VO CertificateAttribute in one proxy. The other one was acknowledge as a an error in +the formal description (an Errata will come) + +The ASN.1 structures are expressed with the declarative API of pyca/cryptography +(cryptography.hazmat.asn1), which does the parsing in Rust. The RFC 3281 module is +DEFINITIONS IMPLICIT TAGS; only the fields actually emitted by VOMS are modelled. +""" +from typing import Annotated + +from cryptography import x509 +from cryptography.hazmat import asn1 + +from DIRAC.Core.Security import ( + DIRAC_GROUP_OID, + DN_MAPPING, + PROXY_CERT_INFO_EXTENSION_OID, + VOMS_EXTENSION_OID, + VOMS_FQANS_OID, + VOMS_TAGS_EXT_OID, +) + +# GeneralName is a 9-way CHOICE in the RFC, but x509.Name is not supported +# inside asn1.Variant, so we model the two alternatives VOMS actually emits +# (directoryName [4], uniformResourceIdentifier [6]) as concrete per-context +# types instead. + + +@asn1.sequence +class AttributeTypeAndValue: + type: x509.ObjectIdentifier + value: asn1.TLV # ANY: a DirectoryString variant, decoded by tag in _decodeASN1String + + +#: RelativeDistinguishedName ::= SET OF AttributeTypeAndValue +Rdn = asn1.SetOf[AttributeTypeAndValue] +#: GeneralName/directoryName: [4] EXPLICIT Name (explicit because Name is a CHOICE) +DirName = Annotated[list[Rdn], asn1.Explicit(4)] +#: GeneralName/uniformResourceIdentifier: [6] IMPLICIT IA5String +UriName = Annotated[asn1.IA5String, asn1.Implicit(6)] + + +@asn1.sequence +class IssuerSerial: + issuer: list[DirName] + serial: int + issuerUID: asn1.BitString | None + + +@asn1.sequence +class Holder: + baseCertificateID: Annotated[IssuerSerial | None, asn1.Implicit(0)] + entityName: Annotated[list[DirName] | None, asn1.Implicit(1)] + # objectDigestInfo [2] OPTIONAL omitted: never present in VOMS ACs + + +@asn1.sequence +class V2Form: + issuerName: list[DirName] | None + baseCertificateID: Annotated[IssuerSerial | None, asn1.Implicit(0)] + + +@asn1.sequence +class AttCertValidityPeriod: + notBeforeTime: asn1.GeneralizedTime + notAfterTime: asn1.GeneralizedTime + + +@asn1.sequence +class Attribute: + type: x509.ObjectIdentifier + values: asn1.SetOf[asn1.TLV] + + +@asn1.sequence +class Extension: + extnID: x509.ObjectIdentifier + critical: Annotated[bool, asn1.Default(False)] + extnValue: bytes + + +@asn1.sequence +class AttributeCertificateInfo: + version: int + holder: Holder + # AttCertIssuer is a CHOICE, but v2 attribute certificates always use v2Form + issuer: Annotated[V2Form, asn1.Implicit(0)] + # AlgorithmIdentifier contains an `ANY OPTIONAL`, and we never read it, + # so keep it as an opaque TLV + signature: asn1.TLV + serialNumber: int + attrCertValidityPeriod: AttCertValidityPeriod + attributes: list[Attribute] + issuerUniqueID: asn1.BitString | None + extensions: list[Extension] | None + + +@asn1.sequence +class AttributeCertificate: + acinfo: AttributeCertificateInfo + signatureAlgorithm: asn1.TLV + signatureValue: asn1.BitString + + +@asn1.sequence +class IetfAttrSyntax: + policyAuthority: Annotated[list[UriName] | None, asn1.Implicit(0)] + values: list[bytes] + + +# The Tag structure is used to describe things like the nickname +# (See OGF 3.6.4) +@asn1.sequence +class _VOMSTag: + name: bytes + value: bytes + qualifier: bytes + + +@asn1.sequence +class _TagList: + policyAuthority: list[UriName] + tags: list[_VOMSTag] + + +# asn1.decode_der only accepts a registered class as the root, so the outer +# SEQUENCE OF layers get one-field wrapper classes. This is byte-identical to +# a single-element SEQUENCE OF, which is what is emitted in practice (and all +# that was ever supported: the previous implementation only read [0][0]). +@asn1.sequence +class _ACSequenceOfSequence: + acs: list[AttributeCertificate] + + +@asn1.sequence +class _TagContainers: + tagLists: list[_TagList] + + +# RFC 3820 proxyCertInfo, used both for generating and validating proxies +@asn1.sequence +class ProxyPolicy: + policyLanguage: x509.ObjectIdentifier + policy: bytes | None + + +@asn1.sequence +class ProxyCertInfo: + pCPathLenConstraint: int | None + proxyPolicy: ProxyPolicy + + +def retrieveExtension(cert, extensionOID): + """Retrieves the raw content of a certificate extension from its OID + + :param cert: cryptography.x509.Certificate object + :param extensionOID: the OID we are looking for, as a dotted string + + :returns: bytes, the DER content of the extension + (it still needs to be deserialized, depending on the extension !) + + :raises: LookupError if it does not have the extension + """ + try: + ext = cert.extensions.get_extension_for_oid(x509.ObjectIdentifier(extensionOID)) + except x509.ExtensionNotFound as e: + raise LookupError(f"Could not find extension with OID {extensionOID}") from e + value = ext.value + return value.public_bytes() if isinstance(value, x509.UnrecognizedExtension) else value.value + + +def decodeDIRACGroup(cert): + """Decode the content of the dirac group extension + + :param cert: cryptography.x509.Certificate object + + :returns: the dirac group + + :raises: same as retrieveExtension + """ + diracGroupData = retrieveExtension(cert, DIRAC_GROUP_OID) + return asn1.decode_der(asn1.IA5String, diracGroupData).as_str() + + +def encodeDIRACGroup(diracGroup): + """Encode a dirac group as the content of the dirac group extension + + :param diracGroup: the group name + + :returns: bytes, DER encoded IA5String + """ + return asn1.encode_der(asn1.IA5String(diracGroup)) + + +def encodeProxyCertInfo(policyOID): + """Encode an RFC 3820 proxyCertInfo extension content + + :param policyOID: dotted string of the proxy policy language OID + + :returns: bytes, DER encoded ProxyCertInfo + """ + proxyCertInfo = ProxyCertInfo( + pCPathLenConstraint=None, + proxyPolicy=ProxyPolicy(policyLanguage=x509.ObjectIdentifier(policyOID), policy=None), + ) + return asn1.encode_der(proxyCertInfo) + + +def decodeProxyCertInfo(cert): + """Retrieve and decode the RFC 3820 proxyCertInfo extension of a certificate + + :param cert: cryptography.x509.Certificate object + + :returns: ProxyCertInfo object + + :raises: LookupError if the certificate has no proxyCertInfo extension + """ + proxyCertInfoData = retrieveExtension(cert, PROXY_CERT_INFO_EXTENSION_OID) + return asn1.decode_der(ProxyCertInfo, proxyCertInfoData) + + +def _decodeASN1String(tlv): + """Decode an ASN.1 DirectoryString-ish value kept as a raw TLV. + + Historically (RFC 3280 & reality) the following types are found: + BMPString, IA5String, PrintableString, TeletexString, UTF8String. + BMPString is UTF-16-BE; everything else decodes fine as UTF-8. + + :param tlv: asn1.TLV object, the value part of AttributeTypeAndValue + + :returns: the decoded string + """ + codec = "utf-16-be" if tlv.tag_bytes[0] == 0x1E else "utf-8" + return bytes(tlv.data).decode(codec) + + +def _dirNameToDN(rdnSequence): + """Reconstruct a DN string like '/O=Dirac Computing/O=CERN/CN=MrUser' from + a decoded directoryName (list of Rdn) + + :param rdnSequence: list of Rdn objects + + :returns: the DN as a string + """ + dn = "" + for rdn in rdnSequence: + # The GFD 182 and RFC 3281 give enough restriction such that we can afford + # taking the first attribute only, like the previous implementation did + attr = rdn.as_list()[0] + dn += f"{DN_MAPPING[attr.type.dotted_string]}{_decodeASN1String(attr.value)}" + return dn + + +def hasVOMSExtension(cert): + """Utility function to check if the certificate has VOMS extensions + + :param cert: cryptography.x509.Certificate object + + :returns: boolean + """ + try: + retrieveExtension(cert, VOMS_EXTENSION_OID) + return True + except LookupError: + return False + + +def decodeVOMSExtension(cert): + """Decode the content of the VOMS extension + + :param cert: cryptography.x509.Certificate object + + :returns: A dictionary containing the following fields: + + * notBefore: datetime.datetime + * notAfter: datetime.datetime + * attribute: (string). Comma separated list of VOMS tags presented as bellow + + " = ()" + Typically, the nickname will look like + 'nickname = chaen (lhcb)', + + * fqan: List of VOMS "position" (['/lhcb/Role=production/Capability=NULL', '/lhcb/Role=NULL/Capability=NULL']) + * vo: name of the VO, + * subject: subject DN to which the attributes were granted, + * issuer: typically the DN of the VOMS server (e.g '/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch') + + :raises: LookupError if the certificate has no VOMS extension + """ + vomsExtensionDict = {} + vomsExtensionData = retrieveExtension(cert, VOMS_EXTENSION_OID) + + # In principle, according to GFD 182, there could be more than one VO VOMS AC per proxy. + # The standard specifies that we have to accept at least the first one, which is what + # we do... + vomsCertAttribute = asn1.decode_der(_ACSequenceOfSequence, vomsExtensionData).acs[0] + + ###### + # TODO in principle, we should check the signature of the Attribute... + # (vomsCertAttribute.signatureAlgorithm / signatureValue) + ###### + + certAttrInfo = vomsCertAttribute.acinfo + + # The declarative API does things correctly by setting a timezone info in the datetime + # However, we do not in DIRAC, and so we can't compare the dates. + # We have to remove the timezone info from the datetime objects + validity = certAttrInfo.attrCertValidityPeriod + vomsExtensionDict["notBefore"] = validity.notBeforeTime.as_datetime().replace(tzinfo=None) + vomsExtensionDict["notAfter"] = validity.notAfterTime.as_datetime().replace(tzinfo=None) + + # The issuer and holder DNs have to be reconstructed from the rdnSequence + # of their directoryName + vomsExtensionDict["issuer"] = _dirNameToDN(certAttrInfo.issuer.issuerName[0]) + vomsExtensionDict["subject"] = _dirNameToDN(certAttrInfo.holder.baseCertificateID.issuer[0]) + + # ### Retrieving the FQAN #### + + # According to GFD182, there may be more attributes than just the FQAN, even though it + # does not seem to be the case in practice. So we make sure to have the good one + fqanOIDObj = x509.ObjectIdentifier(VOMS_FQANS_OID) + + # There shall be only one, hence the [0] + fqanAttrObj = [attrObj for attrObj in certAttrInfo.attributes if attrObj.type == fqanOIDObj][0] + + # According to GFD182 3.4.1, we decode the value as a IetfAttrSyntax. + # Since multiple values are not allowed, just take the first item + fqanObj = fqanAttrObj.values.as_list()[0].parse(IetfAttrSyntax) + + # We retrieve the VO and the VOMS server + voName, _, _ = fqanObj.policyAuthority[0].as_str().split(":") + vomsExtensionDict["vo"] = voName + + # Now retrieve the position of the holder (group, role) + vomsExtensionDict["fqan"] = [fqanPosition.decode() for fqanPosition in fqanObj.values] + + # ############ End of the FQAN ################ + + # Now the Tags, called attributes in the dict... + + tagDescriptions = [] + vomsTagsOIDObj = x509.ObjectIdentifier(VOMS_TAGS_EXT_OID) + + # First find the tag containers + tagExtensionObj = [extObj for extObj in (certAttrInfo.extensions or []) if extObj.extnID == vomsTagsOIDObj] + + # If we found tags + if tagExtensionObj: + # Multiple is forbidden, so only one tag container + tagContainers = asn1.decode_der(_TagContainers, tagExtensionObj[0].extnValue) + + # TODO in principle, we should check that the policyAuthority + # and the one of the fqan are the same + for tagList in tagContainers.tagLists: + for tag in tagList.tags: + # This gives a string like + # nickname = chaen (lhcb) + tagDescriptions.append(f"{tag.name.decode()} = {tag.value.decode()} ({tag.qualifier.decode()})") + + vomsExtensionDict["attribute"] = ",".join(tagDescriptions) + + # #### Tags are done ################ + + return vomsExtensionDict + + +# Mapping from OID to the OpenSSL short names, used to render DNs the same way +# X509_NAME_oneline (i.e. M2Crypto's str(X509_Name)) used to +_OID_TO_SHORT_NAME = { + "2.5.4.3": "CN", + "2.5.4.4": "SN", + "2.5.4.5": "serialNumber", + "2.5.4.6": "C", + "2.5.4.7": "L", + "2.5.4.8": "ST", + "2.5.4.9": "street", + "2.5.4.10": "O", + "2.5.4.11": "OU", + "2.5.4.12": "title", + "2.5.4.42": "GN", + "2.5.4.43": "initials", + "2.5.4.46": "dnQualifier", + "2.5.4.65": "pseudonym", + "0.9.2342.19200300.100.1.1": "UID", + "0.9.2342.19200300.100.1.25": "DC", + "1.2.840.113549.1.9.1": "emailAddress", +} + + +def _onelineEscape(value) -> str: + """Escape an attribute value the way X509_NAME_oneline does: + bytes outside the printable ASCII range are rendered as ``\\xHH``. + This keeps DNs of certificates with non-ASCII characters identical + to what they always were (e.g. in the Registry). + """ + raw = value.encode("utf-8") if isinstance(value, str) else bytes(value) + if all(0x20 <= char <= 0x7E for char in raw): + return raw.decode("ascii") + return "".join(chr(char) if 0x20 <= char <= 0x7E else f"\\x{char:02X}" for char in raw) + + +def nameToDN(name: x509.Name) -> str: + """Render a cryptography x509.Name in the OpenSSL oneline format, + e.g. '/O=Dirac Computing/O=CERN/CN=MrUser' + + :param name: cryptography.x509.Name object + + :returns: the DN as a string + """ + dn = "" + for rdn in name.rdns: + for attr in rdn: + shortName = _OID_TO_SHORT_NAME.get(attr.oid.dotted_string, attr.oid.dotted_string) + dn += f"/{shortName}={_onelineEscape(attr.value)}" + return dn diff --git a/src/DIRAC/Core/Security/m2crypto/X509Certificate.py b/src/DIRAC/Core/Security/m2crypto/X509Certificate.py deleted file mode 100644 index 6e9f10d3b05..00000000000 --- a/src/DIRAC/Core/Security/m2crypto/X509Certificate.py +++ /dev/null @@ -1,500 +0,0 @@ -""" X509Certificate is a class for managing X509 certificates - -Proxy RFC: https://tools.ietf.org/html/rfc38200 - -X509RFC: https://tools.ietf.org/html/rfc5280 - -""" -import datetime -import os -import secrets -import time - -import M2Crypto.m2 -import M2Crypto.ASN1 -import M2Crypto.X509 - - -from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities import DErrno -from DIRAC.ConfigurationSystem.Client.Helpers import Registry -from DIRAC.Core.Security.m2crypto import asn1_utils, DEFAULT_PROXY_STRENGTH -from DIRAC.Core.Utilities.Decorators import executeOnlyIf - -# Decorator to execute the method only of the certificate has been loaded -executeOnlyIfCertLoaded = executeOnlyIf("_certLoaded", S_ERROR(DErrno.ENOCERT)) - - -class X509Certificate: - """The X509Certificate object represents ... a X509Certificate. - - It is a wrapper around a lower level implementation (M2Crypto in this case) of a certificate. - In theory, tt can be a host or user certificate. Also, a proxy certificate is a X509Certificate, - however it is useless without all the chain of issuers. - That's why one has the X509Chain. - - In practice, X509Certificate is just used for checking if the host certificate has expired. - This class will most probably disappear once we get ride of pyGSI. After all, a X509Certificate - is nothing but a X509Chain of length 1. - - Note that the SSL connection itself does not use this class, it gives directly the certificate to the library - """ - - def __init__(self, x509Obj=None, certString=None): - """ - Constructor. - You can give either nothing, or the x509Obj or the certString - - :param x509Obj: (optional) certificate instance - :type x509Obj: M2Crypto.X509.X509 - :param certString: text representation of certificate - :type certString: String - - """ - - self._certLoaded = False - if x509Obj: - self.__certObj = x509Obj - self._certLoaded = True - elif certString: - self.loadFromString(certString) - - # Pylint is surprisingly picky here, so remove that warning - # pylint: disable=protected-access - @classmethod - def generateProxyCertFromIssuer(cls, x509Issuer, x509ExtensionStack, proxyKey, lifetime=3600): - """This class method is meant to generate a new X509Certificate out of an existing one. - Basically, it generates a proxy... However, you can't have a proxy certificate working on - its own, you need all the chain of certificates. This method is meant to be called - only by the X509Chain class. - - Inspired from https://github.com/eventbrite/m2crypto/blob/master/demo/x509/ca.py#L45 - - :param x509Issuer: X509Certificate instance from which we generate the next one - :param x509ExtensionStack: M2Crypto.X509.X509_Extension_Stack object to add to the new certificate. - It contains all the X509 extensions needed for the proxy (e.g. DIRAC group). - See ~X509Chain.__getProxyExtensionList - :param proxyKey: a M2Crypto.EVP.PKey instance with private and public key - :param lifetime: duration of the proxy in second. Default 3600 - - - :returns: a new X509Certificate - - """ - proxyCert = cls() - - proxyCert.__certObj = M2Crypto.X509.X509() - - # According to the proxy RFC, the serial number just needs to be unique - # among the proxy generated by the issuer. - # We need to avoid birthday-style collisions for a given cert (such as - # the pilot) which may issue thousands of proxies per year. - serial = secrets.randbits(64) - proxyCert.__certObj.set_serial_number(serial) - - # No easy way to deep-copy certificate subject, since they are swig object - - # We basically get a string like 'O=Dirac Computing, O=CERN, CN=MrUser' - # So we split it, and then re-add each entry after the other. - - proxySubject = M2Crypto.X509.X509_Name() - - issuerSubjectObj = x509Issuer.__certObj.get_subject() - - # Copy the X509 entry components into the new name - for entry in issuerSubjectObj: - # pylint: disable=no-member - M2Crypto.m2.x509_name_add_entry(proxySubject.x509_name, entry.x509_name_entry, -1, 0) - - # Finally we add a random Common Name component. And we might as well use the serial.. :) - proxySubject.add_entry_by_txt( - field="CN", type=M2Crypto.ASN1.MBSTRING_ASC, entry=str(serial), len=-1, loc=-1, set=0 - ) - - proxyCert.__certObj.set_subject(proxySubject) - - # We now add all the extensions we wish to add - for extension in x509ExtensionStack: - proxyCert.__certObj.add_ext(extension) - - proxyCert.__certObj.set_issuer(issuerSubjectObj) - - # According to the X509 RFC, we are safe if we just copy the version - # number from the issuer certificate - proxyCert.__certObj.set_version(x509Issuer.__certObj.get_version()) - - proxyCert.__certObj.set_pubkey(proxyKey) - - # Set the start of the validity a bit in the past - # to be sure to be able to use it right now - proxyNotBefore = M2Crypto.ASN1.ASN1_UTCTIME() - proxyNotBefore.set_time(int(time.time()) - 900) - proxyCert.__certObj.set_not_before(proxyNotBefore) - - # Set the end date of the validity according to the lifetime - proxyNotAfter = M2Crypto.ASN1.ASN1_UTCTIME() - proxyNotAfter.set_time(int(time.time()) + lifetime) - proxyCert.__certObj.set_not_after(proxyNotAfter) - - # Finally set it as loaded. Care that it is not yet signed !! - proxyCert._certLoaded = True - - return S_OK(proxyCert) - - def load(self, certificate): - """Load an x509 certificate either from a file or from a string - - :param certificate: path to the file or PEM encoded string - - :returns: S_OK on success, otherwise S_ERROR - """ - - if os.path.exists(certificate): - return self.loadFromFile(certificate) - - return self.loadFromString(certificate) - - def loadFromFile(self, certLocation): - """ - Load a x509 cert from a pem file - - :param certLocation: path to the certificate file - - :returns: S_OK / S_ERROR. - - """ - try: - with open(certLocation) as fd: - pemData = fd.read() - return self.loadFromString(pemData) - except OSError: - return S_ERROR(DErrno.EOF, f"Can't open {certLocation} file") - - def loadFromString(self, pemData): - """ - Load a x509 cert from a string containing the pem data - - :param pemData: pem encoded string - - :returns: S_OK / S_ERROR - """ - if not isinstance(pemData, bytes): - pemData = pemData.encode("ascii") - try: - self.__certObj = M2Crypto.X509.load_cert_string(pemData, M2Crypto.X509.FORMAT_PEM) - except Exception as e: - return S_ERROR(DErrno.ECERTREAD, f"Can't load pem data: {e}") - - self._certLoaded = True - return S_OK() - - @executeOnlyIfCertLoaded - def hasExpired(self): - """ - Check if the loaded certificate is still valid - - :returns: S_OK( True/False )/S_ERROR - """ - - res = self.getNotAfterDate() - if not res["OK"]: - return res - - notAfter = res["Value"] - now = datetime.datetime.utcnow() - - return S_OK(notAfter < now) - - @executeOnlyIfCertLoaded - def getNotAfterDate(self): - """ - Get not after date of a certificate - - :returns: S_OK( datetime )/S_ERROR - """ - # Here we use the M2Crypto low level API, as the high level API is notably - # slower due to the conversion to a string and then back to an ASN1_TIME. - rawNotAfter = M2Crypto.m2.x509_get_not_after(self.__certObj.x509) # pylint: disable=no-member - notAfter = M2Crypto.ASN1.ASN1_TIME(rawNotAfter).get_datetime() - - # M2Crypto does things correctly by setting a timezone info in the datetime - # However, we do not in DIRAC, and so we can't compare the dates. - # We have to remove the timezone info from M2Crypto - notAfter = notAfter.replace(tzinfo=None) - - return S_OK(notAfter) - - @executeOnlyIfCertLoaded - def getStrength(self): - """ - Get the length of the key of the certificate in bit - - :returns: S_OK( size )/S_ERROR - """ - - try: - return S_OK(self.__certObj.get_pubkey().size() * 8) - except Exception as e: - return S_ERROR(f"Cannot get certificate strength: {e}") - - @executeOnlyIfCertLoaded - def getNotBeforeDate(self): - """ - Get not before date of a certificate - - :returns: S_OK( datetime )/S_ERROR - - """ - # Here we use the M2Crypto low level API, as the high level API is notably - # slower due to the conversion to a string and then back to an ASN1_TIME. - rawNotBefore = M2Crypto.m2.x509_get_not_before(self.__certObj.x509) # pylint: disable=no-member - return S_OK(M2Crypto.ASN1.ASN1_TIME(rawNotBefore).get_datetime()) - - # @executeOnlyIfCertLoaded - # def setNotBefore(self, notbefore): - # """ - # Set not before date of a certificate This method is not meant to be used, but to generate a proxy. - - # :returns: S_OK/S_ERROR - # """ - # self.__certObj.set_not_before(notbefore) - # return S_OK() - - @executeOnlyIfCertLoaded - def getSubjectDN(self): - """ - Get subject DN - - :returns: S_OK( string )/S_ERROR - """ - return S_OK(str(self.__certObj.get_subject())) - - @executeOnlyIfCertLoaded - def getIssuerDN(self): - """ - Get issuer DN - - :returns: S_OK( string )/S_ERROR - """ - return S_OK(str(self.__certObj.get_issuer())) - - @executeOnlyIfCertLoaded - def getSubjectNameObject(self): - """ - Get subject name object - - :returns: S_OK( X509Name )/S_ERROR - """ - return S_OK(self.__certObj.get_subject()) - - # The following method is in pyGSI, - # but are only used by the pyGSI SSL implementation - # So I do not really need them - - # @executeOnlyIfCertLoaded - # def getIssuerNameObject(self): - # """ - # Get issuer name object - - # :returns: S_OK( X509Name )/S_ERROR - # """ - # return S_OK(self.__certObj.get_issuer()) - - @executeOnlyIfCertLoaded - def getPublicKey(self): - """ - Get the public key of the certificate - - :returns: S_OK(M2crypto.EVP.PKey) - - """ - return S_OK(self.__certObj.get_pubkey()) - - @executeOnlyIfCertLoaded - def getSerialNumber(self): - """ - Get certificate serial number - - :returns: S_OK( serial )/S_ERROR - """ - return S_OK(self.__certObj.get_serial_number()) - - @executeOnlyIfCertLoaded - def sign(self, key, algo): - """ - Sign the cerificate using provided key and algorithm. - - :param key: M2crypto.EVP.PKey object with private and public key - :param algo: algorithm to sign the certificate - - :returns: S_OK/S_ERROR - """ - try: - self.__certObj.sign(key, algo) - except Exception as e: - return S_ERROR(repr(e)) - - return S_OK() - - @executeOnlyIfCertLoaded - def getDIRACGroup(self, ignoreDefault=False): - """ - Get the dirac group if present - - If no group is found in the certificate, we query the CS to get the default group - for the given user. This can be disabled using the ignoreDefault parameter - - Note that the lookup in the CS only can work for a proxy of first generation, - since we search based on the issuer DN - - :param ignoreDefault: if True, do not lookup the CS - - :returns: S_OK(group name/bool) - """ - try: - return S_OK(asn1_utils.decodeDIRACGroup(self.__certObj)) - except LookupError: - pass - - if ignoreDefault: - return S_OK(False) - - # And here is the flaw :) - result = self.getIssuerDN() - if not result["OK"]: - return result - return Registry.findDefaultGroupForDN(result["Value"]) - - @executeOnlyIfCertLoaded - def hasVOMSExtensions(self): - """ - Has voms extensions - - :returns: S_OK(bool) if voms extensions are found - """ - - # `get_ext` would be the correct thing to do. - # However, it does not work for the moment, as the extension - # is not registered with an alias - # https://gitlab.com/m2crypto/m2crypto/issues/231 - # try: - # self.__certObj.get_ext('vomsExtensions') - # return S_OK(True) - # except LookupError: - # # no extension found - # pass - - return S_OK(asn1_utils.hasVOMSExtension(self.__certObj)) - - @executeOnlyIfCertLoaded - def getVOMSData(self): - """ - Get voms extensions data - - :returns: S_ERROR/S_OK(dict). For the content of the dict, - see :py:func:`~DIRAC.Core.Security.m2crypto.asn1_utils.decodeVOMSExtension` - """ - try: - vomsExt = asn1_utils.decodeVOMSExtension(self.__certObj) - return S_OK(vomsExt) - except LookupError: - return S_ERROR(DErrno.EVOMS, "No VOMS data available") - - @executeOnlyIfCertLoaded - def generateProxyRequest(self, bitStrength=DEFAULT_PROXY_STRENGTH, limited=False): - """ - Generate a proxy request. See :py:class:`DIRAC.Core.Security.m2crypto.X509Request.X509Request` - - In principle, there is no reason to have this here, since a the X509Request is independant of - the 509Certificate when generating it. The only reason is to check whether the current Certificate - is limited or not. - - :param bitStrength: strength of the key - :param limited: if True or if the current certificate is limited (see proxy RFC), - creates a request for a limited proxy - - :returns: S_OK( :py:class:`DIRAC.Core.Security.m2crypto.X509Request.X509Request` ) / S_ERROR - """ - if not limited: - # We check whether "limited proxy" is in the subject - subj = self.__certObj.get_subject() - # M2Crypto does not understand the [-1] syntax... - lastEntry = subj[len(subj) - 1] - if lastEntry.get_data() == "limited proxy": - limited = True - - # The import is done here to avoid circular import - # X509Certificate -> X509Request -> X509Chain -> X509Certificate - from DIRAC.Core.Security.m2crypto.X509Request import X509Request - - req = X509Request() - req.generateProxyRequest(bitStrength=bitStrength, limited=limited) - - return S_OK(req) - - @executeOnlyIfCertLoaded - def getRemainingSecs(self): - """ - Get remaining lifetime in secs - - :returns: S_OK(remaining seconds) - """ - notAfter = self.getNotAfterDate()["Value"] - now = datetime.datetime.utcnow() - remainingSeconds = max(0, int((notAfter - now).total_seconds())) - - return S_OK(remainingSeconds) - - @executeOnlyIfCertLoaded - def getExtensions(self): - """ - Get a decoded list of extensions - - :returns: S_OK( list of tuple (extensionName, extensionValue)) - """ - extList = [] - for i in range(self.__certObj.get_ext_count()): - sn = self.__certObj.get_ext_at(i).get_name() - try: - value = self.__certObj.get_ext_at(i).get_value() - except Exception: - value = "Cannot decode value" - extList.append((sn, value)) - - return S_OK(sorted(extList)) - - @executeOnlyIfCertLoaded - def verify(self, pkey): - """ - Verify the signature of the certificate using the public key provided - - :param pkey: ~M2Crypto.EVP.PKey object - - :returns: S_OK(bool) where the boolean shows the success of the verification - """ - ret = self.__certObj.verify(pkey) - return S_OK(ret == 1) - - @executeOnlyIfCertLoaded - def asPem(self): - """ - Return certificate as PEM string - - :returns: pem string - """ - return self.__certObj.as_pem().decode("ascii") - - @executeOnlyIfCertLoaded - def getExtension(self, name): - """ - Return X509 Extension with given name - - :param name: name of the extension - - :returns: S_OK with M2Crypto.X509.X509_Extension object, or S_ERROR - """ - try: - ext = self.__certObj.get_ext(name) - except LookupError as e: - return S_ERROR(e) - return S_OK(ext) diff --git a/src/DIRAC/Core/Security/m2crypto/__init__.py b/src/DIRAC/Core/Security/m2crypto/__init__.py deleted file mode 100644 index a9205971485..00000000000 --- a/src/DIRAC/Core/Security/m2crypto/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -# List of OIDs used in handling VOMS extension. -# VOMS extension is encoded in ASN.1 format and it's surprisingly hard to decode. OIDs describe content of sections -# of the data. There is no "official list of OIDs", ones used here are sourced from analyzing VOMS extensions itself -# and different pieces of code and presentations in subject of X509 certificates, certificate extensions and VOMS. -# Googling names or values of those OIDs, especially VOMS related, usually result in up to three pages of results, -# mainly Java code defining those values like code below. -# This is literally lookup table, so I know WTH is this, when I read value and see '1.3.6.1.4.1.8005.100.100.4'. - -DOMAIN_COMPONENT_OID = "0.9.2342.19200300.100.1.25" -DIRAC_GROUP_OID = "1.2.42.42" -VOMS_FQANS_OID = "1.3.6.1.4.1.8005.100.100.4" -VOMS_EXTENSION_OID = "1.3.6.1.4.1.8005.100.100.5" -VOMS_TAGS_EXT_OID = "1.3.6.1.4.1.8005.100.100.11" -COMMON_NAME_OID = "2.5.4.3" -SURNAME_OID = "2.5.4.4" -SERIALNUMBER_OID = "2.5.4.5" -COUNTRY_NAME = "2.5.4.6" -LOCALITY_NAME = "2.5.4.7" -STATE_OR_PROVINCE_NAME = "2.5.4.8" -ORGANIZATION_NAME = "2.5.4.10" -ORGANIZATIONAL_UNIT_NAME_OID = "2.5.4.11" -TITLE_OID = "2.5.4.12" -GIVEN_NAME_OID = "2.5.4.42" - -# See https://tools.ietf.org/html/rfc3820#appendix-A -PROXY_OID = "1.3.6.1.5.5.7.21.1" -LIMITED_PROXY_OID = "1.3.6.1.4.1.3536.1.1.1.9" - -# Some specific distinguished names: https://www.cryptosys.net/pki/manpki/pki_distnames.html - -DN_MAPPING = { - COMMON_NAME_OID: "/CN=", - COUNTRY_NAME: "/C=", - DOMAIN_COMPONENT_OID: "/DC=", - GIVEN_NAME_OID: "/G=", - LOCALITY_NAME: "/L=", - ORGANIZATION_NAME: "/O=", - ORGANIZATIONAL_UNIT_NAME_OID: "/OU=", - SERIALNUMBER_OID: "/SERIALNUMBER=", - STATE_OR_PROVINCE_NAME: "/ST=", - SURNAME_OID: "/SN=", - TITLE_OID: "/T=", -} - - -#: Default strength of the proxy in bit -DEFAULT_PROXY_STRENGTH = 2048 diff --git a/src/DIRAC/Core/Security/m2crypto/asn1_utils.py b/src/DIRAC/Core/Security/m2crypto/asn1_utils.py deleted file mode 100644 index 72d80b2c43f..00000000000 --- a/src/DIRAC/Core/Security/m2crypto/asn1_utils.py +++ /dev/null @@ -1,368 +0,0 @@ -""" This module contains utilities for parsing extensions in general, but mostly the VOMS extensions. -It has been done based on the reading of the VOMS standard (https://www.ogf.org/documents/GFD.182.pdf) -and on the RFC 5755 (http://www.ietf.org/rfc/rfc5755.txt) - -This module relies on definition of the RFC 3281, which is the predecessor of 5755, but it still -seems to work for what we are interested in. - -To summarize, the attributes we are interested in are called CertificateAttributes, and are stored in proxy extensions. -The VOMS extension in a proxy is a Sequence of Sequence (??) of CertificateAttribute. One Sequence is due to the fact -that you can embed more than one VO CertificateAttribute in one proxy. The other one was acknowledge as a an error in -the formal description (an Errata will come) - -This is now pure python, but it might be interesting to wrap the existing -C library (https://github.com/italiangrid/voms) instead... - -""" -from functools import lru_cache - -from pyasn1.codec.der.decoder import decode as der_decode -from pyasn1.error import PyAsn1Error -from pyasn1.type import namedtype, univ, char as asn1char -from pyasn1_modules import rfc2459, rfc3281 -from DIRAC.Core.Security.m2crypto import ( - VOMS_EXTENSION_OID, - VOMS_FQANS_OID, - DN_MAPPING, - VOMS_TAGS_EXT_OID, - DIRAC_GROUP_OID, -) - - -class _ACSequence(univ.SequenceOf): - """This describe a sequence of AC, as per GFD 182""" - - componentType = rfc3281.AttributeCertificate() - - -class _ACSequenceOfSequence(univ.SequenceOf): - """A Sequence of Sequence of AC. In contradiction to GFD formal description, - but that's what it is - """ - - componentType = _ACSequence() - - -# The Tag structure is used to describe things like the nickname -# (See OGF 3.6.4) -# Similarely to above, there is an extra Sequence layer above the TagContainer -class _VOMSTag(univ.Sequence): - """Tag as per GOF 182""" - - componentType = namedtype.NamedTypes( - namedtype.NamedType("name", univ.OctetString()), - namedtype.NamedType("value", univ.OctetString()), - namedtype.NamedType("qualifier", univ.OctetString()), - ) - - -class _VOMSTags(univ.SequenceOf): - """Sequence of VOMSTag as per GOF 182""" - - componentType = _VOMSTag() - - -class _TagList(univ.Sequence): - """TagList as per GOF 182""" - - componentType = namedtype.NamedTypes( - namedtype.NamedType("policyAuthority", rfc2459.GeneralNames()), namedtype.NamedType("tags", _VOMSTags()) - ) - - -class _TagContainer(univ.SequenceOf): - """TagContainer as per GOF 182""" - - componentType = _TagList() - - -class _TagContainers(univ.SequenceOf): - """Sequence of TagContainer - Note sure why it is here, but that's how it is encoded - """ - - componentType = _TagContainer() - - -def decodeDIRACGroup(m2cert): - """Decode the content of the dirac group extension - - :param m2cert: M2crypto x509 object, a certificate - - :returns: the dirac group - - :raises: same as retrieveExtension - """ - - diracGroupOctetString = retrieveExtension(m2cert, DIRAC_GROUP_OID) - return _decodeDIRACGroup(diracGroupOctetString) - - -@lru_cache -def _decodeDIRACGroup(diracGroupOctetString): - diracGroupUTF8Str, _rest = der_decode(diracGroupOctetString, asn1Spec=asn1char.IA5String()) - - return diracGroupUTF8Str.asOctets().decode() - - -def _decodeASN1String(rdnNameAttrValue): - """Tries to decode a string encoded with the following type: - * BMPString - * IA5String - * PrintableString - * TeletexString - * UTF8String - - Most of these types come from the definition of the issuer field in RFC3280: - * The basic attributes, defined as DirectoryString (4.1.2.4 Issuer) - * the optional attributes (Appendix A. Psuedo-ASN.1 Structures and OIDs) - - This utility function is needed for 2 reasons: - * Not all the attributes are encoded the same way, and as we do not want to bother - with zillions of `if` conditions, we may just as well try - * It seems like the RFCs are not always respected, and the encoding is not always the correct one - - http://www.oid-info.com/ is a very good source of information for looking up the type of - a specific OID - - :param rdnNameAttrValue: the value part of rfc3280.AttributeTypeAndValue - - :returns: the decoded value or raises PyAsn1Error if nothing worked - """ - for decodeType in ( - asn1char.UTF8String, - asn1char.PrintableString, - asn1char.IA5String, - asn1char.TeletexString, - asn1char.BMPString, - ): - try: - attrValStr, _rest = der_decode(rdnNameAttrValue, decodeType()) - # Decoding error, try the next type - except PyAsn1Error: - pass - else: - # If the decoding worked, return it - return attrValStr - raise PyAsn1Error("Could not find a correct decoding type") - - -def hasVOMSExtension(m2cert): - """Utility fonction to check if the certificate has VOMS extensions - - :param m2cert: M2Crypto X509 object, a certificate - - :returns: boolean - """ - try: - retrieveExtension(m2cert, VOMS_EXTENSION_OID) - return True - except LookupError: - return False - - -def decodeVOMSExtension(m2cert): - """Decode the content of the VOMS extension - - :param m2cert: M2Crypto X509 object, a certificate - - :returns: A dictionary containing the following fields: - - * notBefore: datetime.datetime - * notAfter: datetime.datetime - * attribute: (string). Comma separated list of VOMS tags presented as below - - " = ()" - Typically, the nickname will look like - 'nickname = chaen (lhcb)', - - * fqan: List of VOMS "position" (['/lhcb/Role=production/Capability=NULL', '/lhcb/Role=NULL/Capability=NULL']) - * vo: name of the VO, - * subject: subject DN to which the attributes were granted, - * issuer: typically the DN of the VOMS server (e.g '/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch') - - """ - vomsExtensionDict = {} - vomsExtensionOctetString = retrieveExtension(m2cert, VOMS_EXTENSION_OID) - # Decode it as a ACSequenceOfSequence, which is what it is... - vomsExtensionSeqOfSeq, _rest = der_decode(vomsExtensionOctetString, asn1Spec=_ACSequenceOfSequence()) - - # In principle, according to GFD 182, there could be more than one VO VOMS AC per proxy. - # The standard specifies that we have to accept at least the first one, which is what - # I will do... - vomsCertAttribute = vomsExtensionSeqOfSeq[0][0] - - ###### - # TODO in principle, we should check the signature of the Attribute... - # _signatureAlgorith = vomsCertAttribute['signatureAlgorithm'] - # _signatureValue = vomsCertAttribute['signatureValue'] - ###### - - certAttrInfo = vomsCertAttribute["acinfo"] - - # pyasn1 does things correctly by setting a timezone info in the datetime - # However, we do not in DIRAC, and so we can't compare the dates. - # We have to remove the timezone info from the datetime objects - - notBefore = certAttrInfo["attrCertValidityPeriod"]["notBeforeTime"].asDateTime - vomsExtensionDict["notBefore"] = notBefore.replace(tzinfo=None) - - notAfter = certAttrInfo["attrCertValidityPeriod"]["notAfterTime"].asDateTime - vomsExtensionDict["notAfter"] = notAfter.replace(tzinfo=None) - - # ######### Retrieving the issuer ########## - # Get the issuer. A bit tricky, because we have to reconstruct the full DN ourselves - # The GFD 182 and RFC 3281 give enough restriction such that we can afford some direct - # [0] access - - issuer = "" - - # rdnName is a rfc3280.RelativeDistinguishedName object - for rdnName in certAttrInfo["issuer"]["v2Form"]["issuerName"][0]["directoryName"]["rdnSequence"]: - # rdnNameAttr rfc3280.AttributeTypeAndValue' - rdnNameAttr = rdnName[0] - - attrOid = ".".join([str(e) for e in rdnNameAttr["type"].asTuple()]) - - # Now finally convert the last part into a asn1char.*String - attrValStr = _decodeASN1String(rdnNameAttr["value"]) - attrVal = attrValStr.asOctets().decode() - # - issuer += f"{DN_MAPPING[attrOid]}{attrVal}" - - vomsExtensionDict["issuer"] = issuer - - # ### Issuer retrieved ##### - - # ## Retrieving the Subject #### - # We have to do the same for the subject than for the issuer - - subject = "" - - # rdnName is a rfc3280.RelativeDistinguishedName object - for rdnName in certAttrInfo["holder"]["baseCertificateID"]["issuer"][0]["directoryName"]["rdnSequence"]: - # rdnNameAttr rfc3280.AttributeTypeAndValue' - rdnNameAttr = rdnName[0] - - attrOid = ".".join([str(e) for e in rdnNameAttr["type"].asTuple()]) - # # Because there are non printable characters in the values (new line, etc) - # # we have to get ride of them. The best way is to get them as number, and make sure it is a - # # a printable char (between 32 and 126) - # - # attrVal = ''.join([chr(c) for c in rdnNameAttr['value'].asNumbers() if 32 <= c <= 126 ]) - - # Now finally convert the last part into a asn1char.*String - attrValStr = _decodeASN1String(rdnNameAttr["value"]) - attrVal = attrValStr.asOctets().decode() - - subject += f"{DN_MAPPING[attrOid]}{attrVal}" - - vomsExtensionDict["subject"] = subject - - # ### Retrieving the FQAN #### - - # According to GFD182, there may be more attributes that just the FQAN, even though it - # does not seem to be the case in practice. So we make sure to have the good one - fqanOIDObj = univ.ObjectIdentifier(VOMS_FQANS_OID) - - # There shall be only one, hense the [0] - # This is an rfc3280.Attribute object - fqanAttrObj = [attrObj for attrObj in certAttrInfo["attributes"] if attrObj["type"] == fqanOIDObj][0] - - # According to GFD182 3.4.1, we decode the value as a IetfAttrSyntax. - # Since multiple values are not allowed, just take the first item - # - fqanObj, _rest = der_decode(fqanAttrObj["values"][0], asn1Spec=rfc3281.IetfAttrSyntax()) - - # We retrieve the VO and the VOMS server - voName, _, _ = fqanObj["policyAuthority"][0]["uniformResourceIdentifier"].asOctets().decode().split(":") - - vomsExtensionDict["vo"] = voName - - # Now retrieve the position of the holder (group, role) - fqanList = [] - for fqanPositionObj in fqanObj["values"]: - fqanList.append(fqanPositionObj["octets"].asOctets().decode()) - - vomsExtensionDict["fqan"] = fqanList - - # ############ End of the FQAN ################ - - # Now the Tags, called attributes in the dict... - - tagDescriptions = [] - vomsTagsOIDObj = univ.ObjectIdentifier(VOMS_TAGS_EXT_OID) - - # First find the tag containers - tagExtensionObj = [extObj for extObj in certAttrInfo["extensions"] if extObj["extnID"] == vomsTagsOIDObj] - - # If we found tags - if tagExtensionObj: - # Multiple is forbiden, so only one tag container - tagExtensionObj = tagExtensionObj[0] - - tagContainersObj, _rest = der_decode(tagExtensionObj["extnValue"], asn1Spec=_TagContainers()) - - # TODO in principle, we should check that this value - # and the one of the policyAuthority of the fqan are the same - # _tagPolicyAuthority = tagContainersObj[0][0]['policyAuthority'][0]['uniformResourceIdentifier'] \ - # .asOctets().decode() - ###### - - for tagContainer in tagContainersObj: - for tagList in tagContainer: - # Note: it is here that I should check the policyAuthority - tagList = tagList["tags"] - for tag in tagList: - # This gives a string like - # nickname = chaen (lhcb) - tagDescriptions.append( - "%s = %s (%s)" - % ( - tag["name"].asOctets().decode(), - tag["value"].asOctets().decode(), - tag["qualifier"].asOctets().decode(), - ) - ) - - vomsExtensionDict["attribute"] = ",".join(tagDescriptions) - - # #### Tags are done ################ - - return vomsExtensionDict - - -def retrieveExtension(m2Cert, extensionOID): - """Retrieves the extension from a certificate from its OID - - :param m2Cert: M2Crypto X509 object, a certificate - :param extensionOID: the OID we are looking for - - :returns: an ~pyasn1.type.univ.OctetString object, which is the content of the extension - (it still needs to be deserialized, depending on the extension !) - - :raises: LookupError if it does not have the extension - """ - extensions = _extensionsFromCertDER(m2Cert.as_der()) - - # Construct an OID object for comparison purpose - extensionOIDObj = univ.ObjectIdentifier(extensionOID) - - # We check every extension OID. This will be necessary until M2Crypto - # allows to register OID alias (https://gitlab.com/m2crypto/m2crypto/issues/231) - for extension in extensions: - # We found the good extension - if extension["extnID"] == extensionOIDObj: - return extension["extnValue"] - - # If we are here, it means that we could not find the expected extension. - raise LookupError(f"Could not find extension with OID {extensionOID}") - - -@lru_cache(maxsize=1024) -def _extensionsFromCertDER(der): - # Decode the certificate as a RFC2459 Certificate object.It is compatible - # with the RFC proxy definition - cert, _rest = der_decode(der, asn1Spec=rfc2459.Certificate()) - extensions = cert["tbsCertificate"]["extensions"] - return extensions diff --git a/src/DIRAC/Core/Security/test/Test_X509Certificate.py b/src/DIRAC/Core/Security/test/Test_X509Certificate.py index 2bac7d7792b..0d18daa4c4f 100644 --- a/src/DIRAC/Core/Security/test/Test_X509Certificate.py +++ b/src/DIRAC/Core/Security/test/Test_X509Certificate.py @@ -29,7 +29,7 @@ parametrize = mark.parametrize -X509CERTTYPES = ("M2_X509Certificate",) +X509CERTTYPES = ("PYCA_X509Certificate",) # This fixture will return a X509Certificate class # https://docs.pytest.org/en/latest/fixture.html#automatic-grouping-of-tests-by-fixture-instances @@ -45,8 +45,8 @@ def get_X509Certificate_class(request): x509Class = request.param - if x509Class == "M2_X509Certificate": - from DIRAC.Core.Security.m2crypto.X509Certificate import X509Certificate + if x509Class == "PYCA_X509Certificate": + from DIRAC.Core.Security.X509Certificate import X509Certificate else: raise NotImplementedError() diff --git a/src/DIRAC/Core/Security/test/Test_X509Chain.py b/src/DIRAC/Core/Security/test/Test_X509Chain.py index bd30201182e..3ab891a5e16 100644 --- a/src/DIRAC/Core/Security/test/Test_X509Chain.py +++ b/src/DIRAC/Core/Security/test/Test_X509Chain.py @@ -79,8 +79,8 @@ def get_proxy(request): deimportDIRAC() x509Class = request.param - if x509Class == "M2_X509Chain": - from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain + if x509Class == "PYCA_X509Chain": + from DIRAC.Core.Security.X509Chain import X509Chain else: raise NotImplementedError() diff --git a/src/DIRAC/Core/Security/test/x509TestUtilities.py b/src/DIRAC/Core/Security/test/x509TestUtilities.py index 530223ca9be..b9418329df5 100644 --- a/src/DIRAC/Core/Security/test/x509TestUtilities.py +++ b/src/DIRAC/Core/Security/test/x509TestUtilities.py @@ -298,7 +298,7 @@ def deimportDIRAC(): # sys.modules.pop(mod) -X509CHAINTYPES = ("M2_X509Chain",) +X509CHAINTYPES = ("PYCA_X509Chain",) # This fixture will return a pyGSI or M2Crypto X509Chain class # https://docs.pytest.org/en/latest/fixture.html#automatic-grouping-of-tests-by-fixture-instances @@ -314,8 +314,8 @@ def get_X509Chain_class(request): x509Class = request.param - if x509Class == "M2_X509Chain": - from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain + if x509Class == "PYCA_X509Chain": + from DIRAC.Core.Security.X509Chain import X509Chain else: raise NotImplementedError() @@ -325,7 +325,7 @@ def get_X509Chain_class(request): deimportDIRAC() -X509REQUESTTYPES = ("M2_X509Request",) +X509REQUESTTYPES = ("PYCA_X509Request",) # This fixture will return a X509Request class # https://docs.pytest.org/en/latest/fixture.html#automatic-grouping-of-tests-by-fixture-instances @@ -341,8 +341,8 @@ def get_X509Request(request): x509Class = request.param - if x509Class == "M2_X509Request": - from DIRAC.Core.Security.m2crypto.X509Request import X509Request + if x509Class == "PYCA_X509Request": + from DIRAC.Core.Security.X509Request import X509Request else: raise NotImplementedError() @@ -368,8 +368,8 @@ def get_X509Chain_from_X509Request(x509ReqObj): """ # In principle, we should deimport Dirac everywhere, but I am not even sure it makes any difference - if "m2crypto" in x509ReqObj.__class__.__module__: - from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain + if "X509Request" in x509ReqObj.__class__.__module__: + from DIRAC.Core.Security.X509Chain import X509Chain else: raise NotImplementedError() diff --git a/src/DIRAC/Core/Tornado/Server/TornadoServer.py b/src/DIRAC/Core/Tornado/Server/TornadoServer.py index f10eccd5742..12aa1f84091 100644 --- a/src/DIRAC/Core/Tornado/Server/TornadoServer.py +++ b/src/DIRAC/Core/Tornado/Server/TornadoServer.py @@ -9,27 +9,19 @@ import psutil import secrets -import M2Crypto.SSL - -import tornado.iostream - -tornado.iostream.SSLIOStream.configure( # pylint: disable=no-member - "tornado_m2crypto.m2iostream.M2IOStream" -) # pylint: disable=wrong-import-position - import tornado.platform.asyncio import tornado.ioloop from tornado.httpserver import HTTPServer from tornado.web import Application, RequestHandler from DIRAC import gConfig, gLogger, S_OK +from DIRAC.Core.DISET.private.Transports.SSLTransport import getSSLContext from DIRAC.Core.Security import Locations from DIRAC.Core.Utilities import Network, TimeUtilities from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations sLog = gLogger.getSubLogger(__name__) -DEBUG_M2CRYPTO = os.getenv("DIRAC_DEBUG_M2CRYPTO", "No").lower() in ("yes", "true") class NotFoundHandler(RequestHandler): @@ -188,19 +180,16 @@ def startTornado(self): sLog.debug("Starting Tornado") - # Prepare SSL settings + # Prepare SSL settings. + # The client certificate is optional: some requests are authenticated + # with a token or as visitor instead (see BaseRequestHandler) + try: + ssl_options = getSSLContext(bServerMode=True, optionalClientCert=True) + except RuntimeError as e: + sLog.fatal("Unable to prepare the TLS settings ! Can't start the Server", repr(e)) + raise ImportError(f"Unable to prepare the TLS settings: {e}") from e certs = Locations.getHostCertificateAndKeyLocation() - if certs is False: - sLog.fatal("Host certificates not found ! Can't start the Server") - raise ImportError("Unable to load certificates") ca = Locations.getCAsLocation() - ssl_options = { - "certfile": certs[0], - "keyfile": certs[1], - "cert_reqs": M2Crypto.SSL.verify_peer, - "ca_certs": ca, - "sslDebug": DEBUG_M2CRYPTO, # Set to true if you want to see the TLS debug messages - } # Init monitoring if self.activityMonitoring: @@ -214,7 +203,7 @@ def startTornado(self): tornado.ioloop.PeriodicCallback(self.__reportToMonitoring, self.__monitoringLoopDelay * 1000).start() for port, app in self.__appsSettings.items(): - sLog.debug(" - %s" % "\n - ".join([f"{k} = {ssl_options[k]}" for k in ssl_options])) + sLog.debug(f" - certfile = {certs[0]}\n - keyfile = {certs[1]}\n - ca_certs = {ca}") # Default server configuration settings = dict(compress_response=True, cookie_secret=secrets.token_hex(32)) diff --git a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py index bb57e93f2df..ae8043ec49a 100644 --- a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py +++ b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py @@ -736,31 +736,29 @@ def _authzSSL(self): :return: S_OK(dict)/S_ERROR() """ try: - derCert = self.request.get_ssl_certificate() + # The peer chain, as sent by the client (leaf first, DER encoded). + # It was validated by OpenSSL during the TLS handshake + derChain = self.request.connection.stream.socket.get_unverified_chain() except Exception: - # If 'IOStream' object has no attribute 'get_ssl_certificate' - derCert = None + # Plain HTTP connection, or no client certificate presented + derChain = None # Boolean whether we are behind a balancer and can trust headers balancer = gConfig.getValue("/WebApp/Balancer", "none") != "none" - # Get client certificate as pem - if derCert: - chainAsText = derCert.as_pem().decode("ascii") - # Read all certificate chain - chainAsText += "".join([cert.as_pem().decode("ascii") for cert in self.request.get_ssl_certificate_chain()]) + # Get the client certificate chain + if derChain: + peerChain = X509Chain.generateX509ChainFromDERList(derChain) elif balancer: if self.request.headers.get("X-Ssl_client_verify") == "SUCCESS" and self.request.headers.get("X-SSL-CERT"): chainAsText = unquote(self.request.headers.get("X-SSL-CERT")) + peerChain = X509Chain() + peerChain.loadChainFromString(chainAsText) else: return S_ERROR(DErrno.ECERTFIND, "Valid certificate not found.") else: return S_ERROR(DErrno.ECERTFIND, "Valid certificate not found.") - # Load full certificate chain - peerChain = X509Chain() - peerChain.loadChainFromString(chainAsText) - # Retrieve the credentials res = peerChain.getCredentials(withRegistryInfo=False) if not res["OK"]: diff --git a/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py b/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py index 3329e90718e..cf7f4cfcd32 100644 --- a/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py +++ b/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py @@ -6,7 +6,7 @@ from prompt_toolkit import prompt from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.Core.Base.Script import Script -from DIRAC.Core.Security.m2crypto import DEFAULT_PROXY_STRENGTH +from DIRAC.Core.Security import DEFAULT_PROXY_STRENGTH class CLIParams: diff --git a/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py b/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py index 37f75140487..218ab5a64c3 100644 --- a/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py +++ b/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py @@ -31,17 +31,55 @@ """ import re -import time import secrets import datetime import collections -from M2Crypto import m2, util, X509, ASN1, EVP, RSA +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.Core.Security import asn1_utils from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Resources.ProxyProvider.ProxyProvider import ProxyProvider +# Mapping between the distinguished name aliases and their OID, replacing +# the M2Crypto X509_Name.nid lookup table +_FIELDS_TO_OID = { + "C": "2.5.4.6", + "countryName": "2.5.4.6", + "SP": "2.5.4.8", + "ST": "2.5.4.8", + "stateOrProvinceName": "2.5.4.8", + "L": "2.5.4.7", + "localityName": "2.5.4.7", + "O": "2.5.4.10", + "organizationName": "2.5.4.10", + "OU": "2.5.4.11", + "organizationalUnitName": "2.5.4.11", + "CN": "2.5.4.3", + "commonName": "2.5.4.3", + "Email": "1.2.840.113549.1.9.1", + "emailAddress": "1.2.840.113549.1.9.1", + "SN": "2.5.4.4", + "surname": "2.5.4.4", + "GN": "2.5.4.42", + "givenName": "2.5.4.42", + "serialNumber": "2.5.4.5", + "SERIALNUMBER": "2.5.4.5", + "DC": "0.9.2342.19200300.100.1.25", + "domainComponent": "0.9.2342.19200300.100.1.25", +} + +_ALGORITHMS = { + "sha1": hashes.SHA1, + "sha224": hashes.SHA224, + "sha256": hashes.SHA256, + "sha384": hashes.SHA384, + "sha512": hashes.SHA512, +} + class DIRACCAProxyProvider(ProxyProvider): def __init__(self, parameters=None): @@ -54,21 +92,18 @@ def __init__(self, parameters=None): self.bits = 2048 self.algoritm = "sha256" self.match = [] - self.supplied = ["CN"] - self.optional = ["C", "O", "OU", "emailAddress"] + self.supplied = [_FIELDS_TO_OID["CN"]] + self.optional = [_FIELDS_TO_OID[f] for f in ("C", "O", "OU", "emailAddress")] self.dnList = ["C", "O", "OU", "CN", "emailAddress"] # Distinguished names - self.fields2nid = X509.X509_Name.nid.copy() - self.fields2nid["DC"] = -1 # Add DN that is not liested in X509.X509_Name - self.fields2nid["domainComponent"] = -1 # Add DN description that is not liested in X509.X509_Name - self.fields2nid["organizationalUnitName"] = 18 # Add 'OU' description - self.fields2nid["countryName"] = 14 # Add 'C' description - self.fields2nid["SERIALNUMBER"] = 105 # Add 'SERIALNUMBER' distinguished name - self.nid2fields = {} # nid: list of distinguished names + self.fields2oid = dict(_FIELDS_TO_OID) + self.oid2fields = {} # oid: list of distinguished names # Specify standart fields - for field, nid in self.fields2nid.items(): - self.nid2fields.setdefault(nid, []).append(field) + for field, oid in self.fields2oid.items(): + self.oid2fields.setdefault(oid, []).append(field) self.dnInfoDictCA = {} + # List of x509.NameAttribute of the DN being built + self.__nameAttributes = [] def setParameters(self, parameters): """Set new parameters @@ -78,7 +113,7 @@ def setParameters(self, parameters): :return: S_OK()/S_ERROR() """ for k, v in parameters.items(): - if not isinstance(v, list) and k in ["Match", "Supplied", "Optional", "DNOrder"] + list(self.fields2nid): + if not isinstance(v, list) and k in ["Match", "Supplied", "Optional", "DNOrder"] + list(self.fields2oid): parameters[k] = v.replace(", ", ",").split(",") self.parameters = parameters # If CA configuration file exist @@ -89,26 +124,26 @@ def setParameters(self, parameters): if "Algoritm" in parameters: self.algoritm = parameters["Algoritm"] if "Match" in parameters: - self.match = [self.fields2nid[f] for f in parameters["Match"]] + self.match = [self.fields2oid[f] for f in parameters["Match"]] if "Supplied" in parameters: - self.supplied = [self.fields2nid[f] for f in parameters["Supplied"]] + self.supplied = [self.fields2oid[f] for f in parameters["Supplied"]] if "Optional" in parameters: - self.optional = [self.fields2nid[f] for f in parameters["Optional"]] + self.optional = [self.fields2oid[f] for f in parameters["Optional"]] allFields = self.optional + self.supplied + self.match if "DNOrder" in parameters: self.dnList = [] - if not any([any([f in parameters["DNOrder"] for f in self.nid2fields[n]]) for n in allFields]): + if not any([any([f in parameters["DNOrder"] for f in self.oid2fields[n]]) for n in allFields]): return S_ERROR("DNOrder must contain all configured fields.") for field in parameters["DNOrder"]: - if self.fields2nid[field] in allFields: + if self.fields2oid[field] in allFields: self.dnList.append(field) # Set defaults for distridutes names - self.nid2defField = {} + self.oid2defField = {} for field, value in list(self.parameters.items()): - if field in self.fields2nid and self.fields2nid[field] in allFields: - self.parameters[self.fields2nid[field]] = value - self.nid2defField[self.fields2nid[field]] = field + if field in self.fields2oid and self.fields2oid[field] in allFields: + self.parameters[self.fields2oid[field]] = value + self.oid2defField[self.fields2oid[field]] = field # Read CA certificate chain = X509Chain() @@ -136,52 +171,52 @@ def checkStatus(self, userDN): dnInfoDict = result["Value"] try: - userNIDs = [self.fields2nid[f.split("=")[0]] for f in userDN.lstrip("/").split("/")] + userOIDs = [self.fields2oid[f.split("=")[0]] for f in userDN.lstrip("/").split("/")] except (ValueError, KeyError) as e: return S_ERROR(f"Unknown DN field in used DN: {e}") - nidOrder = [self.fields2nid[f] for f in self.dnList] - for index, nid in enumerate(userNIDs): - if nid not in nidOrder: + oidOrder = [self.fields2oid[f] for f in self.dnList] + for index, oid in enumerate(userOIDs): + if oid not in oidOrder: return S_ERROR( - f'"{self.nid2defField.get(nid, min(self.nid2fields[nid], key=len))}" field not found in order.' + f'"{self.oid2defField.get(oid, min(self.oid2fields[oid], key=len))}" field not found in order.' ) - if index > nidOrder.index(nid): + if index > oidOrder.index(oid): return S_ERROR("Bad DNs order") - for i in range(nidOrder.index(nid) - 1): + for i in range(oidOrder.index(oid) - 1): try: - if userNIDs.index(nidOrder[i]) > index: + if userOIDs.index(oidOrder[i]) > index: return S_ERROR("Bad DNs order") except (ValueError, KeyError): continue - for i in range(nidOrder.index(nid) + 1, len(nidOrder)): + for i in range(oidOrder.index(oid) + 1, len(oidOrder)): try: - if userNIDs.index(nidOrder[i]) < index: + if userOIDs.index(oidOrder[i]) < index: return S_ERROR("Bad DNs order") except (ValueError, KeyError): continue - for nid in self.supplied: - if nid not in [self.fields2nid[f] for f in dnInfoDict]: + for oid in self.supplied: + if oid not in [self.fields2oid[f] for f in dnInfoDict]: return S_ERROR( 'Current DN is invalid, "%s" field must be set.' - % self.nid2defField.get(nid, min(self.nid2fields[nid], key=len)) + % self.oid2defField.get(oid, min(self.oid2fields[oid], key=len)) ) for field, values in dnInfoDict.items(): - nid = self.fields2nid[field] + oid = self.fields2oid[field] err = f'Current DN is invalid, "{field}" field' - if nid not in self.supplied + self.match + self.optional: + if oid not in self.supplied + self.match + self.optional: return S_ERROR(f"{err} is not found for current CA.") - if nid in self.match and not self.dnInfoDictCA[field] == values: + if oid in self.match and not self.dnInfoDictCA[field] == values: return S_ERROR(f"{err} must be /{field}={('/%s=' % field).joing(self.dnInfoDictCA[field])}.") - if nid in self.maxDict: - rangeMax = list(range(min(len(values), len(self.maxDict[nid])))) - if any([True if len(values[i]) > self.maxDict[nid][i] else False for i in rangeMax]): - return S_ERROR(f"{err} values must be less then {', '.join(self.maxDict[nid])}.") - if nid in self.minDict: - rangeMin = list(range(min(len(values), len(self.minDict[nid])))) - if any([True if len(values[i]) < self.minDict[nid][i] else False for i in rangeMin]): - return S_ERROR(f"{err} values must be more then {', '.join(self.minDict[nid])}.") + if oid in self.maxDict: + rangeMax = list(range(min(len(values), len(self.maxDict[oid])))) + if any([True if len(values[i]) > self.maxDict[oid][i] else False for i in rangeMax]): + return S_ERROR(f"{err} values must be less then {', '.join(self.maxDict[oid])}.") + if oid in self.minDict: + rangeMin = list(range(min(len(values), len(self.minDict[oid])))) + if any([True if len(values[i]) < self.minDict[oid][i] else False for i in rangeMin]): + return S_ERROR(f"{err} values must be more then {', '.join(self.minDict[oid])}.") result = self.__fillX509Name(field, values) if not result["OK"]: @@ -196,10 +231,10 @@ def getProxy(self, userDN): :return: S_OK(str)/S_ERROR() -- contain a proxy string """ - self.__X509Name = X509.X509_Name() + self.__nameAttributes = [] result = self.checkStatus(userDN) if result["OK"]: - result = self.__createCertM2Crypto() + result = self.__createCertificate() if result["OK"]: certStr, keyStr = result["Value"] @@ -226,40 +261,40 @@ def generateDN(self, **kwargs): if kwargs.get("Email"): kwargs["emailAddress"] = [kwargs["Email"]] - self.__X509Name = X509.X509_Name() + self.__nameAttributes = [] self.log.info("Creating distinguished names chain") - for nid in self.supplied: - if nid not in [self.fields2nid[f] for f in self.dnList]: + for oid in self.supplied: + if oid not in [self.fields2oid[f] for f in self.dnList]: return S_ERROR( 'DNs order list does not contain supplied DN "%s"' - % self.nid2defField.get(nid, min(self.nid2fields[nid], key=len)) + % self.oid2defField.get(oid, min(self.oid2fields[oid], key=len)) ) for field in self.dnList: values = [] - nid = self.fields2nid[field] - if nid in self.match: - for field in self.nid2fields[nid]: + oid = self.fields2oid[field] + if oid in self.match: + for field in self.oid2fields[oid]: if field in self.dnInfoDictCA: values = self.dnInfoDictCA[field] if not values: return S_ERROR(f'Not found "{field}" match DN in CA') - for field in self.nid2fields[nid]: + for field in self.oid2fields[oid]: if kwargs.get(field): values = kwargs[field] if isinstance(kwargs[field], list) else [kwargs[field]] - if not values and nid in self.supplied: + if not values and oid in self.supplied: # Search default value - if nid not in self.nid2defField: - return S_ERROR(f'No values set for "{min(self.nid2fields[nid], key=len)}" DN') - values = self.parameters[nid] + if oid not in self.oid2defField: + return S_ERROR(f'No values set for "{min(self.oid2fields[oid], key=len)}" DN') + values = self.parameters[oid] result = self.__fillX509Name(field, values) if not result["OK"]: return result # WARN: This logic not support list of distribtes name elements - resDN = m2.x509_name_oneline(self.__X509Name.x509_name) # pylint: disable=no-member + resDN = asn1_utils.nameToDN(x509.Name(self.__nameAttributes)) result = self.checkStatus(resDN) if not result["OK"]: @@ -302,28 +337,28 @@ def __parseCACFG(self): self.parameters["KeyFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["private_key"] # Read distinguished names for k, v in self.cfg[self.cfg[self.cfg["ca"]["default_ca"]]["policy"]].items(): - nid = self.fields2nid[k] - self.parameters[nid], self.minDict[nid], self.maxDict[nid] = [], [], [] + oid = self.fields2oid[k] + self.parameters[oid], self.minDict[oid], self.maxDict[oid] = [], [], [] for k in [f"{i}.{k}" for i in range(0, 5)] + [k]: if k + "_default" in self.cfg["req"]["distinguished_name"]: - self.parameters[nid].append(self.cfg["req"]["distinguished_name"][k + "_default"]) + self.parameters[oid].append(self.cfg["req"]["distinguished_name"][k + "_default"]) if k + "_min" in self.cfg["req"]["distinguished_name"]: - self.minDict[nid].append(self.cfg["req"]["distinguished_name"][k + "_min"]) + self.minDict[oid].append(self.cfg["req"]["distinguished_name"][k + "_min"]) if k + "_max" in self.cfg["req"]["distinguished_name"]: - self.maxDict[nid].append(self.cfg["req"]["distinguished_name"][k + "_max"]) + self.maxDict[oid].append(self.cfg["req"]["distinguished_name"][k + "_max"]) if v == "supplied": - self.supplied.append(nid) + self.supplied.append(oid) elif v == "optional": - self.optional.append(nid) + self.optional.append(oid) elif v == "match": - self.match.append(nid) + self.match.append(oid) def __parseDN(self, dn): """Return DN fields :param str dn: DN - :return: list -- contain tuple with positionOfField.fieldName, fieldNID, fieldValue + :return: list -- contain tuple with positionOfField.fieldName, fieldOID, fieldValue """ dnInfoDict = collections.OrderedDict() for f, v in [f.split("=") for f in dn.lstrip("/").split("/")]: @@ -336,7 +371,7 @@ def __parseDN(self, dn): return S_OK(dnInfoDict) def __fillX509Name(self, field, values): - """Fill x509_Name object by M2Crypto + """Collect the DN attributes for the certificate subject :param str field: DN field name :param list values: values of field, order important @@ -344,63 +379,65 @@ def __fillX509Name(self, field, values): :return: S_OK()/S_ERROR() """ for value in values: - if ( - value - and m2.x509_name_set_by_nid( # pylint: disable=no-member - self.__X509Name.x509_name, self.fields2nid[field], value.encode() - ) - == 0 - ): - if ( - not self.__X509Name.add_entry_by_txt( - field=field, type=ASN1.MBSTRING_ASC, entry=value, len=-1, loc=-1, set=0 + if value: + try: + self.__nameAttributes.append( + x509.NameAttribute(x509.ObjectIdentifier(self.fields2oid[field]), value) ) - == 1 - ): - return S_ERROR(f'Cannot set "{field}" field.') + except (KeyError, TypeError, ValueError) as e: + return S_ERROR(f'Cannot set "{field}" field: {e!r}.') return S_OK() - def __createCertM2Crypto(self): + def __createCertificate(self): """Create new certificate for user - :return: S_OK(tuple)/S_ERROR() -- tuple contain certificate and pulic key as strings + :return: S_OK(tuple)/S_ERROR() -- tuple contain certificate and private key as strings """ - # Create public key - userPubKey = EVP.PKey() - userPubKey.assign_rsa(RSA.gen_key(self.bits, 65537, util.quiet_genparam_callback)) - # Create certificate - userCert = X509.X509() - userCert.set_pubkey(userPubKey) - userCert.set_version(2) - userCert.set_subject(self.__X509Name) - userCert.set_serial_number(secrets.randbits(64)) - # Add extentionals - userCert.add_ext(X509.new_extension("basicConstraints", "CA:" + str(False).upper())) - userCert.add_ext(X509.new_extension("extendedKeyUsage", "clientAuth", critical=1)) - # Set livetime - validityTime = datetime.timedelta(days=400) - notBefore = ASN1.ASN1_UTCTIME() - notBefore.set_time(int(time.time())) - notAfter = ASN1.ASN1_UTCTIME() - notAfter.set_time(int(time.time()) + int(validityTime.total_seconds())) - userCert.set_not_before(notBefore) - userCert.set_not_after(notAfter) - # Add subject from CA - with open(self.parameters["CertFile"]) as cf: - caCertStr = cf.read() - caCert = X509.load_cert_string(caCertStr) - userCert.set_issuer(caCert.get_subject()) - # Use CA key - with open(self.parameters["KeyFile"], "rb") as cf: - caKeyStr = cf.read() - pkey = EVP.PKey() - pkey.assign_rsa(RSA.load_key_string(caKeyStr, callback=util.no_passphrase_callback)) - # Sign - userCert.sign(pkey, self.algoritm) - - userCertStr = userCert.as_pem().decode("ascii") - userPubKeyStr = userPubKey.as_pem(cipher=None, callback=util.no_passphrase_callback).decode("ascii") - return S_OK((userCertStr, userPubKeyStr)) + if self.algoritm not in _ALGORITHMS: + return S_ERROR(f'Unsupported signing algorithm "{self.algoritm}"') + hashAlgo = _ALGORITHMS[self.algoritm]() + + # Create user key pair + userKey = rsa.generate_private_key(public_exponent=65537, key_size=self.bits) + + # Read CA certificate and key + try: + with open(self.parameters["CertFile"], "rb") as cf: + caCert = x509.load_pem_x509_certificate(cf.read()) + with open(self.parameters["KeyFile"], "rb") as cf: + caKey = serialization.load_pem_private_key(cf.read(), password=None) + except Exception as e: + return S_ERROR(f"Cannot load CA credentials: {e!r}") + + serial = 0 + while not serial: + serial = secrets.randbits(64) + + now = datetime.datetime.now(datetime.timezone.utc) + builder = ( + x509.CertificateBuilder() + .subject_name(x509.Name(self.__nameAttributes)) + .issuer_name(caCert.subject) + .public_key(userKey.public_key()) + .serial_number(serial) + .not_valid_before(now) + .not_valid_after(now + datetime.timedelta(days=400)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=False) + .add_extension(x509.ExtendedKeyUsage([x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH]), critical=True) + ) + + try: + userCert = builder.sign(caKey, hashAlgo) + except Exception as e: + return S_ERROR(f"Cannot sign the user certificate: {e!r}") + + userCertStr = userCert.public_bytes(serialization.Encoding.PEM).decode("ascii") + userKeyStr = userKey.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode("ascii") + return S_OK((userCertStr, userKeyStr)) def _forceGenerateProxyForDN(self, dn, time, group=None): """An additional helper method for creating a proxy without any substantial validation, @@ -413,7 +450,7 @@ def _forceGenerateProxyForDN(self, dn, time, group=None): :return: S_OK(tuple)/S_ERROR() -- contain proxy as chain and as string """ - self.__X509Name = X509.X509_Name() + self.__nameAttributes = [] result = self.__parseDN(dn) if not result["OK"]: return result @@ -424,7 +461,7 @@ def _forceGenerateProxyForDN(self, dn, time, group=None): if not result["OK"]: return result - result = self.__createCertM2Crypto() + result = self.__createCertificate() if result["OK"]: certStr, keyStr = result["Value"] chain = X509Chain() diff --git a/src/DIRAC/__init__.py b/src/DIRAC/__init__.py index a408647cc85..9282a0fc841 100755 --- a/src/DIRAC/__init__.py +++ b/src/DIRAC/__init__.py @@ -65,13 +65,6 @@ __path__ = extend_path(__path__, __name__) -# Set the environment variable such that openssl accepts proxy cert -# Sadly, this trick was removed in openssl >= 1.1.0 -# https://github.com/openssl/openssl/commit/8e21938ce3a5306df753eb40a20fe30d17cf4a68 -# Lets see if they would accept to put it back -# https://github.com/openssl/openssl/issues/8177 -os.environ["OPENSSL_ALLOW_PROXY_CERTS"] = "True" - # Now that's one hell of a hack :) # _strptime is not thread safe, resulting in obscure callstack # whenever you would have multiple threads and calling datetime.datetime.strptime diff --git a/tests/py3CheckDirs.txt b/tests/py3CheckDirs.txt index 1a3ad63d306..c98c282c290 100644 --- a/tests/py3CheckDirs.txt +++ b/tests/py3CheckDirs.txt @@ -9,7 +9,7 @@ src/DIRAC/Core/Base/Client.py src/DIRAC/Core/DISET/private/BaseClient.py src/DIRAC/Core/DISET/private/Service.py src/DIRAC/Core/DISET/RequestHandler.py -src/DIRAC/Core/Security/m2crypto/X509Chain.py +src/DIRAC/Core/Security/X509Chain.py src/DIRAC/Core/Tornado src/DIRAC/DataManagementSystem/Service/TornadoFileCatalogHandler.py src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py From ee97933d8ca30c1f53778f5cfe33b53ab5e5efc4 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 9 Jul 2026 06:34:11 +0200 Subject: [PATCH 2/7] refactor: do not re-derive peer credentials already returned by getCredentials X509Chain.getCredentials already resolves DN (identity for proxies, subject otherwise), isProxy and isLimitedProxy, so both the DISET getPeerInfo and the Tornado _authzSSL only need to attach the chain to the credential dict instead of re-deriving these fields. --- .../DISET/private/Transports/SSLTransport.py | 16 ++-------------- .../Tornado/Server/private/BaseRequestHandler.py | 16 ++-------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py index 1626abff7d6..54066d29d91 100755 --- a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py @@ -168,21 +168,9 @@ def getPeerInfo(sslSocket): raise RuntimeError(f"Failed to get SSL peer info ({creds['Message']}).") peer = creds["Value"] + # getCredentials already resolves DN (identity for proxies, subject + # otherwise), isProxy and isLimitedProxy peer["x509Chain"] = chain - isProxy = chain.isProxy() - if not isProxy["OK"]: - raise RuntimeError(f"Failed to get SSL peer isProxy ({isProxy['Message']}).") - peer["isProxy"] = isProxy["Value"] - - if peer["isProxy"]: - peer["DN"] = creds["Value"]["identity"] - else: - peer["DN"] = creds["Value"]["subject"] - - isLimited = chain.isLimitedProxy() - if not isLimited["OK"]: - raise RuntimeError(f"Failed to get SSL peer isLimitedProxy ({isLimited['Message']}).") - peer["isLimitedProxy"] = isLimited["Value"] return peer diff --git a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py index ae8043ec49a..b021b1c5514 100644 --- a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py +++ b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py @@ -766,21 +766,9 @@ def _authzSSL(self): credDict = res["Value"] + # getCredentials already resolves DN (identity for proxies, subject + # otherwise), isProxy and isLimitedProxy credDict["x509Chain"] = peerChain - res = peerChain.isProxy() - if not res["OK"]: - return res - credDict["isProxy"] = res["Value"] - - if credDict["isProxy"]: - credDict["DN"] = credDict["identity"] - else: - credDict["DN"] = credDict["subject"] - - res = peerChain.isLimitedProxy() - if not res["OK"]: - return res - credDict["isLimitedProxy"] = res["Value"] # We check if client sends extra credentials... if "extraCredentials" in self.request.arguments: From 5dacefe76249da7152028fb0d8d08c2be4ca6837 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 9 Jul 2026 06:35:50 +0200 Subject: [PATCH 3/7] perf: cache client-side SSL contexts between connections Building a client SSL context reads and parses the credential files for every RPC connection. Cache the contexts in a small LRU keyed on the credential file stat info (invalidated by proxy renewal), the CA location and the cipher configuration, with a TTL so changes in the CA directory are eventually picked up. Contexts are safe to share between connections and threads, as already done on the server side. --- .../DISET/private/Transports/SSLTransport.py | 80 ++++++++++++++++++- .../Transports/test/Test_SSLTransport.py | 21 +++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py index 54066d29d91..3e4d5ef8bd6 100755 --- a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py @@ -14,6 +14,9 @@ import socket import ssl import tempfile +import threading +import time +from collections import OrderedDict from DIRAC import gLogger from DIRAC.Core.DISET import DEFAULT_CONNECTION_TIMEOUT, DEFAULT_RPC_TIMEOUT @@ -141,6 +144,78 @@ def getSSLContext(**kwargs): return ctx +# Client-side SSL contexts are cached and shared between connections, as +# building one means reading and parsing the credential files for every RPC. +# An entry is invalidated when the credential file changes (e.g. proxy +# renewal), and expires after _CLIENT_CTX_CACHE_TTL seconds so that changes +# in the CA directory are eventually picked up. +_CLIENT_CTX_CACHE: OrderedDict = OrderedDict() +_CLIENT_CTX_CACHE_LOCK = threading.Lock() +_CLIENT_CTX_CACHE_MAX = 8 +_CLIENT_CTX_CACHE_TTL = 300 + + +def _clientContextCacheKey(kwargs): + """Compute the cache key for a client SSL context, or None if the + configuration is not cacheable. The credential selection mirrors + :py:func:`getSSLContext`, and the key contains the stat info of the + credential files so the cache is invalidated when they change. + + :param kwargs: the connection keywords, as given to getSSLContext + """ + if kwargs.get("bServerMode") or kwargs.get("proxyString"): + return None + if kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False): + credentialFiles = Locations.getHostCertificateAndKeyLocation() + if not credentialFiles: + return None + else: + proxyPath = kwargs.get("proxyLocation") or Locations.getProxyLocation() + if not proxyPath: + return None + credentialFiles = (proxyPath,) + + fileStats = [] + try: + for path in credentialFiles: + st = os.stat(path) + fileStats.append((path, st.st_mtime_ns, st.st_size, st.st_ino)) + except OSError: + return None + + skipCACheck = bool(kwargs.get("skipCACheck", False)) + caPath = None if skipCACheck else Locations.getCAsLocation() + ciphers = ( + kwargs.get("sslCiphers") or os.environ.get("DIRAC_SSL_CIPHERS") or os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") + ) + return (tuple(fileStats), skipCACheck, caPath, ciphers) + + +def _getClientSSLContext(**kwargs): + """Return a client SSL context, shared from the cache whenever possible + (ssl.SSLContext objects are safe to share between connections and threads) + """ + cacheKey = _clientContextCacheKey(kwargs) + if cacheKey is None: + return getSSLContext(**kwargs) + + now = time.monotonic() + with _CLIENT_CTX_CACHE_LOCK: + cached = _CLIENT_CTX_CACHE.get(cacheKey) + if cached and now - cached[0] < _CLIENT_CTX_CACHE_TTL: + _CLIENT_CTX_CACHE.move_to_end(cacheKey) + return cached[1] + + ctx = getSSLContext(**kwargs) + + with _CLIENT_CTX_CACHE_LOCK: + _CLIENT_CTX_CACHE[cacheKey] = (now, ctx) + _CLIENT_CTX_CACHE.move_to_end(cacheKey) + while len(_CLIENT_CTX_CACHE) > _CLIENT_CTX_CACHE_MAX: + _CLIENT_CTX_CACHE.popitem(last=False) + return ctx + + def getPeerInfo(sslSocket): """Gets the details of the current peer as a standard dict. The peer details are obtained from the supplied ssl.SSLSocket. @@ -204,7 +279,10 @@ def __init__(self, *args, **kwargs): self.__ctx = kwargs.pop("ctx", None) if not self.__ctx: - self.__ctx = getSSLContext(**kwargs) + if kwargs.get("bServerMode", False): + self.__ctx = getSSLContext(**kwargs) + else: + self.__ctx = _getClientSSLContext(**kwargs) # Note that kwargs is already kept in BaseTransport # as self.extraArgsDict, but at least I am sure that diff --git a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py index 98ee3ac045c..45494caf2cc 100644 --- a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py @@ -198,6 +198,27 @@ def test_simpleMessage(create_serverAndClient): assert serverAnswer == MAGIC_ANSWER +def test_clientContextCache(tmp_path): + """Client SSL contexts are shared until the credential file changes""" + from DIRAC.Core.DISET.private.Transports.SSLTransport import _CLIENT_CTX_CACHE, _getClientSSLContext + + proxyCopy = tmp_path / "proxy.pem" + proxyCopy.write_bytes(open(proxyFile, "rb").read()) + + _CLIENT_CTX_CACHE.clear() + ctx1 = _getClientSSLContext(proxyLocation=str(proxyCopy), skipCACheck=True) + ctx2 = _getClientSSLContext(proxyLocation=str(proxyCopy), skipCACheck=True) + assert ctx1 is ctx2 + + # Changing the credential file invalidates the cache entry + st = os.stat(proxyCopy) + os.utime(proxyCopy, ns=(st.st_atime_ns, st.st_mtime_ns + 10**9)) + ctx3 = _getClientSSLContext(proxyLocation=str(proxyCopy), skipCACheck=True) + assert ctx3 is not ctx1 + + _CLIENT_CTX_CACHE.clear() + + def test_getRemoteInfo(create_serverAndClient): """Check the information from remote peer""" serv, client = create_serverAndClient From 881506739eca4f5a87130c618a76251e096c728d Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 11 Jul 2026 00:15:06 +0200 Subject: [PATCH 4/7] feat: restore the M2Crypto based X509 and SSL implementation Reinstate, unchanged, the M2Crypto/pyasn1 based implementation that was removed by the migration to pyca/cryptography and the standard library ssl module: the DIRAC.Core.Security.m2crypto subpackage, the DISET M2SSLTransport/M2Utils modules and the default SSL cipher/method constants they rely on. The code is not used yet: a follow-up commit makes it selectable at runtime with DIRAC_USE_M2CRYPTO. --- src/DIRAC/Core/DISET/__init__.py | 18 + .../private/Transports/M2SSLTransport.py | 505 ++++++++ .../DISET/private/Transports/SSL/M2Utils.py | 219 ++++ src/DIRAC/Core/Security/m2crypto/X509CRL.py | 102 ++ .../Core/Security/m2crypto/X509Certificate.py | 500 ++++++++ src/DIRAC/Core/Security/m2crypto/X509Chain.py | 1028 +++++++++++++++++ .../Core/Security/m2crypto/X509Request.py | 203 ++++ src/DIRAC/Core/Security/m2crypto/__init__.py | 47 + .../Core/Security/m2crypto/asn1_utils.py | 368 ++++++ 9 files changed, 2990 insertions(+) create mode 100755 src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py create mode 100644 src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py create mode 100644 src/DIRAC/Core/Security/m2crypto/X509CRL.py create mode 100644 src/DIRAC/Core/Security/m2crypto/X509Certificate.py create mode 100644 src/DIRAC/Core/Security/m2crypto/X509Chain.py create mode 100644 src/DIRAC/Core/Security/m2crypto/X509Request.py create mode 100644 src/DIRAC/Core/Security/m2crypto/__init__.py create mode 100644 src/DIRAC/Core/Security/m2crypto/asn1_utils.py diff --git a/src/DIRAC/Core/DISET/__init__.py b/src/DIRAC/Core/DISET/__init__.py index 6aa018fca56..32fc83b15f1 100755 --- a/src/DIRAC/Core/DISET/__init__.py +++ b/src/DIRAC/Core/DISET/__init__.py @@ -4,3 +4,21 @@ DEFAULT_RPC_TIMEOUT = 600 #: Default timeout to establish a connection DEFAULT_CONNECTION_TIMEOUT = 10 + +# The following constants are only used by the M2Crypto based SSL +# implementation (DIRAC_USE_M2CRYPTO=Yes) + +#: Default SSL Ciher accepted. Current default is for pyGSI/M2crypto compatibility +#: Can be changed with DIRAC_M2CRYPTO_SSL_CIPHERS +#: Recommandation (incompatible with pyGSI) +# pylint: disable=line-too-long +#: AES256-GCM-SHA384:AES256-SHA256:AES128-GCM-SHA256:AES128-SHA256:HIGH:MEDIUM:RSA:!3DES:!RC4:!aNULL:!eNULL:!MD5:!SEED:!IDEA:!SHA # noqa +# Cipher line should be as readable as possible, sorry pylint +# pylint: disable=line-too-long +DEFAULT_SSL_CIPHERS = "AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:CAMELLIA256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:HIGH:MEDIUM:RSA:!3DES:!RC4:!aNULL:!eNULL:!MD5:!SEED:!IDEA" # noqa + +#: Default SSL methods accepted. Current default accepts TLSv1 for pyGSI/M2crypto compatibility +#: Can be changed with DIRAC_M2CRYPTO_SSL_METHODS +#: Recommandation (incompatible with pyGSI) +# TLSv2:TLSv3 +DEFAULT_SSL_METHODS = "TLSv1:TLSv2:TLSv3" diff --git a/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py new file mode 100755 index 00000000000..a7685e010d6 --- /dev/null +++ b/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python +""" +M2Crypto SSLTransport Library +""" +import os +import socket +from M2Crypto import SSL, threading as M2Threading +from M2Crypto.SSL.Checker import SSLVerificationError + +from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR +from DIRAC.Core.DISET.private.Transports.BaseTransport import BaseTransport +from DIRAC.Core.DISET.private.Transports.SSL.M2Utils import getM2SSLContext, getM2PeerInfo + +from DIRAC.Core.DISET import DEFAULT_CONNECTION_TIMEOUT, DEFAULT_RPC_TIMEOUT + +# TODO: For now we have to set an environment variable for proxy support in OpenSSL +# Eventually we may need to add API support for this to M2Crypto... +os.environ["OPENSSL_ALLOW_PROXY_CERTS"] = "1" +M2Threading.init() + +# TODO: CRL checking should be implemented but this will require support adding +# to M2Crypto: Quite a few functions will need mapping through from OpenSSL to +# allow the CRL stack to be set on the X509 CTX used for verification. + +# TODO: Log useful messages to the logger + + +class SSLTransport(BaseTransport): + """SSL Transport implementation using the M2Crypto library.""" + + # This name is the same as BaseClient, + # and is used a bit everywhere, so it should be factorized out + # eventually + KW_TIMEOUT = "timeout" + + def __getConnection(self): + """Helper function to get a connection object, + Tries IPv6 (AF_INET6) first, then falls back to IPv4 (AF_INET). + """ + try: + conn = SSL.Connection(self.__ctx, family=socket.AF_INET6) + except OSError: + # Maybe no IPv6 support? Try IPv4 only socket. + conn = SSL.Connection(self.__ctx, family=socket.AF_INET) + return conn + + def __init__(self, *args, **kwargs): + """Create an SSLTransport object, parameters are the same + as for other transports. If ctx is specified (as an instance of + SSL.Context) then use that rather than creating a new context. + + kwargs can contain all the parameters defined in BaseClient, + in particular timeout + """ + # The thread init of M2Crypto is not really thread safe. + # So we put it a second time + M2Threading.init() + self.remoteAddress = None + self.peerCredentials = {} + + # The timeout used here is different from what it was in pyGSI. + # It is to be understood here as the timeout for socket operations + # involved in the RPC call, but NOT the establishment of the connection, + # for which there is a different timeout. + # + # The timeout management of pyGSI was a bit off. + # This is proven by that type of trace (look at the timestamp): + # + # 2020-07-16 09:48:55 UTC dirac-proxy-init [140013698656064] DEBUG: Connection timeout set to: 1 + # 2020-07-16 09:58:55 UTC dirac-proxy-init [140013698656064] WARN: Issue getting socket: + # + + self.__timeout = kwargs.get(SSLTransport.KW_TIMEOUT, DEFAULT_RPC_TIMEOUT) + + self.__locked = False # We don't support locking, so this is always false. + + # If not specified in the arguments (never is in DIRAC code...) + # and we are setting up a server listing connection, set the accepted + # ssl methods and ciphers + if kwargs.get("bServerMode"): + if "sslMethods" not in kwargs: + kwargs["sslMethods"] = os.environ.get("DIRAC_M2CRYPTO_SSL_METHODS") + if "sslCiphers" not in kwargs: + kwargs["sslCiphers"] = os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") + + self.__ctx = kwargs.pop("ctx", None) + if not self.__ctx: + self.__ctx = getM2SSLContext(**kwargs) + + # Note that kwargs is already kept in BaseTransport + # as self.extraArgsDict, but at least I am sure that + # self.__kwargs will never be modified + self.__kwargs = kwargs + + BaseTransport.__init__(self, *args, **kwargs) + + def setSocketTimeout(self, timeout): + """Set the timeout for RPC calls. + + .. warning: This needs to be called before initAsClient. + It is used as a timeout for RPC calls, not connection. + + :param timeout: timeout for socket operation in seconds + + """ + self.__timeout = timeout + + def initAsClient(self): + """Prepare this client socket for use.""" + if self.serverMode(): + raise RuntimeError("SSLTransport is in server mode.") + + errors = [] + host, port = self.stServerAddress + + # The following piece of code was inspired by the python socket documentation + # as well as the implementation of M2Crypto.httpslib.HTTPSConnection + + # Get all available addresses (IPv6 and IPv4) and try them in order + try: + addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except OSError as e: + return S_ERROR(f"DNS lookup failed {e!r}") + for family, _socketType, _proto, _canonname, socketAddress in addrInfoList: + try: + self.oSocket = SSL.Connection(self.__ctx, family=family) + + # First set a short connection timeout, that will trigger + # during blocking operations (read/write) use to + # establish the SSL connection + self.oSocket.settimeout(DEFAULT_CONNECTION_TIMEOUT) + + # Enable keepAlive, with default options + # (see more comments about keepalive in :py:meth:`.acceptConnection`) + self.oSocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + + # set SNI server name since we know it at this point + self.oSocket.set_tlsext_host_name(host) + + # tell the connection which host we are connecting to so we can + # use the address we obtained from DNS + self.oSocket.set1_host(host) + self.oSocket.connect(socketAddress) + + # Once the connection is established, we can use the timeout + # asked for RPC + self.oSocket.settimeout(self.__timeout) + + self.remoteAddress = self.oSocket.getpeername() + + return S_OK() + # warning: do NOT catch SSL related error here + # They should be propagated upwards and caught by the BaseClient + # not to enter the retry loop + except OSError as e: + errors.append(f"{socketAddress} {e}:{repr(e)}") + + if self.oSocket is not None: + self.close() + + return S_ERROR("; ".join(errors)) + + def initAsServer(self): + """Prepare this server socket for use.""" + if not self.serverMode(): + raise RuntimeError("SSLTransport is in client mode.") + # Before getting the connection object, we need to set + # a server session ID in the context + host = self.stServerAddress[0] + port = self.stServerAddress[1] + self.__ctx.set_session_id_ctx((f"DIRAC-{host}-{port}").encode()) + self.oSocket = self.__getConnection() + # Make sure reuse address is set correctly + if self.bAllowReuseAddress: + param = 1 + else: + param = 0 + + self.oSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, param) + + self.oSocket.bind(self.stServerAddress) + self.oSocket.listen(self.iListenQueueSize) + return S_OK() + + def close(self): + # pylint: disable=line-too-long + """Close this socket.""" + + if self.oSocket: + # TL;DR: + # Do NOT touch that method + # + # Surprisingly (to me at least), M2Crypto does not close + # the underlying socket when calling SSL.Connection.close + # It only does it when the garbage collector kicks in (see ~M2Crypto.SSL.Connection.Connection.__del__) + # If the socket is not closed, the connection may hang forever. + # + # Thus, we are setting self.oSocket to None to allow the GC to do the work, but since we are not sure + # that it will run, we anyway force the connection to be closed + # + # However, we should close the underlying socket only after SSL was shutdown properly. + # This is because OpenSSL `ssl3_shutdown` (see callstack below) may still read some data + # (see https://github.com/openssl/openssl/blob/master/ssl/s3_lib.c#L4509):: + # + # + # 1 0x00007fffe9d48fc0 in sock_read () from /lib/libcrypto.so.1.0.0 + # 2 0x00007fffe9d46e83 in BIO_read () from /lib/libcrypto.so.1.0.0 + # 3 0x00007fffe9eab9dd in ssl3_read_n () from /lib/libssl.so.1.0.0 + # 4 0x00007fffe9ead216 in ssl3_read_bytes () from /lib/libssl.so.1.0.0 + # 5 0x00007fffe9ea999c in ssl3_shutdown () from /lib/libssl.so.1.0.0 + # 6 0x00007fffe9ed4f93 in ssl_free () from /lib/libssl.so.1.0.0 + # 7 0x00007fffe9d46d5b in BIO_free () from /lib/libcrypto.so.1.0.0 + # 8 0x00007fffe9f30a96 in bio_free (bio=0x5555556f3200) at SWIG/_m2crypto_wrap.c:5008 + # 9 0x00007fffe9f30b1e in _wrap_bio_free (self=, args=) at SWIG/_m2crypto_wrap.c + # + # We unfortunately have no way to force that order, and there is a risk of deadlock + # when running in a multi threaded environment like the agents:: + # + # Thread A opens socket, gets FD = 111 + # Thread A works on it + # Thread A closes FD 111 (underlying socket.close()) + # Thread B opens socket, gets FD = 111 + # Thread A calls read on FD=111 from ssl3_shutdown + # + # This is illustrated on the strace below:: + # + # 26461 14:25:15.266692 write(111]:42688->[]:9140]>, + # "blabla", 37 + # 26464 14:25:15.266857 <... connect resumed>) = 0 <0.000195> + # 26464 14:25:15.267023 getsockname(120:44252->188.185.84.86:9140]>, + # 26461 14:25:15.267176 <... write resumed>) = 37 <0.000453> + # 26464 14:25:15.267425 <... getsockname resumed>{sa_family=AF_INET, sin_port=htons(44252), + # sin_addr=inet_addr("")}, [28->16]) = 0 <0.000292> + # 26461 14:25:15.267466 close(111]:42688->[]:9140]> + # 26464 14:25:15.267637 close(120:44252->188.185.84.86:9140]> + # 26464 14:25:15.267738 <... close resumed>) = 0 <0.000086> + # 26461 14:25:15.267768 <... close resumed>) = 0 <0.000285> + # 26464 14:25:15.267827 socket(AF_INET6, SOCK_DGRAM|SOCK_CLOEXEC, IPPROTO_IP + # 26461 14:25:15.267888 futex(0x21f8620, FUTEX_WAKE_PRIVATE, 1 + # 26464 14:25:15.267976 <... socket resumed>) = 111 <0.000138> + # 26461 14:25:15.268092 <... futex resumed>) = 1 <0.000196> + # 26464 14:25:15.268195 connect(111, + # {sa_family=AF_INET6, sin6_port=htons(9140), + # inet_pton(AF_INET6, "", &sin6_addr), + # sin6_flowinfo=htonl(0), sin6_scope_id=0 + # }, 28 + # 26461 14:25:15.268294 read(111]:42480->[]:9140]>, + # 26464 14:25:15.268503 <... connect resumed>) = 0 <0.000217> + # 26464 14:25:15.268673 getsockname(111]:42480->[]:9140]>, + # 26464 14:25:15.268862 <... getsockname resumed>{sa_family=AF_INET6, sin6_port=htons(42480), + # inet_pton(AF_INET6, "", &sin6_addr), sin6_flowinfo=htonl(0), sin6_scope_id= + # 0}, [28]) = 0 <0.000168> + # 26464 14:25:15.269048 + # close(111]:42480->[]:9140]> + # + # + # + # Update 16.07.20: + # M2Crypto 0.36 contains the bug fix https://gitlab.com/m2crypto/m2crypto/-/merge_requests/247 + # that allows proper closing. So manual closing of the underlying socket should not be needed anymore + + # Update 16.07.20 + # I add this shutdown call without being 100% sure + # it solves some hanging connections issues, but it seems + # to work. it does not appear in any M2Crypto doc, but comparing + # some internals of M2Crypto and official python SSL library, + # it seems to make sense + self.oSocket.shutdown(socket.SHUT_RDWR) + + # Update 16.07.20 + # With freeBio=True, we force the + # closing of the socket before the GC runs + self.oSocket.close(freeBio=True) + # underlyingSocket = self.oSocket.socket + self.oSocket = None + # underlyingSocket.close() + return S_OK() + + def renewServerContext(self): + # pylint: disable=line-too-long + """Renews the server context. + This reloads the certificates and re-initialises the SSL context. + + NOTE: Chris 15.05.20 + I noticed python segfault on a regular time interval. The stack trace always looks like that:: + + #0 0x00007fdb5bbe2388 in ?? () from /opt/dirac/pro/diracos/usr/lib64/python2.7/lib-dynload/../../libcrypto.so.10 + #1 0x00007fdb5bbd8742 in X509_STORE_load_locations () from /opt/dirac/pro/diracos/usr/lib64/python2.7/lib-dynload/../../libcrypto.so.10 + #2 0x00007fdb57edcc9d in _wrap_ssl_ctx_load_verify_locations (self=, args=) at SWIG/_m2crypto_wrap.c:20602 + #3 0x00007fdb644ec484 in PyEval_EvalFrameEx () from /opt/dirac/versions/v10r0_1587978031/diracos/usr/bin/../lib64/libpython2.7.so.1.0 + + I could not find anything fundamentaly wrong, and the context renewal is the only place I could think of. + + GSI based SSLTransport did the following: renew the context, and renew the Connection object using the same raw socket + This still seems very fishy to me though, especially that the ServiceReactor still has the old object in self.__listeningConnections[svcName]['socket']] + + Here, we were are refreshing the CA store. What was missing was the call to the parent class, thus entering some sort of infinite loop. + The parent's call seems to have fixed it. + """ # noqa # pylint: disable=line-too-long + if not self.serverMode(): + raise RuntimeError("SSLTransport is in client mode.") + super().renewServerContext() + self.__ctx = getM2SSLContext(self.__ctx, **self.__kwargs) + + return S_OK() + + def handshake_singleStep(self): + """Used to perform SSL handshakes. + These are now done automatically. + """ + # This isn't used any more, the handshake is done inside the M2Crypto library + return S_OK() + + def handshake_multipleSteps(self): + """Perform SSL handshakes. + This has to be called after the connection was accepted (acceptConnection_multipleSteps) + + The remote credentials are gathered here + """ + try: + # M2Crypto does not provide public method to + # accept and handshake in two steps. + # So we have to do it manually + # The following lines are basically a copy/paste + # of the end of SSL.Connection.accept method + self.oSocket.setup_ssl() + self.oSocket.set_accept_state() + self.oSocket.accept_ssl() + check = getattr(self.oSocket, "postConnectionCheck", self.oSocket.serverPostConnectionCheck) + if check is not None: + if not check(self.oSocket.get_peer_cert(), self.oSocket.addr[0]): + raise SSL.Checker.SSLVerificationError("post connection check failed") + + self.peerCredentials = getM2PeerInfo(self.oSocket) + + # Now that the handshake has been performed on the server + # we can set the timeout for the RPC operations. + # In practice, since we are on the server side, the + # timeout we set here represents the timeout for receiving + # the arguments and sending back the response. This should + # in principle be reasonably quick, but just to be sure + # we can set it to the DEFAULT_RPC_TIMEOUT + self.oSocket.settimeout(DEFAULT_RPC_TIMEOUT) + + return S_OK() + except (OSError, SSL.SSLError, SSLVerificationError) as e: + return S_ERROR(f"Error in handhsake: {e} {repr(e)}") + + def setClientSocket_singleStep(self, oSocket): + """Set the inner socket (i.e. SSL.Connection object) of this instance + to the value of oSocket. + We also gather the remote peer credentials + This method is intended to be used to create client connection objects + from a server and should be considered to be an internal function. + + :param oSocket: client socket SSL.Connection object + + """ + + # TODO: The calling method (ServiceReactor.__acceptIncomingConnection) expects + # socket.error to be thrown in case of issue. Maybe we should catch the M2Crypto + # errors here and raise socket.error instead + + self.oSocket = oSocket + self.remoteAddress = self.oSocket.getpeername() + self.peerCredentials = getM2PeerInfo(self.oSocket) + + def setClientSocket_multipleSteps(self, oSocket): + """Set the inner socket (i.e. SSL.Connection object) of this instance + to the value of oSocket. + This method is intended to be used to create client connection objects + from a server and should be considered to be an internal function. + + :param oSocket: client socket SSL.Connection object + + """ + # warning: do NOT catch socket.error here, because for who knows what reason + # exceptions are actually properly used for once, and the calling method + # relies on it (ServiceReactor.__acceptIncomingConnection) + self.oSocket = oSocket + self.remoteAddress = self.oSocket.getpeername() + + def acceptConnection_multipleSteps(self): + """Accept a new client, returns a new SSLTransport object representing + the client connection. + + The connection is accepted, but no SSL handshake is performed + + :returns: S_OK(SSLTransport object) + """ + # M2Crypto does not provide public method to + # accept and handshake in two steps. + # So we have to do it manually + # The following lines are basically a copy/paste + # of the begining of SSL.Connection.accept method + # with added options and timeout + try: + sock, addr = self.oSocket.socket.accept() + oClient = SSL.Connection(self.oSocket.ctx, sock) + + # Set the keep alive to true. This keepalive will ensure that we + # detect remote peer crashing or network interruption + # Note that this is ineffective if we are in the middle of blocking + # operations. + oClient.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + + # I am adding here for reference the code that would allow to change + # the keepalive settings, although we should be fine with the default + # (connection would be closed after 7200 + 9 * 75 ~= 2h and 10mn ) + + # Duration between two keepalive probes, in seconds + # oClient.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 75) + # Number of consecutive bad probes to cut the connection + # oClient.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 9) + # Delay before the keepalive starts ticking, in seconds + # oClient.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 7200) + + # Here we set the timeout server side. + # We first set the connection timeout, which will + # be effective for TLS handshake + oClient.settimeout(DEFAULT_CONNECTION_TIMEOUT) + + oClient.addr = addr + oClientTrans = SSLTransport(self.stServerAddress, ctx=self.__ctx) + oClientTrans.setClientSocket(oClient) + return S_OK(oClientTrans) + except (OSError, SSL.SSLError, SSLVerificationError) as e: + return S_ERROR(f"Error in acceptConnection: {e} {repr(e)}") + + def acceptConnection_singleStep(self): + """Accept a new client, returns a new SSLTransport object representing + the client connection. + + The SSL handshake is performed here. + + :returns: S_OK(SSLTransport object) + """ + try: + oClient, _ = self.oSocket.accept() + oClientTrans = SSLTransport(self.stServerAddress, ctx=self.__ctx) + oClientTrans.setClientSocket(oClient) + return S_OK(oClientTrans) + except (OSError, SSL.SSLError, SSLVerificationError) as e: + return S_ERROR(f"Error in acceptConnection: {e} {repr(e)}") + + # Depending on the DIRAC_M2CRYPTO_SPLIT_HANDSHAKE we either do the + # handshake separately or not + if os.getenv("DIRAC_M2CRYPTO_SPLIT_HANDSHAKE", "Yes").lower() in ("yes", "true"): + acceptConnection = acceptConnection_multipleSteps + handshake = handshake_multipleSteps + setClientSocket = setClientSocket_multipleSteps + else: + acceptConnection = acceptConnection_singleStep + handshake = handshake_singleStep + setClientSocket = setClientSocket_singleStep + + def _read(self, bufSize=4096, skipReadyCheck=False): + """Read bufSize bytes from the buffer. + + :param bufSize: size of the buffer to read + :param skipReadyCheck: ignored. + + + :returns: S_OK(number of byte read) + """ + try: + read = self.oSocket.read(bufSize) + return S_OK(read) + except (OSError, SSL.SSLError, SSLVerificationError) as e: + return S_ERROR(f"Error in _read: {e} {repr(e)}") + + def isLocked(self): + """Returns if this instance is locked. + Always returns false. + + :returns: False + """ + return self.__locked + + def _write(self, buf): + """Write all bytes contained within iterable "buf" to the + connected peer. + + :param buf: iterable buffer + + :returns: S_OK(number of bytes written) + """ + try: + # If the client application has abruptly terminated + # the connection will be in CLOSE_WAIT on the server side. + # And when the server side will anyway try to send back its data. + # The first call to this _write method will succeed, + # and we will never know that the connection was broken. + # However, the client would answer with an RST packet. + # And writting on a socket that received an RST packet + # triggers a SIGPIPE. + # In practice, this means that if the server replies to a + # dead client with less that 16384 bytes + # (see https://datatracker.ietf.org/doc/html/rfc8446#section-5.1), + # we will never notice that we sent the answer to the vacuum. + # And don't look for a fix, there just isn't. + wrote = self.oSocket.write(buf) + return S_OK(wrote) + except (OSError, SSL.SSLError, SSLVerificationError) as e: + return S_ERROR(f"Error in _write: {e} {repr(e)}") diff --git a/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py b/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py new file mode 100644 index 00000000000..79d4b1da003 --- /dev/null +++ b/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +""" +Utilities for using M2Crypto SSL with DIRAC. +""" +import os +import tempfile +import M2Crypto +from packaging.version import Version +from M2Crypto import SSL, m2, X509 + + +from DIRAC.Core.DISET import DEFAULT_SSL_CIPHERS, DEFAULT_SSL_METHODS +from DIRAC.Core.Security import Locations +from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain + +# Verify depth of peer certs +VERIFY_DEPTH = 50 +DEBUG_M2CRYPTO = os.getenv("DIRAC_DEBUG_M2CRYPTO", "No").lower() in ("yes", "true") + + +VERIFY_ALLOW_PROXY_CERTS = 0 + +# If the version of M2Crypto is recent enough, there is an API +# to accept proxy certificate, and we do not need to rely on +# OPENSSL_ALLOW_PROXY_CERT environment variable +# which was removed as of openssl 1.1 +# We need this to be merged in M2Crypto: https://gitlab.com/m2crypto/m2crypto/merge_requests/236 +# We set the proper verify flag to the X509Store of the context +# as described here https://www.openssl.org/docs/man1.1.1/man7/proxy-certificates.html +if hasattr(SSL, "verify_allow_proxy_certs"): + VERIFY_ALLOW_PROXY_CERTS = SSL.verify_allow_proxy_certs # pylint: disable=no-member +# As of M2Crypto 0.37, the `verify_allow_proxy_certs` flag was moved +# to X509 (https://gitlab.com/m2crypto/m2crypto/-/merge_requests/238) +# It is more consistent with all the other flags, +# but pySSL had it in SSL. Well... +elif hasattr(X509, "verify_allow_proxy_certs"): + VERIFY_ALLOW_PROXY_CERTS = X509.verify_allow_proxy_certs # pylint: disable=no-member +# As of M2Crypto 0.38, M2Crypto did not export the symbol correctly +# Anymore +# https://gitlab.com/m2crypto/m2crypto/-/issues/298 +elif Version(M2Crypto.__version__) >= Version("0.38.0"): + VERIFY_ALLOW_PROXY_CERTS = 64 + + +def __loadM2SSLCTXHostcert(ctx): + """Load hostcert & key from the default location and set them as the + credentials for SSL context ctx. + Returns None. + """ + certKeyTuple = Locations.getHostCertificateAndKeyLocation() + if not certKeyTuple: + raise RuntimeError("Hostcert/key location not set") + hostcert, hostkey = certKeyTuple + if not os.path.isfile(hostcert): + raise RuntimeError(f"Hostcert file ({hostcert}) is missing") + if not os.path.isfile(hostkey): + raise RuntimeError(f"Hostkey file ({hostkey}) is missing") + # Make sure we never stall on a password prompt if the hostkey has a password + # by specifying a blank string. + ctx.load_cert(hostcert, hostkey, callback=lambda: "") + + +def __loadM2SSLCTXProxy(ctx, proxyPath=None): + """Load proxy from proxyPath (or default location if not specified) and + set it as the certificate & key to use for this SSL context. + Returns None. + """ + if not proxyPath: + proxyPath = Locations.getProxyLocation() + if not proxyPath: + raise RuntimeError("Proxy location not set") + if not os.path.isfile(proxyPath): + raise RuntimeError(f"Proxy file ({proxyPath}) is missing") + # See __loadM2SSLCTXHostcert for description of why lambda is needed. + ctx.load_cert_chain(proxyPath, proxyPath, callback=lambda: "") + + +def ssl_verify_callback_print_error(ok, store): + """This callback method does nothing but printing the error. + It prints a few more useful info than the exception + + :param ok: current validation status + :param store: pointer to the X509_CONTEXT_STORE + """ + errnum = store.get_error() + if errnum: + print(f"SSL DEBUG ERRNUM {errnum} ERRMSG {m2.x509_get_verify_error(errnum)}") # pylint: disable=no-member + return ok + + +def getM2SSLContext(ctx=None, **kwargs): + """Gets an M2Crypto.SSL.Context configured using the standard + DIRAC connection keywords from kwargs. The keywords are: + + - clientMode: Boolean, if False hostcerts are always used. If True + a proxy is used unless other flags are set. + - useCertificates: Boolean, Set to true to use hostcerts in client + mode. + - proxyString: String, allow a literal proxy string to be provided. + - proxyLocation: String, Path to file to use as proxy, defaults to + usual location(s) if not set. + - skipCACheck: Boolean, if True, don't verify peer certificates. + - sslMethods: String, List of SSL algorithms to enable in OpenSSL style + cipher format, e.g. "SSLv3:TLSv1". + - sslCiphers: String, OpenSSL style cipher string of ciphers to allow + on this connection. + + If an existing context "ctx" is provided, it is just reconfigured with + the selected arguments. + + Returns the new or updated context. + """ + if not ctx: + ctx = SSL.Context() + # Set certificates for connection + # CHRIS: I think clientMode was just an internal of pyGSI implementation + # if kwargs.get('clientMode', False) and not kwargs.get('useCertificates', False): + # if not kwargs.get('useCertificates', False): + if kwargs.get("bServerMode", False) or ( + kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False) + ): + # Server mode always uses hostcert + __loadM2SSLCTXHostcert(ctx) + + else: + # Client mode has a choice of possible options + if kwargs.get("proxyString", None): + # M2Crypto cannot take an inmemory location or a string, so + # so write it to a temp file and use proxyLocation + with tempfile.NamedTemporaryFile(mode="w") as tmpFile: + tmpFilePath = tmpFile.name + tmpFile.write(kwargs["proxyString"]) + # Flush, otherwise the file is empty in the subsequent call + tmpFile.flush() + __loadM2SSLCTXProxy(ctx, proxyPath=tmpFilePath) + else: + # Use normal proxy + __loadM2SSLCTXProxy(ctx, proxyPath=kwargs.get("proxyLocation", None)) + + verify_callback = ssl_verify_callback_print_error if DEBUG_M2CRYPTO else None + + # Set peer verification + if kwargs.get("skipCACheck", False): + # Don't validate peer, but still request creds + ctx.set_verify(SSL.verify_none, VERIFY_DEPTH, callback=verify_callback) + else: + # Do validate peer + ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, VERIFY_DEPTH, callback=verify_callback) + # Set CA location + caPath = Locations.getCAsLocation() + if not caPath: + raise RuntimeError("Failed to find CA location") + if not os.path.isdir(caPath): + raise RuntimeError(f"CA path ({caPath}) is not a valid directory") + ctx.load_verify_locations(capath=caPath) + + # Allow proxy certificates to be used + if VERIFY_ALLOW_PROXY_CERTS: + ctx.get_cert_store().set_flags(VERIFY_ALLOW_PROXY_CERTS) + + # Other parameters + sslMethods = kwargs.get("sslMethods", DEFAULT_SSL_METHODS) + if sslMethods: + # Pylint can't see the m2 constants due to the way the library is loaded + # We just have to disable that warning for the next bit... + # pylint: disable=no-member + methods = [("SSLv2", m2.SSL_OP_NO_SSLv2), ("SSLv3", m2.SSL_OP_NO_SSLv3), ("TLSv1", m2.SSL_OP_NO_TLSv1)] + allowed_methods = sslMethods.split(":") + # If a method isn't explicitly allowed, set the flag to disable it... + for method, method_flag in methods: + if method not in allowed_methods: + ctx.set_options(method_flag) + # SSL_OP_NO_SSLv2, SSL_OP_NO_SSLv3, SSL_OP_NO_TLSv1 + ciphers = kwargs.get("sslCiphers", DEFAULT_SSL_CIPHERS) + ctx.set_cipher_list(ciphers) + + # log the debug messages + if DEBUG_M2CRYPTO: + ctx.set_info_callback() + + return ctx + + +def getM2PeerInfo(conn): + """Gets the details of the current peer as a standard dict. The peer + details are obtained from the supplied M2 SSL Connection obj "conn". + The details returned are those from ~X509Chain.getCredentials, without Registry info: + + DN - Full peer DN as string + x509Chain - Full chain of peer + isProxy - Boolean, True if chain ends with proxy + isLimitedProxy - Boolean, True if chain ends with limited proxy + group - String, DIRAC group for this peer, if known + + Returns a dict of details. + """ + chain = X509Chain.generateX509ChainFromSSLConnection(conn) + creds = chain.getCredentials(withRegistryInfo=False) + if not creds["OK"]: + raise RuntimeError(f"Failed to get SSL peer info ({creds['Message']}).") + peer = creds["Value"] + + peer["x509Chain"] = chain + isProxy = chain.isProxy() + if not isProxy["OK"]: + raise RuntimeError(f"Failed to get SSL peer isProxy ({isProxy['Message']}).") + peer["isProxy"] = isProxy["Value"] + + if peer["isProxy"]: + peer["DN"] = creds["Value"]["identity"] + else: + peer["DN"] = creds["Value"]["subject"] + + isLimited = chain.isLimitedProxy() + if not isLimited["OK"]: + raise RuntimeError(f"Failed to get SSL peer isProxy ({isLimited['Message']}).") + peer["isLimitedProxy"] = isLimited["Value"] + + return peer diff --git a/src/DIRAC/Core/Security/m2crypto/X509CRL.py b/src/DIRAC/Core/Security/m2crypto/X509CRL.py new file mode 100644 index 00000000000..f942c2c41a6 --- /dev/null +++ b/src/DIRAC/Core/Security/m2crypto/X509CRL.py @@ -0,0 +1,102 @@ +""" X509CRL is a class for managing X509CRL +This class is used to manage the revoked certificates.... +""" +import re +import datetime + +import M2Crypto.X509 +from DIRAC import S_OK, S_ERROR +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.File import secureOpenForWrite + +# pylint: disable=broad-except + + +class X509CRL: + def __init__(self, cert=None): + self.__pemData = "" + + if cert: + self.__loadedCert = True + self.__revokedCert = cert + else: + self.__loadedCert = False + + @classmethod + def instanceFromFile(cls, crlLocation): + """Instance a X509CRL from a file""" + crl = cls() + result = crl.loadCRLFromFile(crlLocation) + if not result["OK"]: + return result + return S_OK(crl) + + def loadCRLFromFile(self, crlLocation): + """ + Load a x509CRL certificate from a pem file + Return : S_OK / S_ERROR + """ + self.__loadedCert = False + try: + self.__revokedCert = M2Crypto.X509.load_crl(crlLocation) + except Exception as e: + return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')}") + self.__loadedCert = True + with open(crlLocation) as crlFile: + pemData = crlFile.read() + self.__pemData = pemData + return S_OK() + + def __bytes__(self): + if not self.__loadedCert: + return b"No certificate loaded" + return self.__pemData.encode("ascii") + + def __str__(self): + return self.__pemData + + def dumpAllToString(self): + """ + Dump all to string + """ + if not self.__loadedCert: + return S_ERROR(DErrno.ECERTREAD, "No certificate loaded") + return S_OK(self.__pemData) + + def dumpAllToFile(self, filename=False): + """ + Dump all to file. If no filename specified a temporal one will be created + """ + if not self.__loadedCert: + return S_ERROR("No certificate loaded") + try: + with secureOpenForWrite(filename) as (fd, filename): + fd.write(self.__pemData) + except Exception as e: + return S_ERROR(DErrno.EWF, f"{filename}: {repr(e).replace(',)', ')')}") + return S_OK(filename) + + def hasExpired(self): + if not self.__loadedCert: + return S_ERROR("No certificate loaded") + # XXX It should be done better, for now M2Crypto doesn't offer access to fields like Next Update + txt = self.__revokedCert.as_text() + pattern = r"Next Update: (?P.*)\n" + dateStr = re.search(pattern, txt).group("nextUpdate") + nextUpdate = datetime.datetime.strptime(dateStr, "%b %d %H:%M:%S %Y GMT") + return S_OK(datetime.datetime.now() > nextUpdate) + + def getIssuer(self): + if not self.__loadedCert: + return S_ERROR("No certificate loaded") + # XXX It should be done better, for now M2Crypto doesn't offer access to fields like Issuer + txt = self.__revokedCert.as_text() + pattern = r"Issuer: (?P.*)\n" + return S_OK(re.search(pattern, txt).group("issuer")) + + def __repr__(self): + repStr = " X509Request -> X509Chain -> X509Certificate + from DIRAC.Core.Security.m2crypto.X509Request import X509Request + + req = X509Request() + req.generateProxyRequest(bitStrength=bitStrength, limited=limited) + + return S_OK(req) + + @executeOnlyIfCertLoaded + def getRemainingSecs(self): + """ + Get remaining lifetime in secs + + :returns: S_OK(remaining seconds) + """ + notAfter = self.getNotAfterDate()["Value"] + now = datetime.datetime.utcnow() + remainingSeconds = max(0, int((notAfter - now).total_seconds())) + + return S_OK(remainingSeconds) + + @executeOnlyIfCertLoaded + def getExtensions(self): + """ + Get a decoded list of extensions + + :returns: S_OK( list of tuple (extensionName, extensionValue)) + """ + extList = [] + for i in range(self.__certObj.get_ext_count()): + sn = self.__certObj.get_ext_at(i).get_name() + try: + value = self.__certObj.get_ext_at(i).get_value() + except Exception: + value = "Cannot decode value" + extList.append((sn, value)) + + return S_OK(sorted(extList)) + + @executeOnlyIfCertLoaded + def verify(self, pkey): + """ + Verify the signature of the certificate using the public key provided + + :param pkey: ~M2Crypto.EVP.PKey object + + :returns: S_OK(bool) where the boolean shows the success of the verification + """ + ret = self.__certObj.verify(pkey) + return S_OK(ret == 1) + + @executeOnlyIfCertLoaded + def asPem(self): + """ + Return certificate as PEM string + + :returns: pem string + """ + return self.__certObj.as_pem().decode("ascii") + + @executeOnlyIfCertLoaded + def getExtension(self, name): + """ + Return X509 Extension with given name + + :param name: name of the extension + + :returns: S_OK with M2Crypto.X509.X509_Extension object, or S_ERROR + """ + try: + ext = self.__certObj.get_ext(name) + except LookupError as e: + return S_ERROR(e) + return S_OK(ext) diff --git a/src/DIRAC/Core/Security/m2crypto/X509Chain.py b/src/DIRAC/Core/Security/m2crypto/X509Chain.py new file mode 100644 index 00000000000..1ebe73be38b --- /dev/null +++ b/src/DIRAC/Core/Security/m2crypto/X509Chain.py @@ -0,0 +1,1028 @@ +""" X509Chain is a class for managing X509 chains with their Pkeys + +Link to the RFC 3820: https://tools.ietf.org/html/rfc3820 +In particular, limited proxy: https://tools.ietf.org/html/rfc3820#section-3.8 + +""" +from __future__ import annotations + +import copy +import hashlib +import os +import re +from collections import OrderedDict + +import M2Crypto.X509 + +from DIRAC import S_ERROR, S_OK +from DIRAC.ConfigurationSystem.Client.Helpers import Registry +from DIRAC.Core.Security.m2crypto import DEFAULT_PROXY_STRENGTH, DIRAC_GROUP_OID, LIMITED_PROXY_OID, PROXY_OID +from DIRAC.Core.Security.m2crypto.X509Certificate import X509Certificate +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.Decorators import executeOnlyIf +from DIRAC.Core.Utilities.File import secureOpenForWrite + +# Decorator to check that _certList is not empty +needCertList = executeOnlyIf("_certList", S_ERROR(DErrno.ENOCHAIN)) +# Decorator to check that the PKey has been loaded +needPKey = executeOnlyIf("_keyObj", S_ERROR(DErrno.ENOPKEY)) + +# Cache of parsed proxy files keyed by (path, mtime_ns, size, inode). +_PROXY_LOAD_CACHE: OrderedDict[tuple, dict] = OrderedDict() +_PROXY_LOAD_CACHE_MAX = 8 + + +class X509Chain: + """ + An X509Chain is basically a list of X509Certificate object, as well as a PKey object, + which is associated to the X509Certificate the lowest in the chain. + + This is what you will want to use for user certificate (because they will turn into proxy....), and for + proxy. + + A priori, once we get rid of pyGSI, we could even meld the X509Certificate into this one, and use the X509Chain + for host certificates. After all, a certificate is nothing but a chain of length 1... + + There are normally 4 ways you would instanciate an X509Chain object: + + * You are loading a proxy from a file + * Loading the chain from a file + * You are getting information about your peer during an SSL connection + * You are delegating + + Typical usages of X509Chain are illustrated below + + Loading a proxy from a file (this will load the chain and the key, assuming the key is in the same file):: + + proxy = X509Chain() + res = proxy.loadProxyFromFile(myFile) + if not res['OK']: + return res + + + Generating a proxy from a Certificate:: + + cert = X509Chain() + # Load user cert + retVal = cert.loadChainFromFile('/home/chaen/.globus/userkey.pem') + if not retVal['OK']: + return retVal + # Load the key from a different place, with a password + retVal = cert.loadKeyFromFile('/home/chaen/.globus/userkey.pem', password='MySecretKey') + if not retVal['OK']: + return res + + # Generate a limited proxy, valid one hour + retVal = cert.generateProxyToFile('/tmp/proxy.pem', + 3600, # only 1 h + diracGroup = 'lhcb_user', + strength= 2048, + limited=True) + + + Getting information from a peer in an SSL Connection:: + + # conn is an M2Crypto.SSL.Connection instance + chain = X509Chain.generateX509ChainFromSSLConnection(conn) + creds = chain.getCredentials() + + + Delegating a proxy to a service:: + + # The server side generates a request + # Equivalent to ProxyManager.requestDelegationUpload + + x509Req = X509Request() + x509Req.generateProxyRequest() + + # This reqStr object is sent to the client + reqStr = x509Req.dumpRequest()['Value'] + + # This object contains both the public and private key + pkeyReq = x509Req.getPKey() + + ####################################################### + + # The client side signs the request, with its proxy + # Assume the proxy chain was already loaded one way or the otjer + + # The proxy will not contain a private key + res = proxyChain.generateChainFromRequestString(reqStr, lifetime=lifetime) + + # This is sent back to the server + delegatedProxyString = res['Value'] + + ###################################################### + # Equivalent to ProxyManager.completeDelegationUpload + + # Create the new chain + # the pkey was generated together with the Request + delegatedProxy = X509Chain(keyObj=pkeyReq) + delegatedProxy.loadChainFromString(delegatedProxyString) + + # make sure the public key match between Request and the new Chain + # (Stupid, of course it will ! But it is done in the ProxyManager...) + res = x509Req.checkChain(delegatedProxy) + + """ + + def __init__(self, certList=False, keyObj=False): + """ + C'tor + + :param certList: list of X509Certificate to constitute the chain + :param keyObj: ~M2Crypto.EVP.PKey object. The public or public/private key associated to + the last certificate of the chain + + """ + + # __isProxy is True if this chain represents a proxy + self.__isProxy = False + # Whether the proxy is limited or not + self.__isLimitedProxy = False + + # This is the position of the first proxy in the chain + self.__firstProxyStep = 0 + + # Cache for sha256 hash of the object + # This is just used as a unique identifier for + # indexing in the ProxyCache + self.__hash = False + + # List of X509Certificate constituing the chain + # The certificate in position N has been generated from the (N+1) + self._certList = [] + + # Place holder for the EVP.PKey object + self._keyObj = None + + if certList: + # copy the content of the list, without copying the objects themselves + self._certList = copy.copy(certList) + # Immediately check if it is a proxy + self.__checkProxyness() + + if keyObj: + self._keyObj = keyObj + + @staticmethod + def generateX509ChainFromSSLConnection(sslConnection): + """Returns an instance of X509Chain from the SSL connection + + :param sslConnection: ~M2Crypto.SSl.Connection instance + + :returns: a X509Chain instance + """ + certList = [] + + certStack = sslConnection.get_peer_cert_chain() + for cert in certStack: + certList.append(X509Certificate(x509Obj=cert)) + + # Servers don't receive the whole chain, the last cert comes alone + # if not self.infoDict['clientMode']: + certList.insert(0, X509Certificate(x509Obj=sslConnection.get_peer_cert())) + peerChain = X509Chain(certList=certList) + + return peerChain + + def loadChainFromFile(self, chainLocation): + """ + Load a x509 chain from a pem file + + :param chainLocation: path to the file + + :returns: S_OK/S_ERROR + """ + try: + with open(chainLocation) as fd: + pemData = fd.read() + except OSError as e: + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") + return self.loadChainFromString(pemData) + + def loadChainFromString(self, data): + """ + Load a x509 cert from a string containing the pem data + + :param data: data representing the chain of certificate in the + + Return : S_OK / S_ERROR + """ + try: + self._certList = self.__certListFromPemString(data) + except Exception as e: + return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')}") + + if not self._certList: + return S_ERROR(DErrno.EX509) + + # Update internals + self.__checkProxyness() + return S_OK() + + @staticmethod + def __certListFromPemString(certString): + """ + Create certificates list from string. String should contain certificates, just like plain text proxy file. + """ + # To get list of X509 certificates (not X509 Certificate Chain) from string it has to be parsed like that + # (constructors are not able to deal with big string) + certList = [] + pattern = r"(-----BEGIN CERTIFICATE-----((.|\n)*?)-----END CERTIFICATE-----)" + for cert in re.findall(pattern, certString): + certList.append(X509Certificate(certString=cert[0])) + return certList + + # Not used in m2crypto version + # def setChain(self, certList): + # """ + # Set the chain + # Return : S_OK / S_ERROR + # """ + # self._certList = certList + # self.__loadedChain = True + # return S_OK() + + def loadKeyFromFile(self, chainLocation, password=False): + """ + Load a PKey from a pem file + + :param chainLocation: path to the file + :param password: password to decode the file. + + :returns: S_OK / S_ERROR + """ + try: + with open(chainLocation) as fd: + pemData = fd.read() + except Exception as e: + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") + return self.loadKeyFromString(pemData, password) + + def loadKeyFromString(self, pemData, password=False): + """ + Load a PKey from a string containing the pem data + + :param pemData: pem data of the key, potentially encoded with the password + :param password: password to decode the file. + + :returns: S_OK / S_ERROR + """ + self._keyObj = None + if not isinstance(pemData, bytes): + pemData = pemData.encode("ascii") + if password: + password = password.encode() + try: + self._keyObj = M2Crypto.EVP.load_key_string(pemData, lambda x: password) + except Exception as e: + return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')} (Probably bad pass phrase?)") + + return S_OK() + + def setPKey(self, pkeyObj): + """ + Set the chain + Return : S_OK / S_ERROR + """ + self._keyObj = pkeyObj + return S_OK() + + def loadProxyFromFile(self, chainLocation): + """ + Load a Proxy from a pem file, that is both the Cert chain and the PKey + + :param chainLocation: path to the proxy file + + :returns: S_OK / S_ERROR + """ + try: + st = os.stat(chainLocation) + except OSError as e: + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") + + cacheKey = (chainLocation, st.st_mtime_ns, st.st_size, st.st_ino) + cached = _PROXY_LOAD_CACHE.get(cacheKey) + if cached is not None: + _PROXY_LOAD_CACHE.move_to_end(cacheKey) + # Shallow copy so callers mutating self._certList don't poison the cache + self._certList = list(cached["certList"]) + self._keyObj = cached["keyObj"] + self.__isProxy = cached["isProxy"] + self.__isLimitedProxy = cached["isLimitedProxy"] + self.__firstProxyStep = cached["firstProxyStep"] + self.__hash = False + return S_OK() + + try: + with open(chainLocation) as fd: + pemData = fd.read() + except Exception as e: + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") + result = self.loadProxyFromString(pemData) + if result["OK"]: + _PROXY_LOAD_CACHE[cacheKey] = { + "certList": list(self._certList), + "keyObj": self._keyObj, + "isProxy": self.__isProxy, + "isLimitedProxy": self.__isLimitedProxy, + "firstProxyStep": self.__firstProxyStep, + } + while len(_PROXY_LOAD_CACHE) > _PROXY_LOAD_CACHE_MAX: + _PROXY_LOAD_CACHE.popitem(last=False) + return result + + def loadProxyFromString(self, pemData): + """ + Load a Proxy from a pem buffer, that is both the Cert chain and the PKey + + :param pemData: PEM encoded cert chain and pkey + + :returns: S_OK / S_ERROR + """ + retVal = self.loadChainFromString(pemData) + if not retVal["OK"]: + return retVal + + return self.loadKeyFromString(pemData) + + @staticmethod + def __getProxyExtensionList(diracGroup=False, rfcLimited=False): + """ + Get an extension stack containing the necessary extension for a proxy. + Basically the keyUsage, the proxyCertInfo, and eventually the diracGroup + + :param diracGroup: name of the dirac group for the proxy + :param rfcLimited: boolean to generate for a limited proxy + + :returns: M2Crypto.X509.X509_Extension_Stack object. + """ + + extStack = M2Crypto.X509.X509_Extension_Stack() + + # Standard certificate extensions + kUext = M2Crypto.X509.new_extension( + "keyUsage", "digitalSignature, keyEncipherment, dataEncipherment", critical=1 + ) + extStack.push(kUext) + + # Mandatory extension to be a proxy + policyOID = LIMITED_PROXY_OID if rfcLimited else PROXY_OID + ext = M2Crypto.X509.new_extension("proxyCertInfo", f"critical, language:{policyOID}", critical=1) + extStack.push(ext) + + # Add a dirac group + if diracGroup and isinstance(diracGroup, str): + # the str cast is needed because M2Crypto does not play it cool with unicode here it seems + # Also one needs to specify the ASN1 type. That's what it is... + dGext = M2Crypto.X509.new_extension(DIRAC_GROUP_OID, str(f"ASN1:IA5:{diracGroup}")) + extStack.push(dGext) + + return extStack + + @needCertList + def getCertInChain(self, certPos=0): + """ + Get then a certificate in the chain + + :warning: Contrary to the pygsi version, this is not a copy! + + :param certPos: position of the certificate in the chain. Default: 0 + + :returns: S_OK(X509Certificate)/S_ERROR + """ + return S_OK(self._certList[certPos]) + + @needCertList + def getIssuerCert(self): + """ + Returns the issuer certificate of the last one if it is a proxy, otherwise + the last one in the chain + + :returns: S_OK(X509Certificate)/S_ERROR + """ + if self.__isProxy: + return S_OK(self._certList[self.__firstProxyStep + 1]) + return S_OK(self._certList[-1]) + + @needCertList + def getNumCertsInChain(self): + """ + length of the certificate chain + + :returns: length of the certificate chain + + + """ + return S_OK(len(self._certList)) + + # pylint: disable=unused-argument + @needCertList + @needPKey + def generateProxyToString( + self, lifetime, diracGroup=False, strength=DEFAULT_PROXY_STRENGTH, limited=False, proxyKey=False + ): + """ + Generate a proxy and get it as a string. + + Check here: https://github.com/eventbrite/m2crypto/blob/master/demo/x509/ca.py#L45 + + Args: + lifetime (int): expected lifetime in seconds of proxy + diracGroup (str): diracGroup to add to the certificate + strength (int): length in bits of the pair if proxyKey not given (default 2048) + limited (bool): Create a limited proxy (default False) + proxyKey: M2Crypto.EVP.PKey instance with private and public key. If not given, generate one + rfc: placeholder for backward compatibility and ignored + + :returns: S_OK(PEM encoded string), S_ERROR. The PEM string contains all the certificates in the chain + and the private key associated to the last X509Certificate just generated. + """ + + issuerCert = self._certList[0] + + # If this is a certificate signing request then the private key will be + # appended by the server and we don't need to include it in the proxy + include_private_key = not proxyKey + if not proxyKey: + # Generating key is a two step process: create key object and then assign RSA key. + # This contains both the private and public key + proxyKey = M2Crypto.EVP.PKey() + proxyKey.assign_rsa(M2Crypto.RSA.gen_key(strength, 65537, callback=M2Crypto.util.quiet_genparam_callback)) + + # Generate a new X509Certificate object + proxyExtensions = self.__getProxyExtensionList(diracGroup, limited) + res = X509Certificate.generateProxyCertFromIssuer(issuerCert, proxyExtensions, proxyKey, lifetime=lifetime) + if not res["OK"]: + return res + proxyCert = res["Value"] + + # Sign it with one owns key + proxyCert.sign(self._keyObj, "sha256") + + # Generate the proxy string + proxyString = proxyCert.asPem() + if include_private_key: + proxyString += proxyKey.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode("ascii") + for i in range(len(self._certList)): + crt = self._certList[i] + proxyString += crt.asPem() + return S_OK(proxyString) + + # pylint: disable=unused-argument + def generateProxyToFile(self, filePath, lifetime, diracGroup=False, strength=DEFAULT_PROXY_STRENGTH, limited=False): + """ + Generate a proxy and put it into a file + + Args: + filePath: file to write + lifetime: expected lifetime in seconds of proxy + diracGroup: diracGroup to add to the certificate + strength: length in bits of the pair + limited: Create a limited proxy + rfc: placeholder and ignored + """ + retVal = self.generateProxyToString(lifetime, diracGroup, strength, limited) + if not retVal["OK"]: + return retVal + try: + with secureOpenForWrite(filePath) as (fd, _filename): + fd.write(retVal["Value"]) + except Exception as e: + return S_ERROR(DErrno.EWF, f"{filePath} :{repr(e).replace(',)', ')')}") + return S_OK() + + @needCertList + def isProxy(self): + """ + Check whether this chain is a proxy + + :returns: S_OK(boolean) + """ + return S_OK(self.__isProxy) + + @needCertList + def isLimitedProxy(self): + """ + Check whether this chain is a limited proxy + + :returns: S_OK(boolean) + """ + return S_OK(self.__isProxy and self.__isLimitedProxy) + + @needCertList + def isValidProxy(self, ignoreDefault=False): + """ + Check whether this chain is a valid proxy, that is: + * a proxy + * still valid + * with a valid group + + :param ignoreDefault: (what a stupid name) if True, do not lookup the CS + + :returns: S_OK(True) if the proxy is valid, S_ERROR otherwise + + """ + if not self.__isProxy: + return S_ERROR(DErrno.ENOCHAIN, "Chain is not a proxy") + + if self.hasExpired()["Value"]: + return S_ERROR(DErrno.ENOCHAIN) + + if ignoreDefault: + groupRes = self.getDIRACGroup(ignoreDefault=True) + if not groupRes["OK"]: + return groupRes + if not groupRes["Value"]: + return S_ERROR(DErrno.ENOGROUP) + + return S_OK(True) + + def isVOMS(self): + """ + Check whether this proxy contains VOMS extensions. + It is enough for one of the certificate of the chain to have VOMS extension + + :returns: S_OK(boolean) + """ + + if not self.__isProxy: + return S_OK(False) + + for cert in self._certList: + if cert.hasVOMSExtensions()["Value"]: + return S_OK(True) + return S_OK(False) + + def isRFC(self): + """Check whether this is an RFC proxy. It can only be true, providing it is a proxy + + :returns: S_OK(boolean) + """ + + return self.isProxy() + + def getVOMSData(self): + """ + Returns the voms data. + + :returns: See :py:func:`~DIRAC.Core.Security.m2crypto.X509Certificate.getVOMSData` + If no VOMS data is available, return DErrno.EVOMS + :warning: In case the chain is not a proxy, this method will return False. + Yes, it's stupid, but it is for compatibility... + + """ + if not self.__isProxy: + return S_OK(False) + + for cert in self._certList: + res = cert.getVOMSData() + if res["OK"]: + return res + return S_ERROR(DErrno.EVOMS) + + def __checkProxyness(self): + """This method is called upon initialization of a chain and fill in some internal attributes. + + Pure madness... + + To me, this method just seems to work by pure luck.. + """ + + self.__hash = False + self.__firstProxyStep = len(self._certList) - 2 # -1 is user cert by default, -2 is first proxy step + self.__isProxy = True + self.__isLimitedProxy = False + prevDNMatch = 2 + # If less than 2 steps in the chain is no proxy + if len(self._certList) < 2: + self.__isProxy = False + return + + # Here we make sure that each certificate in the chain was + # signed by the previous one + for step in range(len(self._certList) - 1): + # this is a cryptographic check with the keys + issuerMatch = self.__checkIssuer(step, step + 1) + if not issuerMatch: + self.__isProxy = False + return + + # Do we need to check the proxy DN? + if prevDNMatch: + dnMatch = self.__checkProxyDN(step, step + 1) + if dnMatch == 0: + # If we are not in the first step we've found the entity cert + if step > 0: + self.__firstProxyStep = step - 1 + # If we are in the first step this is not a proxy + else: + self.__isProxy = False + return + # Limited proxy DN match + elif dnMatch == 2: + self.__isLimitedProxy = True + if prevDNMatch != 2: + self.__isProxy = False + self.__isLimitedProxy = False + return + prevDNMatch = dnMatch + + def __checkProxyDN(self, certStep, issuerStep): + """ + Checks that the subject of the proxy is properly derived from the issuer subject. + + Args: + certStep: position of the certificate to check in self.__certList + issuerStep: position of the issuer certificate to check in self.__certList + + :returns: an int based on the match: + 0 = no match + 1 = proxy match + 2 = limited proxy match + """ + + issuerSubject = self._certList[issuerStep].getSubjectNameObject() + if not issuerSubject["OK"]: + return 0 + issuerSubject = issuerSubject["Value"] + + proxySubject = self._certList[certStep].getSubjectNameObject() + if not proxySubject["OK"]: + return 0 + proxySubject = proxySubject["Value"] + + lastEntry = str(proxySubject).split("/")[-1].split("=") + limited = False + if lastEntry[0] != "CN": + return 0 + + # For non-RFC proxy, the proxy always had these two strings in the CN + if lastEntry[1] not in ("proxy", "limited proxy"): + # for RFC proxy, one has to check the extension. + ext = self._certList[certStep].getExtension("proxyCertInfo") + if not ext["OK"]: + return 0 + + ext = ext["Value"] + + # Check the RFC + contraint = [ + line.split(":")[1].strip() + for line in ext.get_value().split("\n") + if line.split(":")[0] == "Policy Language" + ] + if not contraint: + return 0 + if contraint[0] == LIMITED_PROXY_OID: + limited = True + else: + if lastEntry[1] == "limited proxy": + limited = True + if not str(issuerSubject) == str(proxySubject)[: str(proxySubject).rfind("/")]: + return 0 + return 1 if not limited else 2 + + def __checkIssuer(self, certStep, issuerStep): + """ + Check that the issuer has signed the certificate with his private key + + :param certStep: position of the certificate in self.__certList + :param issuerStep: position of the issuer certificate + + :returns: S_OK(boolean) + + """ + issuerCert = self._certList[issuerStep] + cert = self._certList[certStep] + pubKey = issuerCert.getPublicKey()["Value"] + + return cert.verify(pubKey)["Value"] + + @needCertList + def getDIRACGroup(self, ignoreDefault=False): + """ + Retrieve the dirac group of the chain + + :param ignoreDefault: (default False) if True, do not lookup the CS for a group if it is not in the proxy + + :returns: S_OK(dirac group)/S_ERROR + """ + if not self.__isProxy: + return S_ERROR(DErrno.EX509, "Chain does not contain a valid proxy") + + # The code below will find the first match of the DIRAC group + for cert in reversed(self._certList): + # We specifically say we do not want the default to first check inside the proxy + retVal = cert.getDIRACGroup(ignoreDefault=True) + if retVal["OK"] and "Value" in retVal and retVal["Value"]: + return retVal + + # No DIRAC group found, try to get the default one + return self.getCertInChain(self.__firstProxyStep)["Value"].getDIRACGroup(ignoreDefault=ignoreDefault) + + @needCertList + def hasExpired(self): + """ + Check whether any element of the chain has expired + + :returns: S_OK(boolean) + """ + for cert in reversed(self._certList): + res = cert.hasExpired() + if not res["OK"]: + return res + # If True, it means the cert has expired + if res["Value"]: + return S_OK(True) + + return S_OK(False) + + @needCertList + def getNotAfterDate(self): + """ + Get the smallest not after date + + :returns: S_OK(datetime.datetime) + """ + notAfter = self._certList[0].getNotAfterDate() + if not notAfter["OK"]: + return notAfter + notAfter = notAfter["Value"] + + for cert in reversed(self._certList): + res = cert.getNotAfterDate() + if not res["OK"]: + return res + stepNotAfter = res["Value"] + + # If the current cert has already expired + # we return this as notAfter date + res = cert.hasExpired() + if not res["OK"]: + return res + if res["Value"]: + return S_OK(stepNotAfter) + + # if the current cert has a shorter lifetime + # as the current reference, take it as new reference + notAfter = min(notAfter, stepNotAfter) + + return S_OK(notAfter) + + @needCertList + def generateProxyRequest(self, bitStrength=DEFAULT_PROXY_STRENGTH, limited=False): + """ + Generate a proxy request. + See :py:meth:`DIRAC.Core.Security.m2crypto.X509Certificate.X509Certificate.generateProxyRequest` + + Return S_OK( X509Request ) / S_ERROR + """ + + # We use the first certificate of the chain to do the proxy request + x509 = self._certList[0] + return x509.generateProxyRequest(bitStrength, limited) + + @needCertList + def getStrength(self): + """ + Returns the strength in bit of the key of the first certificate in the chain + """ + x509 = self._certList[0] + return x509.getStrength() + + @needCertList + @needPKey + def generateChainFromRequestString(self, pemData, lifetime=86400, requireLimited=False, diracGroup=False): + """ + Generate a x509 chain from a request. + + :param pemData: PEM encoded request + :param lifetime: lifetime of the delegated proxy in seconds (default 1 day) + :param requireLimited: if True, requires a limited proxy + :param diracGroup: DIRAC group to put in the proxy + :param rfc: placeholder for compatibility, ignored + + :returns: S_OK( X509 chain pem encoded string ) / S_ERROR. The new chain will have been signed + with the public key included in the request + + """ + try: + req = M2Crypto.X509.load_request_string(pemData, format=M2Crypto.X509.FORMAT_PEM) + + except Exception as e: + return S_ERROR(DErrno.ECERTREAD, f"Can't load request data: {repr(e).replace(',)', ')')}") + + # I am not sure this test makes sense. + # You can't request a limit proxy if you are yourself not limited ?! + # I think it should be a "or" instead of "and" + limited = requireLimited and self.isLimitedProxy().get("Value", False) + return self.generateProxyToString(lifetime, diracGroup, DEFAULT_PROXY_STRENGTH, limited, req.get_pubkey()) + + @needCertList + def getRemainingSecs(self): + """ + Get remaining time (minimum of all cert in the chain) + + :returns: S_OK(time left in seconds) + """ + remainingSecs = self.getCertInChain(0)["Value"].getRemainingSecs()["Value"] + for cert in self._certList[1:]: + stepRS = cert.getRemainingSecs()["Value"] + remainingSecs = min(remainingSecs, stepRS) + + return S_OK(remainingSecs) + + @needCertList + def dumpAllToString(self): + """ + Dump the current chain as a PEM encoded string + The order would be: + + * first certificate + * private key (without passphrase) + * other certificates + + :returns: S_OK(PEM encoded chain with private key) + """ + data = self._certList[0].asPem() + if self._keyObj: + data += self._keyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode("ascii") + for cert in self._certList[1:]: + data += cert.asPem() + return S_OK(data) + + def dumpAllToFile(self, filename=False): + """ + Dump all to file. + + :param filename: If not specified, a temporary one will be created + + :returns: S_OK(filename)/S_ERROR + """ + retVal = self.dumpAllToString() + if not retVal["OK"]: + return retVal + pemData = retVal["Value"] + try: + with secureOpenForWrite(filename) as (fh, filename): + fh.write(pemData) + except Exception as e: + return S_ERROR(DErrno.EWF, f"{filename} :{repr(e).replace(',)', ')')}") + return S_OK(filename) + + @needCertList + def dumpChainToString(self): + """ + Dump only cert chain to string, without the PKey + + :returns: S_OK(pem chain) + """ + return S_OK("".join(cert.asPem() for cert in self._certList)) + + @needPKey + def dumpPKeyToString(self): + """ + Dump only the key to string, not encoded + + :returns: S_OK(PEM encoded key) + + """ + return S_OK(self._keyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode("ascii")) + + def __str__(self): + """String representation""" + repStr = " = ()" + Typically, the nickname will look like + 'nickname = chaen (lhcb)', + + * fqan: List of VOMS "position" (['/lhcb/Role=production/Capability=NULL', '/lhcb/Role=NULL/Capability=NULL']) + * vo: name of the VO, + * subject: subject DN to which the attributes were granted, + * issuer: typically the DN of the VOMS server (e.g '/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch') + + """ + vomsExtensionDict = {} + vomsExtensionOctetString = retrieveExtension(m2cert, VOMS_EXTENSION_OID) + # Decode it as a ACSequenceOfSequence, which is what it is... + vomsExtensionSeqOfSeq, _rest = der_decode(vomsExtensionOctetString, asn1Spec=_ACSequenceOfSequence()) + + # In principle, according to GFD 182, there could be more than one VO VOMS AC per proxy. + # The standard specifies that we have to accept at least the first one, which is what + # I will do... + vomsCertAttribute = vomsExtensionSeqOfSeq[0][0] + + ###### + # TODO in principle, we should check the signature of the Attribute... + # _signatureAlgorith = vomsCertAttribute['signatureAlgorithm'] + # _signatureValue = vomsCertAttribute['signatureValue'] + ###### + + certAttrInfo = vomsCertAttribute["acinfo"] + + # pyasn1 does things correctly by setting a timezone info in the datetime + # However, we do not in DIRAC, and so we can't compare the dates. + # We have to remove the timezone info from the datetime objects + + notBefore = certAttrInfo["attrCertValidityPeriod"]["notBeforeTime"].asDateTime + vomsExtensionDict["notBefore"] = notBefore.replace(tzinfo=None) + + notAfter = certAttrInfo["attrCertValidityPeriod"]["notAfterTime"].asDateTime + vomsExtensionDict["notAfter"] = notAfter.replace(tzinfo=None) + + # ######### Retrieving the issuer ########## + # Get the issuer. A bit tricky, because we have to reconstruct the full DN ourselves + # The GFD 182 and RFC 3281 give enough restriction such that we can afford some direct + # [0] access + + issuer = "" + + # rdnName is a rfc3280.RelativeDistinguishedName object + for rdnName in certAttrInfo["issuer"]["v2Form"]["issuerName"][0]["directoryName"]["rdnSequence"]: + # rdnNameAttr rfc3280.AttributeTypeAndValue' + rdnNameAttr = rdnName[0] + + attrOid = ".".join([str(e) for e in rdnNameAttr["type"].asTuple()]) + + # Now finally convert the last part into a asn1char.*String + attrValStr = _decodeASN1String(rdnNameAttr["value"]) + attrVal = attrValStr.asOctets().decode() + # + issuer += f"{DN_MAPPING[attrOid]}{attrVal}" + + vomsExtensionDict["issuer"] = issuer + + # ### Issuer retrieved ##### + + # ## Retrieving the Subject #### + # We have to do the same for the subject than for the issuer + + subject = "" + + # rdnName is a rfc3280.RelativeDistinguishedName object + for rdnName in certAttrInfo["holder"]["baseCertificateID"]["issuer"][0]["directoryName"]["rdnSequence"]: + # rdnNameAttr rfc3280.AttributeTypeAndValue' + rdnNameAttr = rdnName[0] + + attrOid = ".".join([str(e) for e in rdnNameAttr["type"].asTuple()]) + # # Because there are non printable characters in the values (new line, etc) + # # we have to get ride of them. The best way is to get them as number, and make sure it is a + # # a printable char (between 32 and 126) + # + # attrVal = ''.join([chr(c) for c in rdnNameAttr['value'].asNumbers() if 32 <= c <= 126 ]) + + # Now finally convert the last part into a asn1char.*String + attrValStr = _decodeASN1String(rdnNameAttr["value"]) + attrVal = attrValStr.asOctets().decode() + + subject += f"{DN_MAPPING[attrOid]}{attrVal}" + + vomsExtensionDict["subject"] = subject + + # ### Retrieving the FQAN #### + + # According to GFD182, there may be more attributes that just the FQAN, even though it + # does not seem to be the case in practice. So we make sure to have the good one + fqanOIDObj = univ.ObjectIdentifier(VOMS_FQANS_OID) + + # There shall be only one, hense the [0] + # This is an rfc3280.Attribute object + fqanAttrObj = [attrObj for attrObj in certAttrInfo["attributes"] if attrObj["type"] == fqanOIDObj][0] + + # According to GFD182 3.4.1, we decode the value as a IetfAttrSyntax. + # Since multiple values are not allowed, just take the first item + # + fqanObj, _rest = der_decode(fqanAttrObj["values"][0], asn1Spec=rfc3281.IetfAttrSyntax()) + + # We retrieve the VO and the VOMS server + voName, _, _ = fqanObj["policyAuthority"][0]["uniformResourceIdentifier"].asOctets().decode().split(":") + + vomsExtensionDict["vo"] = voName + + # Now retrieve the position of the holder (group, role) + fqanList = [] + for fqanPositionObj in fqanObj["values"]: + fqanList.append(fqanPositionObj["octets"].asOctets().decode()) + + vomsExtensionDict["fqan"] = fqanList + + # ############ End of the FQAN ################ + + # Now the Tags, called attributes in the dict... + + tagDescriptions = [] + vomsTagsOIDObj = univ.ObjectIdentifier(VOMS_TAGS_EXT_OID) + + # First find the tag containers + tagExtensionObj = [extObj for extObj in certAttrInfo["extensions"] if extObj["extnID"] == vomsTagsOIDObj] + + # If we found tags + if tagExtensionObj: + # Multiple is forbiden, so only one tag container + tagExtensionObj = tagExtensionObj[0] + + tagContainersObj, _rest = der_decode(tagExtensionObj["extnValue"], asn1Spec=_TagContainers()) + + # TODO in principle, we should check that this value + # and the one of the policyAuthority of the fqan are the same + # _tagPolicyAuthority = tagContainersObj[0][0]['policyAuthority'][0]['uniformResourceIdentifier'] \ + # .asOctets().decode() + ###### + + for tagContainer in tagContainersObj: + for tagList in tagContainer: + # Note: it is here that I should check the policyAuthority + tagList = tagList["tags"] + for tag in tagList: + # This gives a string like + # nickname = chaen (lhcb) + tagDescriptions.append( + "%s = %s (%s)" + % ( + tag["name"].asOctets().decode(), + tag["value"].asOctets().decode(), + tag["qualifier"].asOctets().decode(), + ) + ) + + vomsExtensionDict["attribute"] = ",".join(tagDescriptions) + + # #### Tags are done ################ + + return vomsExtensionDict + + +def retrieveExtension(m2Cert, extensionOID): + """Retrieves the extension from a certificate from its OID + + :param m2Cert: M2Crypto X509 object, a certificate + :param extensionOID: the OID we are looking for + + :returns: an ~pyasn1.type.univ.OctetString object, which is the content of the extension + (it still needs to be deserialized, depending on the extension !) + + :raises: LookupError if it does not have the extension + """ + extensions = _extensionsFromCertDER(m2Cert.as_der()) + + # Construct an OID object for comparison purpose + extensionOIDObj = univ.ObjectIdentifier(extensionOID) + + # We check every extension OID. This will be necessary until M2Crypto + # allows to register OID alias (https://gitlab.com/m2crypto/m2crypto/issues/231) + for extension in extensions: + # We found the good extension + if extension["extnID"] == extensionOIDObj: + return extension["extnValue"] + + # If we are here, it means that we could not find the expected extension. + raise LookupError(f"Could not find extension with OID {extensionOID}") + + +@lru_cache(maxsize=1024) +def _extensionsFromCertDER(der): + # Decode the certificate as a RFC2459 Certificate object.It is compatible + # with the RFC proxy definition + cert, _rest = der_decode(der, asn1Spec=rfc2459.Certificate()) + extensions = cert["tbsCertificate"]["extensions"] + return extensions From e382bc4408b2c927bb5ba39afa36235701830c2f Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 11 Jul 2026 00:16:03 +0200 Subject: [PATCH 5/7] feat: select the X509/SSL implementation with DIRAC_USE_M2CRYPTO Move the pyca/cryptography based X509 modules into the DIRAC.Core.Security.pyca subpackage and select the implementation once, at import time, by extending the DIRAC.Core.Security search path with either the pyca or the m2crypto subpackage, based on the DIRAC_USE_M2CRYPTO environment variable (default No, i.e. pyca). This is the same pkgutil.extend_path mechanism that was used during the pyGSI to M2Crypto transition. The DISET SSLTransport module becomes a selector: the implementation agnostic delegate() and checkSanity() helpers stay there, while the standard library ssl implementation moves to StdSSLTransport, and the SSLTransport class and the getSSLContext/getPeerInfo helpers are picked between it and the M2SSLTransport/M2Utils ones. The HTTPS services honour the switch as well: TornadoServer only configures the tornado_m2crypto iostream and builds the dict style ssl_options when the fallback is enabled (so that the default path never imports M2Crypto or tornado_m2crypto, which remain optional dependencies, listed in the new m2crypto extra), and BaseRequestHandler._authzSSL and DIRACCAProxyProvider dispatch between the two implementations the same way. --- environment.yml | 17 +- setup.cfg | 6 + .../DISET/private/Transports/SSLTransport.py | 529 +------ .../private/Transports/StdSSLTransport.py | 518 +++++++ .../Transports/test/Test_SSLTransport.py | 6 +- src/DIRAC/Core/Security/__init__.py | 40 +- src/DIRAC/Core/Security/{ => pyca}/X509CRL.py | 2 +- .../Security/{ => pyca}/X509Certificate.py | 2 +- .../Core/Security/{ => pyca}/X509Chain.py | 4 +- .../Core/Security/{ => pyca}/X509Request.py | 4 +- src/DIRAC/Core/Security/pyca/__init__.py | 1 + .../Core/Security/{ => pyca}/asn1_utils.py | 0 .../Core/Tornado/Server/TornadoServer.py | 51 +- .../Server/private/BaseRequestHandler.py | 70 + .../ProxyProvider/DIRACCAProxyProvider.py | 1226 +++++++++++------ tests/py3CheckDirs.txt | 2 +- 16 files changed, 1542 insertions(+), 936 deletions(-) mode change 100755 => 100644 src/DIRAC/Core/DISET/private/Transports/SSLTransport.py create mode 100755 src/DIRAC/Core/DISET/private/Transports/StdSSLTransport.py rename src/DIRAC/Core/Security/{ => pyca}/X509CRL.py (98%) rename src/DIRAC/Core/Security/{ => pyca}/X509Certificate.py (99%) rename src/DIRAC/Core/Security/{ => pyca}/X509Chain.py (99%) rename src/DIRAC/Core/Security/{ => pyca}/X509Request.py (97%) create mode 100644 src/DIRAC/Core/Security/pyca/__init__.py rename src/DIRAC/Core/Security/{ => pyca}/asn1_utils.py (100%) diff --git a/environment.yml b/environment.yml index e512a024928..902151350fb 100644 --- a/environment.yml +++ b/environment.yml @@ -23,6 +23,11 @@ dependencies: - gitpython >=2.1.0 - invoke - cryptography >=47.0.0 + # m2crypto, pyasn1 and pyasn1-modules are only needed for the legacy + # M2Crypto based implementation (DIRAC_USE_M2CRYPTO=Yes) + - m2crypto >=0.38.0 + - pyasn1 >0.4.1 + - pyasn1-modules - matplotlib - numpy - paramiko @@ -87,7 +92,10 @@ dependencies: - uritemplate # - readline >=6.2.4 in the standard library - simplejson >=3.8.1 - - tornado >=5.1.1,<6.0.0 + # tornado is installed with pip below: the DIRACGrid fork is needed for the + # legacy M2Crypto based implementation (DIRAC_USE_M2CRYPTO=Yes) and behaves + # as the standard tornado >=5.1.1,<6.0.0 otherwise + #- tornado >=5.1.1,<6.0.0 - typing >=3.6.6 - rucio-clients >=34.4.2 # For mypy @@ -103,6 +111,13 @@ dependencies: - Authlib >=1.0.0 - dominate - pyjwt + # The following two packages are only needed for the legacy M2Crypto + # based implementation (DIRAC_USE_M2CRYPTO=Yes) + # This is a fork of tornado with a patch to allow for configurable iostream + - git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable + # This is an extension of Tornado to use M2Crypto + # It should eventually be part of DIRACGrid + - git+https://github.com/DIRACGrid/tornado_m2crypto - -e .[server] - -e ./dirac-common/ # Add diracdoctools diff --git a/setup.cfg b/setup.cfg index a56e9a47bed..32dc726cd8d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -101,6 +101,12 @@ testing = pytest-mock pytest-rerunfailures pycodestyle +# Only needed for the legacy M2Crypto based implementation (DIRAC_USE_M2CRYPTO=Yes) +m2crypto = + M2Crypto >=0.36 + pyasn1 + pyasn1-modules + tornado-m2crypto [options.entry_points] dirac = diff --git a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py old mode 100755 new mode 100644 index 3e4d5ef8bd6..d411886b5ce --- a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py @@ -1,518 +1,35 @@ -"""SSL Transport implementation using the standard library ssl module. +"""SSL Transport selector. -The peer chain validation (including RFC 3820 proxy support) is delegated to -OpenSSL through ``ssl.SSLContext``: +Two implementations of the SSL transport are available: the default one, +based on the standard library ssl module (:py:mod:`.StdSSLTransport`), and +the legacy one, based on M2Crypto (:py:mod:`.M2SSLTransport`). The +implementation is selected once, at import time, with the DIRAC_USE_M2CRYPTO +environment variable (default No). -* proxy certificates are accepted by setting the ``X509_V_FLAG_ALLOW_PROXY_CERTS`` - verify flag (``ssl.VERIFY_ALLOW_PROXY_CERTS``) -* the peer chain is retrieved after the handshake with - ``ssl.SSLSocket.get_unverified_chain`` (python >= 3.13), and the DIRAC - credentials (DN, group, ...) are extracted from it with - :py:class:`DIRAC.Core.Security.X509Chain.X509Chain` +The helpers defined here (delegate, checkSanity) are implementation agnostic. """ import os -import socket -import ssl -import tempfile -import threading -import time -from collections import OrderedDict from DIRAC import gLogger -from DIRAC.Core.DISET import DEFAULT_CONNECTION_TIMEOUT, DEFAULT_RPC_TIMEOUT -from DIRAC.Core.DISET.private.Transports.BaseTransport import BaseTransport from DIRAC.Core.Security import Locations -from DIRAC.Core.Security.X509Certificate import X509Certificate -from DIRAC.Core.Security.X509Chain import X509Chain from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK - -# Verify depth of peer certs -VERIFY_DEPTH = 50 - -# Re-exported for convenience (exists since python 3.10) -VERIFY_ALLOW_PROXY_CERTS = ssl.VERIFY_ALLOW_PROXY_CERTS - - -def __loadHostCertificate(ctx): - """Load hostcert & key from the default location and set them as the - credentials for SSL context ctx. - Returns None. - """ - certKeyTuple = Locations.getHostCertificateAndKeyLocation() - if not certKeyTuple: - raise RuntimeError("Hostcert/key location not set") - hostcert, hostkey = certKeyTuple - if not os.path.isfile(hostcert): - raise RuntimeError(f"Hostcert file ({hostcert}) is missing") - if not os.path.isfile(hostkey): - raise RuntimeError(f"Hostkey file ({hostkey}) is missing") - ctx.load_cert_chain(hostcert, keyfile=hostkey) - - -def __loadProxy(ctx, proxyPath=None): - """Load proxy from proxyPath (or default location if not specified) and - set it as the certificate & key to use for this SSL context. - Returns None. - """ - if not proxyPath: - proxyPath = Locations.getProxyLocation() - if not proxyPath: - raise RuntimeError("Proxy location not set") - if not os.path.isfile(proxyPath): - raise RuntimeError(f"Proxy file ({proxyPath}) is missing") - # A proxy file contains the leaf certificate, its key, and the rest - # of the chain in a single file, which load_cert_chain handles natively - ctx.load_cert_chain(proxyPath) - - -def getSSLContext(**kwargs): - """Gets an ssl.SSLContext configured using the standard - DIRAC connection keywords from kwargs. The keywords are: - - - bServerMode: Boolean, if True the context is setup for a server - (hostcert is always used). - - useCertificates: Boolean, Set to true to use hostcerts in client - mode. - - proxyString: String, allow a literal proxy string to be provided. - - proxyLocation: String, Path to file to use as proxy, defaults to - usual location(s) if not set. - - skipCACheck: Boolean, if True, don't verify peer certificates. - - optionalClientCert: Boolean, in server mode, request but do not require - the client certificate (used by the HTTPS services, - where token/visitor authentication also exists). - - sslCiphers: String, OpenSSL style cipher string of ciphers to allow - on this connection. - - Returns the new context. - """ - serverMode = kwargs.get("bServerMode", False) - - if serverMode: - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - else: - # PROTOCOL_TLS_CLIENT enables check_hostname and CERT_REQUIRED by default - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - ctx.minimum_version = ssl.TLSVersion.TLSv1_2 - - # Set certificates for connection - if serverMode or (kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False)): - # Server mode always uses hostcert - __loadHostCertificate(ctx) - - else: - # Client mode has a choice of possible options - if kwargs.get("proxyString", None): - # ssl cannot take an in-memory location or a string, - # so write it to a temp file and use proxyLocation - with tempfile.NamedTemporaryFile(mode="w") as tmpFile: - tmpFile.write(kwargs["proxyString"]) - # Flush, otherwise the file is empty in the subsequent call - tmpFile.flush() - __loadProxy(ctx, proxyPath=tmpFile.name) - else: - # Use normal proxy - __loadProxy(ctx, proxyPath=kwargs.get("proxyLocation", None)) - - # Set peer verification - if kwargs.get("skipCACheck", False): - # Don't validate peer - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - else: - # Do validate peer - if serverMode and kwargs.get("optionalClientCert", False): - ctx.verify_mode = ssl.CERT_OPTIONAL - else: - ctx.verify_mode = ssl.CERT_REQUIRED - # Allow proxy certificates to be used - ctx.verify_flags |= VERIFY_ALLOW_PROXY_CERTS - # Set CA location - caPath = Locations.getCAsLocation() - if not caPath: - raise RuntimeError("Failed to find CA location") - if not os.path.isdir(caPath): - raise RuntimeError(f"CA path ({caPath}) is not a valid directory") - ctx.load_verify_locations(capath=caPath) - - # DIRAC_M2CRYPTO_SSL_CIPHERS is accepted for backward compatibility - ciphers = ( - kwargs.get("sslCiphers") or os.environ.get("DIRAC_SSL_CIPHERS") or os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") - ) - if ciphers: - ctx.set_ciphers(ciphers) - - return ctx - - -# Client-side SSL contexts are cached and shared between connections, as -# building one means reading and parsing the credential files for every RPC. -# An entry is invalidated when the credential file changes (e.g. proxy -# renewal), and expires after _CLIENT_CTX_CACHE_TTL seconds so that changes -# in the CA directory are eventually picked up. -_CLIENT_CTX_CACHE: OrderedDict = OrderedDict() -_CLIENT_CTX_CACHE_LOCK = threading.Lock() -_CLIENT_CTX_CACHE_MAX = 8 -_CLIENT_CTX_CACHE_TTL = 300 - - -def _clientContextCacheKey(kwargs): - """Compute the cache key for a client SSL context, or None if the - configuration is not cacheable. The credential selection mirrors - :py:func:`getSSLContext`, and the key contains the stat info of the - credential files so the cache is invalidated when they change. - - :param kwargs: the connection keywords, as given to getSSLContext - """ - if kwargs.get("bServerMode") or kwargs.get("proxyString"): - return None - if kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False): - credentialFiles = Locations.getHostCertificateAndKeyLocation() - if not credentialFiles: - return None - else: - proxyPath = kwargs.get("proxyLocation") or Locations.getProxyLocation() - if not proxyPath: - return None - credentialFiles = (proxyPath,) - - fileStats = [] - try: - for path in credentialFiles: - st = os.stat(path) - fileStats.append((path, st.st_mtime_ns, st.st_size, st.st_ino)) - except OSError: - return None - - skipCACheck = bool(kwargs.get("skipCACheck", False)) - caPath = None if skipCACheck else Locations.getCAsLocation() - ciphers = ( - kwargs.get("sslCiphers") or os.environ.get("DIRAC_SSL_CIPHERS") or os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") +from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error +from DIRAC.Core.Security.X509Certificate import X509Certificate # pylint: disable=import-error + + +# Even though SSLTransport, getSSLContext and getPeerInfo are not all used in +# this file, they are imported from here by other modules, so do not remove +# these imports! +if os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true"): + from DIRAC.Core.DISET.private.Transports.M2SSLTransport import SSLTransport + from DIRAC.Core.DISET.private.Transports.SSL.M2Utils import getM2SSLContext as getSSLContext + from DIRAC.Core.DISET.private.Transports.SSL.M2Utils import getM2PeerInfo as getPeerInfo +else: + from DIRAC.Core.DISET.private.Transports.StdSSLTransport import ( + SSLTransport, + getSSLContext, + getPeerInfo, ) - return (tuple(fileStats), skipCACheck, caPath, ciphers) - - -def _getClientSSLContext(**kwargs): - """Return a client SSL context, shared from the cache whenever possible - (ssl.SSLContext objects are safe to share between connections and threads) - """ - cacheKey = _clientContextCacheKey(kwargs) - if cacheKey is None: - return getSSLContext(**kwargs) - - now = time.monotonic() - with _CLIENT_CTX_CACHE_LOCK: - cached = _CLIENT_CTX_CACHE.get(cacheKey) - if cached and now - cached[0] < _CLIENT_CTX_CACHE_TTL: - _CLIENT_CTX_CACHE.move_to_end(cacheKey) - return cached[1] - - ctx = getSSLContext(**kwargs) - - with _CLIENT_CTX_CACHE_LOCK: - _CLIENT_CTX_CACHE[cacheKey] = (now, ctx) - _CLIENT_CTX_CACHE.move_to_end(cacheKey) - while len(_CLIENT_CTX_CACHE) > _CLIENT_CTX_CACHE_MAX: - _CLIENT_CTX_CACHE.popitem(last=False) - return ctx - - -def getPeerInfo(sslSocket): - """Gets the details of the current peer as a standard dict. The peer - details are obtained from the supplied ssl.SSLSocket. - The details returned are those from ~X509Chain.getCredentials, without Registry info: - - DN - Full peer DN as string - x509Chain - Full chain of peer - isProxy - Boolean, True if chain ends with proxy - isLimitedProxy - Boolean, True if chain ends with limited proxy - group - String, DIRAC group for this peer, if known - - Returns a dict of details. - """ - if not hasattr(sslSocket, "get_unverified_chain"): - raise RuntimeError("Extracting the peer certificate chain requires python >= 3.13 (ssl get_unverified_chain)") - # The chain, as sent by the peer (leaf first). It was already validated - # by OpenSSL during the handshake, and the proxy-ness of it is - # anyway cryptographically re-checked by X509Chain - derChain = sslSocket.get_unverified_chain() - if not derChain: - return {} - chain = X509Chain.generateX509ChainFromDERList(derChain) - creds = chain.getCredentials(withRegistryInfo=False) - if not creds["OK"]: - raise RuntimeError(f"Failed to get SSL peer info ({creds['Message']}).") - peer = creds["Value"] - - # getCredentials already resolves DN (identity for proxies, subject - # otherwise), isProxy and isLimitedProxy - peer["x509Chain"] = chain - - return peer - - -class SSLTransport(BaseTransport): - """SSL Transport implementation based on the standard library ssl module.""" - - # This name is the same as BaseClient, - # and is used a bit everywhere, so it should be factorized out - # eventually - KW_TIMEOUT = "timeout" - - def __init__(self, *args, **kwargs): - """Create an SSLTransport object, parameters are the same - as for other transports. If ctx is specified (as an instance of - ssl.SSLContext) then use that rather than creating a new context. - - kwargs can contain all the parameters defined in BaseClient, - in particular timeout - """ - self.remoteAddress = None - self.peerCredentials = {} - - # The timeout is to be understood here as the timeout for socket - # operations involved in the RPC call, but NOT the establishment - # of the connection, for which there is a different timeout - # (DEFAULT_CONNECTION_TIMEOUT) - self.__timeout = kwargs.get(SSLTransport.KW_TIMEOUT, DEFAULT_RPC_TIMEOUT) - - self.__locked = False # We don't support locking, so this is always false. - - self.__ctx = kwargs.pop("ctx", None) - if not self.__ctx: - if kwargs.get("bServerMode", False): - self.__ctx = getSSLContext(**kwargs) - else: - self.__ctx = _getClientSSLContext(**kwargs) - - # Note that kwargs is already kept in BaseTransport - # as self.extraArgsDict, but at least I am sure that - # self.__kwargs will never be modified - self.__kwargs = kwargs - - BaseTransport.__init__(self, *args, **kwargs) - - def setSocketTimeout(self, timeout): - """Set the timeout for RPC calls. - - .. warning: This needs to be called before initAsClient. - It is used as a timeout for RPC calls, not connection. - - :param timeout: timeout for socket operation in seconds - - """ - self.__timeout = timeout - - def initAsClient(self): - """Prepare this client socket for use.""" - if self.serverMode(): - raise RuntimeError("SSLTransport is in server mode.") - - errors = [] - host, port = self.stServerAddress - - # Get all available addresses (IPv6 and IPv4) and try them in order - try: - addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) - except OSError as e: - return S_ERROR(f"DNS lookup failed {e!r}") - for family, socketType, proto, _canonname, socketAddress in addrInfoList: - rawSocket = None - try: - rawSocket = socket.socket(family, socketType, proto) - - # First set a short connection timeout, that will be used - # for the TCP connection and the TLS handshake - rawSocket.settimeout(DEFAULT_CONNECTION_TIMEOUT) - - # Enable keepAlive, with default options - # (see more comments about keepalive in :py:meth:`.acceptConnection`) - rawSocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - - rawSocket.connect(socketAddress) - - # server_hostname takes care of both SNI and the hostname check - self.oSocket = self.__ctx.wrap_socket(rawSocket, server_hostname=host) - - # Once the connection is established, we can use the timeout - # asked for RPC - self.oSocket.settimeout(self.__timeout) - - self.remoteAddress = self.oSocket.getpeername() - - return S_OK() - # warning: do NOT catch SSL related errors here - # They should be propagated upwards and caught by the BaseClient - # not to enter the retry loop - except ssl.SSLError: - if rawSocket is not None: - rawSocket.close() - raise - except OSError as e: - errors.append(f"{socketAddress} {e}:{repr(e)}") - if self.oSocket is not None: - self.close() - elif rawSocket is not None: - rawSocket.close() - - return S_ERROR("; ".join(errors)) - - def initAsServer(self): - """Prepare this server socket for use. - - Contrary to the client mode, the listening socket is a plain TCP - socket: the TLS layer is only added on the accepted connections, - in :py:meth:`.handshake` - """ - if not self.serverMode(): - raise RuntimeError("SSLTransport is in client mode.") - - try: - self.oSocket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - except OSError: - # Maybe no IPv6 support? Try IPv4 only socket. - self.oSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - # Make sure reuse address is set correctly - param = 1 if self.bAllowReuseAddress else 0 - self.oSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, param) - - self.oSocket.bind(self.stServerAddress) - self.oSocket.listen(self.iListenQueueSize) - return S_OK() - - def close(self): - """Close this socket.""" - if self.oSocket: - try: - self.oSocket.shutdown(socket.SHUT_RDWR) - except OSError: - # The socket may already be in a closed state - pass - self.oSocket.close() - self.oSocket = None - return S_OK() - - def renewServerContext(self): - """Renews the server context. - This reloads the certificates and the CA store by re-creating - the SSL context, which will be used for the subsequent connections. - """ - if not self.serverMode(): - raise RuntimeError("SSLTransport is in client mode.") - super().renewServerContext() - try: - self.__ctx = getSSLContext(**self.__kwargs) - except Exception as e: - gLogger.error("Failed to renew the server context", repr(e)) - return S_ERROR(f"Failed to renew the server context: {e!r}") - - return S_OK() - - def handshake(self): - """Perform the SSL handshake. - This has to be called after the connection was accepted (acceptConnection) - - The remote credentials are gathered here - """ - try: - # Warning: this method is called on the object returned - # by acceptConnection, which shares the context of the - # listening object - self.oSocket = self.__ctx.wrap_socket(self.oSocket, server_side=True) - - self.peerCredentials = getPeerInfo(self.oSocket) - - # Now that the handshake has been performed on the server - # we can set the timeout for the RPC operations. - # In practice, since we are on the server side, the - # timeout we set here represents the timeout for receiving - # the arguments and sending back the response. This should - # in principle be reasonably quick, but just to be sure - # we can set it to the DEFAULT_RPC_TIMEOUT - self.oSocket.settimeout(DEFAULT_RPC_TIMEOUT) - - return S_OK() - except (OSError, RuntimeError) as e: - return S_ERROR(f"Error in handshake: {e} {repr(e)}") - - def setClientSocket(self, oSocket): - """Set the inner socket of this instance to the value of oSocket. - This method is intended to be used to create client connection objects - from a server and should be considered to be an internal function. - - :param oSocket: client socket (plain, the handshake is done in :py:meth:`.handshake`) - - """ - # warning: do NOT catch socket.error here, because for who knows what reason - # exceptions are actually properly used for once, and the calling method - # relies on it (ServiceReactor.__acceptIncomingConnection) - self.oSocket = oSocket - self.remoteAddress = self.oSocket.getpeername() - - def acceptConnection(self): - """Accept a new client, returns a new SSLTransport object representing - the client connection. - - The connection is accepted, but no SSL handshake is performed - - :returns: S_OK(SSLTransport object) - """ - try: - oClient, _ = self.oSocket.accept() - - # Set the keep alive to true. This keepalive will ensure that we - # detect remote peer crashing or network interruption - # Note that this is ineffective if we are in the middle of blocking - # operations. - oClient.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) - - # Here we set the timeout server side. - # We first set the connection timeout, which will - # be effective for the TLS handshake - oClient.settimeout(DEFAULT_CONNECTION_TIMEOUT) - - oClientTrans = SSLTransport(self.stServerAddress, ctx=self.__ctx) - oClientTrans.setClientSocket(oClient) - return S_OK(oClientTrans) - except (OSError, RuntimeError) as e: - return S_ERROR(f"Error in acceptConnection: {e} {repr(e)}") - - def _read(self, bufSize=4096, skipReadyCheck=False): - """Read bufSize bytes from the buffer. - - :param bufSize: size of the buffer to read - :param skipReadyCheck: ignored. - - - :returns: S_OK(data read) - """ - try: - read = self.oSocket.recv(bufSize) - return S_OK(read) - except OSError as e: - return S_ERROR(f"Error in _read: {e} {repr(e)}") - - def isLocked(self): - """Returns if this instance is locked. - Always returns false. - - :returns: False - """ - return self.__locked - - def _write(self, buf): - """Write all bytes contained within iterable "buf" to the - connected peer. - - :param buf: iterable buffer - - :returns: S_OK(number of bytes written) - """ - try: - wrote = self.oSocket.send(buf) - return S_OK(wrote) - except OSError as e: - return S_ERROR(f"Error in _write: {e} {repr(e)}") def delegate(delegationRequest, kwargs): diff --git a/src/DIRAC/Core/DISET/private/Transports/StdSSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/StdSSLTransport.py new file mode 100755 index 00000000000..29457fe3bf5 --- /dev/null +++ b/src/DIRAC/Core/DISET/private/Transports/StdSSLTransport.py @@ -0,0 +1,518 @@ +"""SSL Transport implementation using the standard library ssl module. + +The peer chain validation (including RFC 3820 proxy support) is delegated to +OpenSSL through ``ssl.SSLContext``: + +* proxy certificates are accepted by setting the ``X509_V_FLAG_ALLOW_PROXY_CERTS`` + verify flag (``ssl.VERIFY_ALLOW_PROXY_CERTS``) +* the peer chain is retrieved after the handshake with + ``ssl.SSLSocket.get_unverified_chain`` (python >= 3.13), and the DIRAC + credentials (DN, group, ...) are extracted from it with + :py:class:`DIRAC.Core.Security.X509Chain.X509Chain` +""" +import os +import socket +import ssl +import tempfile +import threading +import time +from collections import OrderedDict + +from DIRAC import gLogger +from DIRAC.Core.DISET import DEFAULT_CONNECTION_TIMEOUT, DEFAULT_RPC_TIMEOUT +from DIRAC.Core.DISET.private.Transports.BaseTransport import BaseTransport +from DIRAC.Core.Security import Locations + +# The pyca based X509Chain is imported through its fully qualified path (like +# M2Utils does with the m2crypto one): getPeerInfo relies on +# generateX509ChainFromDERList, which only the pyca implementation provides +from DIRAC.Core.Security.pyca.X509Chain import X509Chain +from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK + +# Verify depth of peer certs +VERIFY_DEPTH = 50 + +# Re-exported for convenience (exists since python 3.10) +VERIFY_ALLOW_PROXY_CERTS = ssl.VERIFY_ALLOW_PROXY_CERTS + + +def __loadHostCertificate(ctx): + """Load hostcert & key from the default location and set them as the + credentials for SSL context ctx. + Returns None. + """ + certKeyTuple = Locations.getHostCertificateAndKeyLocation() + if not certKeyTuple: + raise RuntimeError("Hostcert/key location not set") + hostcert, hostkey = certKeyTuple + if not os.path.isfile(hostcert): + raise RuntimeError(f"Hostcert file ({hostcert}) is missing") + if not os.path.isfile(hostkey): + raise RuntimeError(f"Hostkey file ({hostkey}) is missing") + ctx.load_cert_chain(hostcert, keyfile=hostkey) + + +def __loadProxy(ctx, proxyPath=None): + """Load proxy from proxyPath (or default location if not specified) and + set it as the certificate & key to use for this SSL context. + Returns None. + """ + if not proxyPath: + proxyPath = Locations.getProxyLocation() + if not proxyPath: + raise RuntimeError("Proxy location not set") + if not os.path.isfile(proxyPath): + raise RuntimeError(f"Proxy file ({proxyPath}) is missing") + # A proxy file contains the leaf certificate, its key, and the rest + # of the chain in a single file, which load_cert_chain handles natively + ctx.load_cert_chain(proxyPath) + + +def getSSLContext(**kwargs): + """Gets an ssl.SSLContext configured using the standard + DIRAC connection keywords from kwargs. The keywords are: + + - bServerMode: Boolean, if True the context is setup for a server + (hostcert is always used). + - useCertificates: Boolean, Set to true to use hostcerts in client + mode. + - proxyString: String, allow a literal proxy string to be provided. + - proxyLocation: String, Path to file to use as proxy, defaults to + usual location(s) if not set. + - skipCACheck: Boolean, if True, don't verify peer certificates. + - optionalClientCert: Boolean, in server mode, request but do not require + the client certificate (used by the HTTPS services, + where token/visitor authentication also exists). + - sslCiphers: String, OpenSSL style cipher string of ciphers to allow + on this connection. + + Returns the new context. + """ + serverMode = kwargs.get("bServerMode", False) + + if serverMode: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + else: + # PROTOCOL_TLS_CLIENT enables check_hostname and CERT_REQUIRED by default + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + + # Set certificates for connection + if serverMode or (kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False)): + # Server mode always uses hostcert + __loadHostCertificate(ctx) + + else: + # Client mode has a choice of possible options + if kwargs.get("proxyString", None): + # ssl cannot take an in-memory location or a string, + # so write it to a temp file and use proxyLocation + with tempfile.NamedTemporaryFile(mode="w") as tmpFile: + tmpFile.write(kwargs["proxyString"]) + # Flush, otherwise the file is empty in the subsequent call + tmpFile.flush() + __loadProxy(ctx, proxyPath=tmpFile.name) + else: + # Use normal proxy + __loadProxy(ctx, proxyPath=kwargs.get("proxyLocation", None)) + + # Set peer verification + if kwargs.get("skipCACheck", False): + # Don't validate peer + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + else: + # Do validate peer + if serverMode and kwargs.get("optionalClientCert", False): + ctx.verify_mode = ssl.CERT_OPTIONAL + else: + ctx.verify_mode = ssl.CERT_REQUIRED + # Allow proxy certificates to be used + ctx.verify_flags |= VERIFY_ALLOW_PROXY_CERTS + # Set CA location + caPath = Locations.getCAsLocation() + if not caPath: + raise RuntimeError("Failed to find CA location") + if not os.path.isdir(caPath): + raise RuntimeError(f"CA path ({caPath}) is not a valid directory") + ctx.load_verify_locations(capath=caPath) + + # DIRAC_M2CRYPTO_SSL_CIPHERS is accepted for backward compatibility + ciphers = ( + kwargs.get("sslCiphers") or os.environ.get("DIRAC_SSL_CIPHERS") or os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") + ) + if ciphers: + ctx.set_ciphers(ciphers) + + return ctx + + +# Client-side SSL contexts are cached and shared between connections, as +# building one means reading and parsing the credential files for every RPC. +# An entry is invalidated when the credential file changes (e.g. proxy +# renewal), and expires after _CLIENT_CTX_CACHE_TTL seconds so that changes +# in the CA directory are eventually picked up. +_CLIENT_CTX_CACHE: OrderedDict = OrderedDict() +_CLIENT_CTX_CACHE_LOCK = threading.Lock() +_CLIENT_CTX_CACHE_MAX = 8 +_CLIENT_CTX_CACHE_TTL = 300 + + +def _clientContextCacheKey(kwargs): + """Compute the cache key for a client SSL context, or None if the + configuration is not cacheable. The credential selection mirrors + :py:func:`getSSLContext`, and the key contains the stat info of the + credential files so the cache is invalidated when they change. + + :param kwargs: the connection keywords, as given to getSSLContext + """ + if kwargs.get("bServerMode") or kwargs.get("proxyString"): + return None + if kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False): + credentialFiles = Locations.getHostCertificateAndKeyLocation() + if not credentialFiles: + return None + else: + proxyPath = kwargs.get("proxyLocation") or Locations.getProxyLocation() + if not proxyPath: + return None + credentialFiles = (proxyPath,) + + fileStats = [] + try: + for path in credentialFiles: + st = os.stat(path) + fileStats.append((path, st.st_mtime_ns, st.st_size, st.st_ino)) + except OSError: + return None + + skipCACheck = bool(kwargs.get("skipCACheck", False)) + caPath = None if skipCACheck else Locations.getCAsLocation() + ciphers = ( + kwargs.get("sslCiphers") or os.environ.get("DIRAC_SSL_CIPHERS") or os.environ.get("DIRAC_M2CRYPTO_SSL_CIPHERS") + ) + return (tuple(fileStats), skipCACheck, caPath, ciphers) + + +def _getClientSSLContext(**kwargs): + """Return a client SSL context, shared from the cache whenever possible + (ssl.SSLContext objects are safe to share between connections and threads) + """ + cacheKey = _clientContextCacheKey(kwargs) + if cacheKey is None: + return getSSLContext(**kwargs) + + now = time.monotonic() + with _CLIENT_CTX_CACHE_LOCK: + cached = _CLIENT_CTX_CACHE.get(cacheKey) + if cached and now - cached[0] < _CLIENT_CTX_CACHE_TTL: + _CLIENT_CTX_CACHE.move_to_end(cacheKey) + return cached[1] + + ctx = getSSLContext(**kwargs) + + with _CLIENT_CTX_CACHE_LOCK: + _CLIENT_CTX_CACHE[cacheKey] = (now, ctx) + _CLIENT_CTX_CACHE.move_to_end(cacheKey) + while len(_CLIENT_CTX_CACHE) > _CLIENT_CTX_CACHE_MAX: + _CLIENT_CTX_CACHE.popitem(last=False) + return ctx + + +def getPeerInfo(sslSocket): + """Gets the details of the current peer as a standard dict. The peer + details are obtained from the supplied ssl.SSLSocket. + The details returned are those from ~X509Chain.getCredentials, without Registry info: + + DN - Full peer DN as string + x509Chain - Full chain of peer + isProxy - Boolean, True if chain ends with proxy + isLimitedProxy - Boolean, True if chain ends with limited proxy + group - String, DIRAC group for this peer, if known + + Returns a dict of details. + """ + if not hasattr(sslSocket, "get_unverified_chain"): + raise RuntimeError("Extracting the peer certificate chain requires python >= 3.13 (ssl get_unverified_chain)") + # The chain, as sent by the peer (leaf first). It was already validated + # by OpenSSL during the handshake, and the proxy-ness of it is + # anyway cryptographically re-checked by X509Chain + derChain = sslSocket.get_unverified_chain() + if not derChain: + return {} + chain = X509Chain.generateX509ChainFromDERList(derChain) + creds = chain.getCredentials(withRegistryInfo=False) + if not creds["OK"]: + raise RuntimeError(f"Failed to get SSL peer info ({creds['Message']}).") + peer = creds["Value"] + + # getCredentials already resolves DN (identity for proxies, subject + # otherwise), isProxy and isLimitedProxy + peer["x509Chain"] = chain + + return peer + + +class SSLTransport(BaseTransport): + """SSL Transport implementation based on the standard library ssl module.""" + + # This name is the same as BaseClient, + # and is used a bit everywhere, so it should be factorized out + # eventually + KW_TIMEOUT = "timeout" + + def __init__(self, *args, **kwargs): + """Create an SSLTransport object, parameters are the same + as for other transports. If ctx is specified (as an instance of + ssl.SSLContext) then use that rather than creating a new context. + + kwargs can contain all the parameters defined in BaseClient, + in particular timeout + """ + self.remoteAddress = None + self.peerCredentials = {} + + # The timeout is to be understood here as the timeout for socket + # operations involved in the RPC call, but NOT the establishment + # of the connection, for which there is a different timeout + # (DEFAULT_CONNECTION_TIMEOUT) + self.__timeout = kwargs.get(SSLTransport.KW_TIMEOUT, DEFAULT_RPC_TIMEOUT) + + self.__locked = False # We don't support locking, so this is always false. + + self.__ctx = kwargs.pop("ctx", None) + if not self.__ctx: + if kwargs.get("bServerMode", False): + self.__ctx = getSSLContext(**kwargs) + else: + self.__ctx = _getClientSSLContext(**kwargs) + + # Note that kwargs is already kept in BaseTransport + # as self.extraArgsDict, but at least I am sure that + # self.__kwargs will never be modified + self.__kwargs = kwargs + + BaseTransport.__init__(self, *args, **kwargs) + + def setSocketTimeout(self, timeout): + """Set the timeout for RPC calls. + + .. warning: This needs to be called before initAsClient. + It is used as a timeout for RPC calls, not connection. + + :param timeout: timeout for socket operation in seconds + + """ + self.__timeout = timeout + + def initAsClient(self): + """Prepare this client socket for use.""" + if self.serverMode(): + raise RuntimeError("SSLTransport is in server mode.") + + errors = [] + host, port = self.stServerAddress + + # Get all available addresses (IPv6 and IPv4) and try them in order + try: + addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except OSError as e: + return S_ERROR(f"DNS lookup failed {e!r}") + for family, socketType, proto, _canonname, socketAddress in addrInfoList: + rawSocket = None + try: + rawSocket = socket.socket(family, socketType, proto) + + # First set a short connection timeout, that will be used + # for the TCP connection and the TLS handshake + rawSocket.settimeout(DEFAULT_CONNECTION_TIMEOUT) + + # Enable keepAlive, with default options + # (see more comments about keepalive in :py:meth:`.acceptConnection`) + rawSocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + + rawSocket.connect(socketAddress) + + # server_hostname takes care of both SNI and the hostname check + self.oSocket = self.__ctx.wrap_socket(rawSocket, server_hostname=host) + + # Once the connection is established, we can use the timeout + # asked for RPC + self.oSocket.settimeout(self.__timeout) + + self.remoteAddress = self.oSocket.getpeername() + + return S_OK() + # warning: do NOT catch SSL related errors here + # They should be propagated upwards and caught by the BaseClient + # not to enter the retry loop + except ssl.SSLError: + if rawSocket is not None: + rawSocket.close() + raise + except OSError as e: + errors.append(f"{socketAddress} {e}:{repr(e)}") + if self.oSocket is not None: + self.close() + elif rawSocket is not None: + rawSocket.close() + + return S_ERROR("; ".join(errors)) + + def initAsServer(self): + """Prepare this server socket for use. + + Contrary to the client mode, the listening socket is a plain TCP + socket: the TLS layer is only added on the accepted connections, + in :py:meth:`.handshake` + """ + if not self.serverMode(): + raise RuntimeError("SSLTransport is in client mode.") + + try: + self.oSocket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + except OSError: + # Maybe no IPv6 support? Try IPv4 only socket. + self.oSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + # Make sure reuse address is set correctly + param = 1 if self.bAllowReuseAddress else 0 + self.oSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, param) + + self.oSocket.bind(self.stServerAddress) + self.oSocket.listen(self.iListenQueueSize) + return S_OK() + + def close(self): + """Close this socket.""" + if self.oSocket: + try: + self.oSocket.shutdown(socket.SHUT_RDWR) + except OSError: + # The socket may already be in a closed state + pass + self.oSocket.close() + self.oSocket = None + return S_OK() + + def renewServerContext(self): + """Renews the server context. + This reloads the certificates and the CA store by re-creating + the SSL context, which will be used for the subsequent connections. + """ + if not self.serverMode(): + raise RuntimeError("SSLTransport is in client mode.") + super().renewServerContext() + try: + self.__ctx = getSSLContext(**self.__kwargs) + except Exception as e: + gLogger.error("Failed to renew the server context", repr(e)) + return S_ERROR(f"Failed to renew the server context: {e!r}") + + return S_OK() + + def handshake(self): + """Perform the SSL handshake. + This has to be called after the connection was accepted (acceptConnection) + + The remote credentials are gathered here + """ + try: + # Warning: this method is called on the object returned + # by acceptConnection, which shares the context of the + # listening object + self.oSocket = self.__ctx.wrap_socket(self.oSocket, server_side=True) + + self.peerCredentials = getPeerInfo(self.oSocket) + + # Now that the handshake has been performed on the server + # we can set the timeout for the RPC operations. + # In practice, since we are on the server side, the + # timeout we set here represents the timeout for receiving + # the arguments and sending back the response. This should + # in principle be reasonably quick, but just to be sure + # we can set it to the DEFAULT_RPC_TIMEOUT + self.oSocket.settimeout(DEFAULT_RPC_TIMEOUT) + + return S_OK() + except (OSError, RuntimeError) as e: + return S_ERROR(f"Error in handshake: {e} {repr(e)}") + + def setClientSocket(self, oSocket): + """Set the inner socket of this instance to the value of oSocket. + This method is intended to be used to create client connection objects + from a server and should be considered to be an internal function. + + :param oSocket: client socket (plain, the handshake is done in :py:meth:`.handshake`) + + """ + # warning: do NOT catch socket.error here, because for who knows what reason + # exceptions are actually properly used for once, and the calling method + # relies on it (ServiceReactor.__acceptIncomingConnection) + self.oSocket = oSocket + self.remoteAddress = self.oSocket.getpeername() + + def acceptConnection(self): + """Accept a new client, returns a new SSLTransport object representing + the client connection. + + The connection is accepted, but no SSL handshake is performed + + :returns: S_OK(SSLTransport object) + """ + try: + oClient, _ = self.oSocket.accept() + + # Set the keep alive to true. This keepalive will ensure that we + # detect remote peer crashing or network interruption + # Note that this is ineffective if we are in the middle of blocking + # operations. + oClient.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) + + # Here we set the timeout server side. + # We first set the connection timeout, which will + # be effective for the TLS handshake + oClient.settimeout(DEFAULT_CONNECTION_TIMEOUT) + + oClientTrans = SSLTransport(self.stServerAddress, ctx=self.__ctx) + oClientTrans.setClientSocket(oClient) + return S_OK(oClientTrans) + except (OSError, RuntimeError) as e: + return S_ERROR(f"Error in acceptConnection: {e} {repr(e)}") + + def _read(self, bufSize=4096, skipReadyCheck=False): + """Read bufSize bytes from the buffer. + + :param bufSize: size of the buffer to read + :param skipReadyCheck: ignored. + + + :returns: S_OK(data read) + """ + try: + read = self.oSocket.recv(bufSize) + return S_OK(read) + except OSError as e: + return S_ERROR(f"Error in _read: {e} {repr(e)}") + + def isLocked(self): + """Returns if this instance is locked. + Always returns false. + + :returns: False + """ + return self.__locked + + def _write(self, buf): + """Write all bytes contained within iterable "buf" to the + connected peer. + + :param buf: iterable buffer + + :returns: S_OK(number of bytes written) + """ + try: + wrote = self.oSocket.send(buf) + return S_OK(wrote) + except OSError as e: + return S_ERROR(f"Error in _write: {e} {repr(e)}") diff --git a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py index 45494caf2cc..edbfeed2cc8 100644 --- a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py @@ -7,7 +7,7 @@ from pytest import fixture from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.Core.DISET.private.Transports import PlainTransport, SSLTransport +from DIRAC.Core.DISET.private.Transports import PlainTransport, StdSSLTransport from DIRAC.Core.Security.test.x509TestUtilities import CERTDIR, USERCERT, getCertOption # TODO: Expired hostcert @@ -122,7 +122,7 @@ def transportByName(transport): if transport.lower() == "plain": return PlainTransport.PlainTransport elif transport.lower() == "ssl": - return SSLTransport.SSLTransport + return StdSSLTransport.SSLTransport raise RuntimeError(f"Unknown Transport Name: {transport}") @@ -200,7 +200,7 @@ def test_simpleMessage(create_serverAndClient): def test_clientContextCache(tmp_path): """Client SSL contexts are shared until the credential file changes""" - from DIRAC.Core.DISET.private.Transports.SSLTransport import _CLIENT_CTX_CACHE, _getClientSSLContext + from DIRAC.Core.DISET.private.Transports.StdSSLTransport import _CLIENT_CTX_CACHE, _getClientSSLContext proxyCopy = tmp_path / "proxy.pem" proxyCopy.write_bytes(open(proxyFile, "rb").read()) diff --git a/src/DIRAC/Core/Security/__init__.py b/src/DIRAC/Core/Security/__init__.py index f85e887a41a..16784f1aff7 100644 --- a/src/DIRAC/Core/Security/__init__.py +++ b/src/DIRAC/Core/Security/__init__.py @@ -1,10 +1,18 @@ """ -DIRAC X509 security infrastructure, based on pyca/cryptography. +DIRAC X509 security infrastructure. This package provides the X509Certificate, X509Chain, X509Request and X509CRL classes as submodules, as well as a set of OID constants used for handling proxies and VOMS extensions. + +Two implementations of the X509 classes are available: the default one, based +on pyca/cryptography (in the ``pyca`` subpackage), and the legacy one, based +on M2Crypto and pyasn1 (in the ``m2crypto`` subpackage). The implementation is +selected once, at import time, with the DIRAC_USE_M2CRYPTO environment +variable (default No). """ +import os +from pkgutil import extend_path # List of OIDs used in handling VOMS extension. # VOMS extension is encoded in ASN.1 format and it's surprisingly hard to decode. OIDs describe content of sections @@ -54,3 +62,33 @@ #: Default strength of the proxy in bit DEFAULT_PROXY_STRENGTH = 2048 + + +##### +# SUPER DISGUSTING HACK +# We define these variables, and then remove them immediately. +# it is to allow something like 'from DIRAC.Core.Security import X509Chain' +# But pylint would complain just like that +# I've spent a lot of time trying to get pylint to work, but... +# https://github.com/PyCQA/pylint/issues/2474 + +X509Chain = None +X509CRL = None +X509Certificate = None +X509Request = None + +locals().pop("X509Chain") +locals().pop("X509CRL") +locals().pop("X509Certificate") +locals().pop("X509Request") +#### + + +# If we want to use M2Crypto, we add the m2crypto subpackage to the search path, +# otherwise the default pyca/cryptography based subpackage is used. +# This allows imports like 'from DIRAC.Core.Security.X509Chain...' to work transparently +# Nice kind of tricks you find in libraries like xml... +if os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true"): + __path__ = extend_path(__path__, __name__ + ".m2crypto") +else: + __path__ = extend_path(__path__, __name__ + ".pyca") diff --git a/src/DIRAC/Core/Security/X509CRL.py b/src/DIRAC/Core/Security/pyca/X509CRL.py similarity index 98% rename from src/DIRAC/Core/Security/X509CRL.py rename to src/DIRAC/Core/Security/pyca/X509CRL.py index fafc0550498..b561403e087 100644 --- a/src/DIRAC/Core/Security/X509CRL.py +++ b/src/DIRAC/Core/Security/pyca/X509CRL.py @@ -6,7 +6,7 @@ from cryptography import x509 from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Security import asn1_utils +from DIRAC.Core.Security.pyca import asn1_utils from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.File import secureOpenForWrite diff --git a/src/DIRAC/Core/Security/X509Certificate.py b/src/DIRAC/Core/Security/pyca/X509Certificate.py similarity index 99% rename from src/DIRAC/Core/Security/X509Certificate.py rename to src/DIRAC/Core/Security/pyca/X509Certificate.py index a4a14c75b75..44cb5bcbccb 100644 --- a/src/DIRAC/Core/Security/X509Certificate.py +++ b/src/DIRAC/Core/Security/pyca/X509Certificate.py @@ -18,7 +18,7 @@ from DIRAC.Core.Utilities import DErrno from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.Core.Security import DEFAULT_PROXY_STRENGTH, DIRAC_GROUP_OID, PROXY_CERT_INFO_EXTENSION_OID -from DIRAC.Core.Security import asn1_utils +from DIRAC.Core.Security.pyca import asn1_utils from DIRAC.Core.Utilities.Decorators import executeOnlyIf # Decorator to execute the method only of the certificate has been loaded diff --git a/src/DIRAC/Core/Security/X509Chain.py b/src/DIRAC/Core/Security/pyca/X509Chain.py similarity index 99% rename from src/DIRAC/Core/Security/X509Chain.py rename to src/DIRAC/Core/Security/pyca/X509Chain.py index 061ec4ae1e0..d336610bf2d 100644 --- a/src/DIRAC/Core/Security/X509Chain.py +++ b/src/DIRAC/Core/Security/pyca/X509Chain.py @@ -25,8 +25,8 @@ PROXY_CERT_INFO_EXTENSION_OID, PROXY_OID, ) -from DIRAC.Core.Security import asn1_utils -from DIRAC.Core.Security.X509Certificate import X509Certificate +from DIRAC.Core.Security.pyca import asn1_utils +from DIRAC.Core.Security.pyca.X509Certificate import X509Certificate from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.Decorators import executeOnlyIf from DIRAC.Core.Utilities.File import secureOpenForWrite diff --git a/src/DIRAC/Core/Security/X509Request.py b/src/DIRAC/Core/Security/pyca/X509Request.py similarity index 97% rename from src/DIRAC/Core/Security/X509Request.py rename to src/DIRAC/Core/Security/pyca/X509Request.py index 0bdf9130922..ab486b3e7f2 100644 --- a/src/DIRAC/Core/Security/X509Request.py +++ b/src/DIRAC/Core/Security/pyca/X509Request.py @@ -10,8 +10,8 @@ from DIRAC import S_OK, S_ERROR from DIRAC.Core.Security import DEFAULT_PROXY_STRENGTH -from DIRAC.Core.Security import asn1_utils -from DIRAC.Core.Security.X509Chain import _PEM_KEY_PATTERN, dumpPrivateKeyPEM +from DIRAC.Core.Security.pyca import asn1_utils +from DIRAC.Core.Security.pyca.X509Chain import _PEM_KEY_PATTERN, dumpPrivateKeyPEM from DIRAC.Core.Utilities import DErrno # pylint: disable=broad-except diff --git a/src/DIRAC/Core/Security/pyca/__init__.py b/src/DIRAC/Core/Security/pyca/__init__.py new file mode 100644 index 00000000000..506e4ce4758 --- /dev/null +++ b/src/DIRAC/Core/Security/pyca/__init__.py @@ -0,0 +1 @@ +"""X509 implementation based on pyca/cryptography and the standard library ssl module""" diff --git a/src/DIRAC/Core/Security/asn1_utils.py b/src/DIRAC/Core/Security/pyca/asn1_utils.py similarity index 100% rename from src/DIRAC/Core/Security/asn1_utils.py rename to src/DIRAC/Core/Security/pyca/asn1_utils.py diff --git a/src/DIRAC/Core/Tornado/Server/TornadoServer.py b/src/DIRAC/Core/Tornado/Server/TornadoServer.py index 12aa1f84091..8361de02651 100644 --- a/src/DIRAC/Core/Tornado/Server/TornadoServer.py +++ b/src/DIRAC/Core/Tornado/Server/TornadoServer.py @@ -9,6 +9,22 @@ import psutil import secrets +# With the legacy M2Crypto implementation (DIRAC_USE_M2CRYPTO=Yes), tornado +# must be configured to use the M2Crypto based iostream (from +# tornado_m2crypto, which requires the patched DIRACGrid tornado fork) before +# anything instantiates an SSLIOStream. With the default implementation +# neither M2Crypto nor tornado_m2crypto may be imported. +DIRAC_USE_M2CRYPTO = os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true") + +if DIRAC_USE_M2CRYPTO: + import M2Crypto.SSL + + import tornado.iostream + + tornado.iostream.SSLIOStream.configure( # pylint: disable=no-member + "tornado_m2crypto.m2iostream.M2IOStream" + ) # pylint: disable=wrong-import-position + import tornado.platform.asyncio import tornado.ioloop from tornado.httpserver import HTTPServer @@ -22,6 +38,7 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations sLog = gLogger.getSubLogger(__name__) +DEBUG_M2CRYPTO = os.getenv("DIRAC_DEBUG_M2CRYPTO", "No").lower() in ("yes", "true") class NotFoundHandler(RequestHandler): @@ -180,16 +197,30 @@ def startTornado(self): sLog.debug("Starting Tornado") - # Prepare SSL settings. - # The client certificate is optional: some requests are authenticated - # with a token or as visitor instead (see BaseRequestHandler) - try: - ssl_options = getSSLContext(bServerMode=True, optionalClientCert=True) - except RuntimeError as e: - sLog.fatal("Unable to prepare the TLS settings ! Can't start the Server", repr(e)) - raise ImportError(f"Unable to prepare the TLS settings: {e}") from e - certs = Locations.getHostCertificateAndKeyLocation() - ca = Locations.getCAsLocation() + # Prepare SSL settings + if DIRAC_USE_M2CRYPTO: + certs = Locations.getHostCertificateAndKeyLocation() + if certs is False: + sLog.fatal("Host certificates not found ! Can't start the Server") + raise ImportError("Unable to load certificates") + ca = Locations.getCAsLocation() + ssl_options = { + "certfile": certs[0], + "keyfile": certs[1], + "cert_reqs": M2Crypto.SSL.verify_peer, + "ca_certs": ca, + "sslDebug": DEBUG_M2CRYPTO, # Set to true if you want to see the TLS debug messages + } + else: + # The client certificate is optional: some requests are authenticated + # with a token or as visitor instead (see BaseRequestHandler) + try: + ssl_options = getSSLContext(bServerMode=True, optionalClientCert=True) + except RuntimeError as e: + sLog.fatal("Unable to prepare the TLS settings ! Can't start the Server", repr(e)) + raise ImportError(f"Unable to prepare the TLS settings: {e}") from e + certs = Locations.getHostCertificateAndKeyLocation() + ca = Locations.getCAsLocation() # Init monitoring if self.activityMonitoring: diff --git a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py index b021b1c5514..d08cbac8298 100644 --- a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py +++ b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py @@ -4,6 +4,7 @@ This module is basic for each of these components and describes the basic concept of access to them. """ +import os import time import inspect import threading @@ -28,6 +29,10 @@ from DIRAC.Resources.IdProvider.Utilities import getIdProviderIdentifiers from DIRAC.Resources.IdProvider.IdProviderFactory import IdProviderFactory +# Whether the legacy M2Crypto based TLS implementation is used instead of the +# default one, based on the standard library ssl module (see TornadoServer) +DIRAC_USE_M2CRYPTO = os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true") + def set_attribute(attr, val): """Decorator to determine target method settings. Set method attribute. @@ -735,6 +740,9 @@ def _authzSSL(self): :return: S_OK(dict)/S_ERROR() """ + if DIRAC_USE_M2CRYPTO: + return self._authzSSLM2() + try: # The peer chain, as sent by the client (leaf first, DER encoded). # It was validated by OpenSSL during the TLS handshake @@ -777,6 +785,68 @@ def _authzSSL(self): credDict["extraCredentials"] = self.decode(extraCred)[0] return S_OK(credDict) + def _authzSSLM2(self): + """Load client certchain in DIRAC and extract information, using the + legacy M2Crypto based TLS implementation (DIRAC_USE_M2CRYPTO=Yes). + + :return: S_OK(dict)/S_ERROR() + """ + try: + derCert = self.request.get_ssl_certificate() + except Exception: + # If 'IOStream' object has no attribute 'get_ssl_certificate' + derCert = None + + # Boolean whether we are behind a balancer and can trust headers + balancer = gConfig.getValue("/WebApp/Balancer", "none") != "none" + + # Get client certificate as pem + if derCert: + chainAsText = derCert.as_pem().decode("ascii") + # Read all certificate chain + chainAsText += "".join([cert.as_pem().decode("ascii") for cert in self.request.get_ssl_certificate_chain()]) + elif balancer: + if self.request.headers.get("X-Ssl_client_verify") == "SUCCESS" and self.request.headers.get("X-SSL-CERT"): + chainAsText = unquote(self.request.headers.get("X-SSL-CERT")) + else: + return S_ERROR(DErrno.ECERTFIND, "Valid certificate not found.") + else: + return S_ERROR(DErrno.ECERTFIND, "Valid certificate not found.") + + # Load full certificate chain + peerChain = X509Chain() + peerChain.loadChainFromString(chainAsText) + + # Retrieve the credentials + res = peerChain.getCredentials(withRegistryInfo=False) + if not res["OK"]: + return res + + credDict = res["Value"] + + credDict["x509Chain"] = peerChain + res = peerChain.isProxy() + if not res["OK"]: + return res + credDict["isProxy"] = res["Value"] + + if credDict["isProxy"]: + credDict["DN"] = credDict["identity"] + else: + credDict["DN"] = credDict["subject"] + + res = peerChain.isLimitedProxy() + if not res["OK"]: + return res + credDict["isLimitedProxy"] = res["Value"] + + # We check if client sends extra credentials... + if "extraCredentials" in self.request.arguments: + extraCred = self.get_argument("extraCredentials") + if extraCred: + credDict["extraCredentials"] = self.decode(extraCred)[0] + return S_OK(credDict) + def _authzJWT(self, accessToken=None): """Load token claims in DIRAC and extract information. diff --git a/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py b/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py index 218ab5a64c3..76e59ffa265 100644 --- a/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py +++ b/src/DIRAC/Resources/ProxyProvider/DIRACCAProxyProvider.py @@ -30,445 +30,855 @@ SERIALNUMBER(serialNumber) """ +import os import re import secrets import datetime import collections -from cryptography import x509 -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa - from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.Security import asn1_utils from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Resources.ProxyProvider.ProxyProvider import ProxyProvider -# Mapping between the distinguished name aliases and their OID, replacing -# the M2Crypto X509_Name.nid lookup table -_FIELDS_TO_OID = { - "C": "2.5.4.6", - "countryName": "2.5.4.6", - "SP": "2.5.4.8", - "ST": "2.5.4.8", - "stateOrProvinceName": "2.5.4.8", - "L": "2.5.4.7", - "localityName": "2.5.4.7", - "O": "2.5.4.10", - "organizationName": "2.5.4.10", - "OU": "2.5.4.11", - "organizationalUnitName": "2.5.4.11", - "CN": "2.5.4.3", - "commonName": "2.5.4.3", - "Email": "1.2.840.113549.1.9.1", - "emailAddress": "1.2.840.113549.1.9.1", - "SN": "2.5.4.4", - "surname": "2.5.4.4", - "GN": "2.5.4.42", - "givenName": "2.5.4.42", - "serialNumber": "2.5.4.5", - "SERIALNUMBER": "2.5.4.5", - "DC": "0.9.2342.19200300.100.1.25", - "domainComponent": "0.9.2342.19200300.100.1.25", -} - -_ALGORITHMS = { - "sha1": hashes.SHA1, - "sha224": hashes.SHA224, - "sha256": hashes.SHA256, - "sha384": hashes.SHA384, - "sha512": hashes.SHA512, -} - - -class DIRACCAProxyProvider(ProxyProvider): - def __init__(self, parameters=None): - """Constructor""" - super().__init__(parameters) - self.log = gLogger.getSubLogger(__name__) - # Initialize - self.maxDict = {} - self.minDict = {} - self.bits = 2048 - self.algoritm = "sha256" - self.match = [] - self.supplied = [_FIELDS_TO_OID["CN"]] - self.optional = [_FIELDS_TO_OID[f] for f in ("C", "O", "OU", "emailAddress")] - self.dnList = ["C", "O", "OU", "CN", "emailAddress"] - # Distinguished names - self.fields2oid = dict(_FIELDS_TO_OID) - self.oid2fields = {} # oid: list of distinguished names - # Specify standart fields - for field, oid in self.fields2oid.items(): - self.oid2fields.setdefault(oid, []).append(field) - self.dnInfoDictCA = {} - # List of x509.NameAttribute of the DN being built - self.__nameAttributes = [] - - def setParameters(self, parameters): - """Set new parameters - - :param dict parameters: provider parameters - - :return: S_OK()/S_ERROR() - """ - for k, v in parameters.items(): - if not isinstance(v, list) and k in ["Match", "Supplied", "Optional", "DNOrder"] + list(self.fields2oid): - parameters[k] = v.replace(", ", ",").split(",") - self.parameters = parameters - # If CA configuration file exist - if parameters.get("CAConfigFile"): - self.__parseCACFG() - if "Bits" in parameters: - self.bits = int(parameters["Bits"]) - if "Algoritm" in parameters: - self.algoritm = parameters["Algoritm"] - if "Match" in parameters: - self.match = [self.fields2oid[f] for f in parameters["Match"]] - if "Supplied" in parameters: - self.supplied = [self.fields2oid[f] for f in parameters["Supplied"]] - if "Optional" in parameters: - self.optional = [self.fields2oid[f] for f in parameters["Optional"]] - allFields = self.optional + self.supplied + self.match - if "DNOrder" in parameters: - self.dnList = [] - if not any([any([f in parameters["DNOrder"] for f in self.oid2fields[n]]) for n in allFields]): - return S_ERROR("DNOrder must contain all configured fields.") - for field in parameters["DNOrder"]: - if self.fields2oid[field] in allFields: - self.dnList.append(field) - - # Set defaults for distridutes names - self.oid2defField = {} - for field, value in list(self.parameters.items()): - if field in self.fields2oid and self.fields2oid[field] in allFields: - self.parameters[self.fields2oid[field]] = value - self.oid2defField[self.fields2oid[field]] = field - - # Read CA certificate - chain = X509Chain() - result = chain.loadChainFromFile(self.parameters["CertFile"]) - if result["OK"]: - result = chain.getCredentials() +# Two implementations are available: the default one, based on +# pyca/cryptography, and the legacy one, based on M2Crypto. The same public +# name (DIRACCAProxyProvider) is defined once, at import time, based on the +# DIRAC_USE_M2CRYPTO environment variable, matching the implementation used +# for the rest of the X509 layer (see DIRAC.Core.Security). M2Crypto may not +# be imported unless the fallback is enabled. +if os.getenv("DIRAC_USE_M2CRYPTO", "No").lower() in ("yes", "true"): + import time + + from M2Crypto import m2, util, X509, ASN1, EVP, RSA + + class DIRACCAProxyProvider(ProxyProvider): + def __init__(self, parameters=None): + """Constructor""" + super().__init__(parameters) + self.log = gLogger.getSubLogger(__name__) + # Initialize + self.maxDict = {} + self.minDict = {} + self.bits = 2048 + self.algoritm = "sha256" + self.match = [] + self.supplied = ["CN"] + self.optional = ["C", "O", "OU", "emailAddress"] + self.dnList = ["C", "O", "OU", "CN", "emailAddress"] + # Distinguished names + self.fields2nid = X509.X509_Name.nid.copy() + self.fields2nid["DC"] = -1 # Add DN that is not liested in X509.X509_Name + self.fields2nid["domainComponent"] = -1 # Add DN description that is not liested in X509.X509_Name + self.fields2nid["organizationalUnitName"] = 18 # Add 'OU' description + self.fields2nid["countryName"] = 14 # Add 'C' description + self.fields2nid["SERIALNUMBER"] = 105 # Add 'SERIALNUMBER' distinguished name + self.nid2fields = {} # nid: list of distinguished names + # Specify standart fields + for field, nid in self.fields2nid.items(): + self.nid2fields.setdefault(nid, []).append(field) + self.dnInfoDictCA = {} + + def setParameters(self, parameters): + """Set new parameters + + :param dict parameters: provider parameters + + :return: S_OK()/S_ERROR() + """ + for k, v in parameters.items(): + if not isinstance(v, list) and k in ["Match", "Supplied", "Optional", "DNOrder"] + list( + self.fields2nid + ): + parameters[k] = v.replace(", ", ",").split(",") + self.parameters = parameters + # If CA configuration file exist + if parameters.get("CAConfigFile"): + self.__parseCACFG() + if "Bits" in parameters: + self.bits = int(parameters["Bits"]) + if "Algoritm" in parameters: + self.algoritm = parameters["Algoritm"] + if "Match" in parameters: + self.match = [self.fields2nid[f] for f in parameters["Match"]] + if "Supplied" in parameters: + self.supplied = [self.fields2nid[f] for f in parameters["Supplied"]] + if "Optional" in parameters: + self.optional = [self.fields2nid[f] for f in parameters["Optional"]] + allFields = self.optional + self.supplied + self.match + if "DNOrder" in parameters: + self.dnList = [] + if not any([any([f in parameters["DNOrder"] for f in self.nid2fields[n]]) for n in allFields]): + return S_ERROR("DNOrder must contain all configured fields.") + for field in parameters["DNOrder"]: + if self.fields2nid[field] in allFields: + self.dnList.append(field) + + # Set defaults for distridutes names + self.nid2defField = {} + for field, value in list(self.parameters.items()): + if field in self.fields2nid and self.fields2nid[field] in allFields: + self.parameters[self.fields2nid[field]] = value + self.nid2defField[self.fields2nid[field]] = field + + # Read CA certificate + chain = X509Chain() + result = chain.loadChainFromFile(self.parameters["CertFile"]) if result["OK"]: - result = self.__parseDN(result["Value"]["subject"]) - if not result["OK"]: - return result - self.dnInfoDictCA = result["Value"] - return S_OK() + result = chain.getCredentials() + if result["OK"]: + result = self.__parseDN(result["Value"]["subject"]) + if not result["OK"]: + return result + self.dnInfoDictCA = result["Value"] + return S_OK() - def checkStatus(self, userDN): - """Read ready to work status of proxy provider + def checkStatus(self, userDN): + """Read ready to work status of proxy provider - :param str userDN: user DN + :param str userDN: user DN - :return: S_OK()/S_ERROR() - """ - self.log.debug("Ckecking work status of", self.parameters["ProviderName"]) - result = self.__parseDN(userDN) - if not result["OK"]: - return result - dnInfoDict = result["Value"] - - try: - userOIDs = [self.fields2oid[f.split("=")[0]] for f in userDN.lstrip("/").split("/")] - except (ValueError, KeyError) as e: - return S_ERROR(f"Unknown DN field in used DN: {e}") - oidOrder = [self.fields2oid[f] for f in self.dnList] - for index, oid in enumerate(userOIDs): - if oid not in oidOrder: - return S_ERROR( - f'"{self.oid2defField.get(oid, min(self.oid2fields[oid], key=len))}" field not found in order.' - ) - if index > oidOrder.index(oid): - return S_ERROR("Bad DNs order") - for i in range(oidOrder.index(oid) - 1): - try: - if userOIDs.index(oidOrder[i]) > index: - return S_ERROR("Bad DNs order") - except (ValueError, KeyError): - continue - for i in range(oidOrder.index(oid) + 1, len(oidOrder)): - try: - if userOIDs.index(oidOrder[i]) < index: - return S_ERROR("Bad DNs order") - except (ValueError, KeyError): - continue - - for oid in self.supplied: - if oid not in [self.fields2oid[f] for f in dnInfoDict]: - return S_ERROR( - 'Current DN is invalid, "%s" field must be set.' - % self.oid2defField.get(oid, min(self.oid2fields[oid], key=len)) - ) - - for field, values in dnInfoDict.items(): - oid = self.fields2oid[field] - err = f'Current DN is invalid, "{field}" field' - if oid not in self.supplied + self.match + self.optional: - return S_ERROR(f"{err} is not found for current CA.") - if oid in self.match and not self.dnInfoDictCA[field] == values: - return S_ERROR(f"{err} must be /{field}={('/%s=' % field).joing(self.dnInfoDictCA[field])}.") - if oid in self.maxDict: - rangeMax = list(range(min(len(values), len(self.maxDict[oid])))) - if any([True if len(values[i]) > self.maxDict[oid][i] else False for i in rangeMax]): - return S_ERROR(f"{err} values must be less then {', '.join(self.maxDict[oid])}.") - if oid in self.minDict: - rangeMin = list(range(min(len(values), len(self.minDict[oid])))) - if any([True if len(values[i]) < self.minDict[oid][i] else False for i in rangeMin]): - return S_ERROR(f"{err} values must be more then {', '.join(self.minDict[oid])}.") - - result = self.__fillX509Name(field, values) + :return: S_OK()/S_ERROR() + """ + self.log.debug("Ckecking work status of", self.parameters["ProviderName"]) + result = self.__parseDN(userDN) if not result["OK"]: return result + dnInfoDict = result["Value"] + + try: + userNIDs = [self.fields2nid[f.split("=")[0]] for f in userDN.lstrip("/").split("/")] + except (ValueError, KeyError) as e: + return S_ERROR(f"Unknown DN field in used DN: {e}") + nidOrder = [self.fields2nid[f] for f in self.dnList] + for index, nid in enumerate(userNIDs): + if nid not in nidOrder: + return S_ERROR( + f'"{self.nid2defField.get(nid, min(self.nid2fields[nid], key=len))}" field not found in order.' + ) + if index > nidOrder.index(nid): + return S_ERROR("Bad DNs order") + for i in range(nidOrder.index(nid) - 1): + try: + if userNIDs.index(nidOrder[i]) > index: + return S_ERROR("Bad DNs order") + except (ValueError, KeyError): + continue + for i in range(nidOrder.index(nid) + 1, len(nidOrder)): + try: + if userNIDs.index(nidOrder[i]) < index: + return S_ERROR("Bad DNs order") + except (ValueError, KeyError): + continue + + for nid in self.supplied: + if nid not in [self.fields2nid[f] for f in dnInfoDict]: + return S_ERROR( + 'Current DN is invalid, "%s" field must be set.' + % self.nid2defField.get(nid, min(self.nid2fields[nid], key=len)) + ) - return S_OK() + for field, values in dnInfoDict.items(): + nid = self.fields2nid[field] + err = f'Current DN is invalid, "{field}" field' + if nid not in self.supplied + self.match + self.optional: + return S_ERROR(f"{err} is not found for current CA.") + if nid in self.match and not self.dnInfoDictCA[field] == values: + return S_ERROR(f"{err} must be /{field}={('/%s=' % field).joing(self.dnInfoDictCA[field])}.") + if nid in self.maxDict: + rangeMax = list(range(min(len(values), len(self.maxDict[nid])))) + if any([True if len(values[i]) > self.maxDict[nid][i] else False for i in rangeMax]): + return S_ERROR(f"{err} values must be less then {', '.join(self.maxDict[nid])}.") + if nid in self.minDict: + rangeMin = list(range(min(len(values), len(self.minDict[nid])))) + if any([True if len(values[i]) < self.minDict[nid][i] else False for i in rangeMin]): + return S_ERROR(f"{err} values must be more then {', '.join(self.minDict[nid])}.") + + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + return S_OK() + + def getProxy(self, userDN): + """Generate user proxy + + :param str userDN: user DN + + :return: S_OK(str)/S_ERROR() -- contain a proxy string + """ + self.__X509Name = X509.X509_Name() + result = self.checkStatus(userDN) + if result["OK"]: + result = self.__createCertM2Crypto() + if result["OK"]: + certStr, keyStr = result["Value"] + + chain = X509Chain() + result = chain.loadChainFromString(certStr) + if result["OK"]: + result = chain.loadKeyFromString(keyStr) + if result["OK"]: + result = chain.generateProxyToString(365 * 24 * 3600) - def getProxy(self, userDN): - """Generate user proxy + return result - :param str userDN: user DN + def generateDN(self, **kwargs): + """Get DN of the user certificate that will be created - :return: S_OK(str)/S_ERROR() -- contain a proxy string - """ - self.__nameAttributes = [] - result = self.checkStatus(userDN) - if result["OK"]: - result = self.__createCertificate() + :param dict kwargs: user description dictionary with possible fields: + - FullName or CN + - Email or emailAddress + + :return: S_OK(str)/S_ERROR() -- contain DN + """ + if kwargs.get("FullName"): + kwargs["CN"] = [kwargs["FullName"]] + if kwargs.get("Email"): + kwargs["emailAddress"] = [kwargs["Email"]] + + self.__X509Name = X509.X509_Name() + self.log.info("Creating distinguished names chain") + + for nid in self.supplied: + if nid not in [self.fields2nid[f] for f in self.dnList]: + return S_ERROR( + 'DNs order list does not contain supplied DN "%s"' + % self.nid2defField.get(nid, min(self.nid2fields[nid], key=len)) + ) + + for field in self.dnList: + values = [] + nid = self.fields2nid[field] + if nid in self.match: + for field in self.nid2fields[nid]: + if field in self.dnInfoDictCA: + values = self.dnInfoDictCA[field] + if not values: + return S_ERROR(f'Not found "{field}" match DN in CA') + for field in self.nid2fields[nid]: + if kwargs.get(field): + values = kwargs[field] if isinstance(kwargs[field], list) else [kwargs[field]] + if not values and nid in self.supplied: + # Search default value + if nid not in self.nid2defField: + return S_ERROR(f'No values set for "{min(self.nid2fields[nid], key=len)}" DN') + values = self.parameters[nid] + + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + # WARN: This logic not support list of distribtes name elements + resDN = m2.x509_name_oneline(self.__X509Name.x509_name) # pylint: disable=no-member + + result = self.checkStatus(resDN) + if not result["OK"]: + return result + return S_OK(resDN) + + def __parseCACFG(self): + """Parse CA configuration file""" + block = "" + self.cfg = {} + self.supplied, self.optional, self.match, self.dnList = [], [], [], [] + with open(self.parameters["CAConfigFile"]) as caCFG: + for line in caCFG: + # Ignore comments + line = re.sub(r"#.*", "", line) + if re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", "")): + block = "".join(re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", ""))) + if block not in self.cfg: + self.cfg[block] = {} + if not block: + continue + if len(re.findall("=", line)) == 1: + field, val = line.split("=") + field = field.strip() + variables = re.findall(r"[$]([A-Za-z0-9_]+)", val) + for v in variables: + for b in self.cfg: + if v in self.cfg[b]: + val = val.replace("$" + v, self.cfg[b][v]) + if "default_ca" in self.cfg.get("ca", {}): + if "policy" in self.cfg.get(self.cfg["ca"]["default_ca"], {}): + if block == self.cfg[self.cfg["ca"]["default_ca"]]["policy"]: + self.dnList.append(field) + self.cfg[block][field] = val.strip() + + self.bits = int(self.cfg["req"].get("default_bits") or self.bits) + self.algoritm = self.cfg[self.cfg["ca"]["default_ca"]].get("default_md") or self.algoritm + if not self.parameters.get("CertFile"): + self.parameters["CertFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["certificate"] + self.parameters["KeyFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["private_key"] + # Read distinguished names + for k, v in self.cfg[self.cfg[self.cfg["ca"]["default_ca"]]["policy"]].items(): + nid = self.fields2nid[k] + self.parameters[nid], self.minDict[nid], self.maxDict[nid] = [], [], [] + for k in [f"{i}.{k}" for i in range(0, 5)] + [k]: + if k + "_default" in self.cfg["req"]["distinguished_name"]: + self.parameters[nid].append(self.cfg["req"]["distinguished_name"][k + "_default"]) + if k + "_min" in self.cfg["req"]["distinguished_name"]: + self.minDict[nid].append(self.cfg["req"]["distinguished_name"][k + "_min"]) + if k + "_max" in self.cfg["req"]["distinguished_name"]: + self.maxDict[nid].append(self.cfg["req"]["distinguished_name"][k + "_max"]) + if v == "supplied": + self.supplied.append(nid) + elif v == "optional": + self.optional.append(nid) + elif v == "match": + self.match.append(nid) + + def __parseDN(self, dn): + """Return DN fields + + :param str dn: DN + + :return: list -- contain tuple with positionOfField.fieldName, fieldNID, fieldValue + """ + dnInfoDict = collections.OrderedDict() + for f, v in [f.split("=") for f in dn.lstrip("/").split("/")]: + if not v: + return S_ERROR(f'No value set for "{f}"') + if f not in dnInfoDict: + dnInfoDict[f] = [v] + else: + dnInfoDict[f].append(v) + return S_OK(dnInfoDict) + + def __fillX509Name(self, field, values): + """Fill x509_Name object by M2Crypto + + :param str field: DN field name + :param list values: values of field, order important + + :return: S_OK()/S_ERROR() + """ + for value in values: + if ( + value + and m2.x509_name_set_by_nid( # pylint: disable=no-member + self.__X509Name.x509_name, self.fields2nid[field], value.encode() + ) + == 0 + ): + if ( + not self.__X509Name.add_entry_by_txt( + field=field, type=ASN1.MBSTRING_ASC, entry=value, len=-1, loc=-1, set=0 + ) + == 1 + ): + return S_ERROR(f'Cannot set "{field}" field.') + return S_OK() + + def __createCertM2Crypto(self): + """Create new certificate for user + + :return: S_OK(tuple)/S_ERROR() -- tuple contain certificate and pulic key as strings + """ + # Create public key + userPubKey = EVP.PKey() + userPubKey.assign_rsa(RSA.gen_key(self.bits, 65537, util.quiet_genparam_callback)) + # Create certificate + userCert = X509.X509() + userCert.set_pubkey(userPubKey) + userCert.set_version(2) + userCert.set_subject(self.__X509Name) + userCert.set_serial_number(secrets.randbits(64)) + # Add extentionals + userCert.add_ext(X509.new_extension("basicConstraints", "CA:" + str(False).upper())) + userCert.add_ext(X509.new_extension("extendedKeyUsage", "clientAuth", critical=1)) + # Set livetime + validityTime = datetime.timedelta(days=400) + notBefore = ASN1.ASN1_UTCTIME() + notBefore.set_time(int(time.time())) + notAfter = ASN1.ASN1_UTCTIME() + notAfter.set_time(int(time.time()) + int(validityTime.total_seconds())) + userCert.set_not_before(notBefore) + userCert.set_not_after(notAfter) + # Add subject from CA + with open(self.parameters["CertFile"]) as cf: + caCertStr = cf.read() + caCert = X509.load_cert_string(caCertStr) + userCert.set_issuer(caCert.get_subject()) + # Use CA key + with open(self.parameters["KeyFile"], "rb") as cf: + caKeyStr = cf.read() + pkey = EVP.PKey() + pkey.assign_rsa(RSA.load_key_string(caKeyStr, callback=util.no_passphrase_callback)) + # Sign + userCert.sign(pkey, self.algoritm) + + userCertStr = userCert.as_pem().decode("ascii") + userPubKeyStr = userPubKey.as_pem(cipher=None, callback=util.no_passphrase_callback).decode("ascii") + return S_OK((userCertStr, userPubKeyStr)) + + def _forceGenerateProxyForDN(self, dn, time, group=None): + """An additional helper method for creating a proxy without any substantial validation, + it can be used for a specific case(such as testing) where just need to generate a proxy + with specific DN on the fly. + + :param str dn: requested proxy DN + :param int time: expired time in a seconds + :param str group: if need to add DIRAC group + + :return: S_OK(tuple)/S_ERROR() -- contain proxy as chain and as string + """ + self.__X509Name = X509.X509_Name() + result = self.__parseDN(dn) + if not result["OK"]: + return result + dnInfoDict = result["Value"] + + for field, values in dnInfoDict.items(): + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + result = self.__createCertM2Crypto() if result["OK"]: certStr, keyStr = result["Value"] - chain = X509Chain() - result = chain.loadChainFromString(certStr) + if chain.loadChainFromString(certStr)["OK"] and chain.loadKeyFromString(keyStr)["OK"]: + result = chain.generateProxyToString(time, diracGroup=group) + if not result["OK"]: + return result + chain = X509Chain() + chain.loadProxyFromString(result["Value"]) + return S_OK((chain, result["Value"])) + +else: + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + + from DIRAC.Core.Security import asn1_utils + + # Mapping between the distinguished name aliases and their OID, replacing + # the M2Crypto X509_Name.nid lookup table + _FIELDS_TO_OID = { + "C": "2.5.4.6", + "countryName": "2.5.4.6", + "SP": "2.5.4.8", + "ST": "2.5.4.8", + "stateOrProvinceName": "2.5.4.8", + "L": "2.5.4.7", + "localityName": "2.5.4.7", + "O": "2.5.4.10", + "organizationName": "2.5.4.10", + "OU": "2.5.4.11", + "organizationalUnitName": "2.5.4.11", + "CN": "2.5.4.3", + "commonName": "2.5.4.3", + "Email": "1.2.840.113549.1.9.1", + "emailAddress": "1.2.840.113549.1.9.1", + "SN": "2.5.4.4", + "surname": "2.5.4.4", + "GN": "2.5.4.42", + "givenName": "2.5.4.42", + "serialNumber": "2.5.4.5", + "SERIALNUMBER": "2.5.4.5", + "DC": "0.9.2342.19200300.100.1.25", + "domainComponent": "0.9.2342.19200300.100.1.25", + } + + _ALGORITHMS = { + "sha1": hashes.SHA1, + "sha224": hashes.SHA224, + "sha256": hashes.SHA256, + "sha384": hashes.SHA384, + "sha512": hashes.SHA512, + } + + class DIRACCAProxyProvider(ProxyProvider): + def __init__(self, parameters=None): + """Constructor""" + super().__init__(parameters) + self.log = gLogger.getSubLogger(__name__) + # Initialize + self.maxDict = {} + self.minDict = {} + self.bits = 2048 + self.algoritm = "sha256" + self.match = [] + self.supplied = [_FIELDS_TO_OID["CN"]] + self.optional = [_FIELDS_TO_OID[f] for f in ("C", "O", "OU", "emailAddress")] + self.dnList = ["C", "O", "OU", "CN", "emailAddress"] + # Distinguished names + self.fields2oid = dict(_FIELDS_TO_OID) + self.oid2fields = {} # oid: list of distinguished names + # Specify standart fields + for field, oid in self.fields2oid.items(): + self.oid2fields.setdefault(oid, []).append(field) + self.dnInfoDictCA = {} + # List of x509.NameAttribute of the DN being built + self.__nameAttributes = [] + + def setParameters(self, parameters): + """Set new parameters + + :param dict parameters: provider parameters + + :return: S_OK()/S_ERROR() + """ + for k, v in parameters.items(): + if not isinstance(v, list) and k in ["Match", "Supplied", "Optional", "DNOrder"] + list( + self.fields2oid + ): + parameters[k] = v.replace(", ", ",").split(",") + self.parameters = parameters + # If CA configuration file exist + if parameters.get("CAConfigFile"): + self.__parseCACFG() + if "Bits" in parameters: + self.bits = int(parameters["Bits"]) + if "Algoritm" in parameters: + self.algoritm = parameters["Algoritm"] + if "Match" in parameters: + self.match = [self.fields2oid[f] for f in parameters["Match"]] + if "Supplied" in parameters: + self.supplied = [self.fields2oid[f] for f in parameters["Supplied"]] + if "Optional" in parameters: + self.optional = [self.fields2oid[f] for f in parameters["Optional"]] + allFields = self.optional + self.supplied + self.match + if "DNOrder" in parameters: + self.dnList = [] + if not any([any([f in parameters["DNOrder"] for f in self.oid2fields[n]]) for n in allFields]): + return S_ERROR("DNOrder must contain all configured fields.") + for field in parameters["DNOrder"]: + if self.fields2oid[field] in allFields: + self.dnList.append(field) + + # Set defaults for distridutes names + self.oid2defField = {} + for field, value in list(self.parameters.items()): + if field in self.fields2oid and self.fields2oid[field] in allFields: + self.parameters[self.fields2oid[field]] = value + self.oid2defField[self.fields2oid[field]] = field + + # Read CA certificate + chain = X509Chain() + result = chain.loadChainFromFile(self.parameters["CertFile"]) + if result["OK"]: + result = chain.getCredentials() if result["OK"]: - result = chain.loadKeyFromString(keyStr) - if result["OK"]: - result = chain.generateProxyToString(365 * 24 * 3600) - - return result - - def generateDN(self, **kwargs): - """Get DN of the user certificate that will be created - - :param dict kwargs: user description dictionary with possible fields: - - FullName or CN - - Email or emailAddress - - :return: S_OK(str)/S_ERROR() -- contain DN - """ - if kwargs.get("FullName"): - kwargs["CN"] = [kwargs["FullName"]] - if kwargs.get("Email"): - kwargs["emailAddress"] = [kwargs["Email"]] - - self.__nameAttributes = [] - self.log.info("Creating distinguished names chain") - - for oid in self.supplied: - if oid not in [self.fields2oid[f] for f in self.dnList]: - return S_ERROR( - 'DNs order list does not contain supplied DN "%s"' - % self.oid2defField.get(oid, min(self.oid2fields[oid], key=len)) - ) - - for field in self.dnList: - values = [] - oid = self.fields2oid[field] - if oid in self.match: - for field in self.oid2fields[oid]: - if field in self.dnInfoDictCA: - values = self.dnInfoDictCA[field] - if not values: - return S_ERROR(f'Not found "{field}" match DN in CA') - for field in self.oid2fields[oid]: - if kwargs.get(field): - values = kwargs[field] if isinstance(kwargs[field], list) else [kwargs[field]] - if not values and oid in self.supplied: - # Search default value - if oid not in self.oid2defField: - return S_ERROR(f'No values set for "{min(self.oid2fields[oid], key=len)}" DN') - values = self.parameters[oid] - - result = self.__fillX509Name(field, values) + result = self.__parseDN(result["Value"]["subject"]) if not result["OK"]: return result + self.dnInfoDictCA = result["Value"] + return S_OK() - # WARN: This logic not support list of distribtes name elements - resDN = asn1_utils.nameToDN(x509.Name(self.__nameAttributes)) + def checkStatus(self, userDN): + """Read ready to work status of proxy provider - result = self.checkStatus(resDN) - if not result["OK"]: - return result - return S_OK(resDN) - - def __parseCACFG(self): - """Parse CA configuration file""" - block = "" - self.cfg = {} - self.supplied, self.optional, self.match, self.dnList = [], [], [], [] - with open(self.parameters["CAConfigFile"]) as caCFG: - for line in caCFG: - # Ignore comments - line = re.sub(r"#.*", "", line) - if re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", "")): - block = "".join(re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", ""))) - if block not in self.cfg: - self.cfg[block] = {} - if not block: - continue - if len(re.findall("=", line)) == 1: - field, val = line.split("=") - field = field.strip() - variables = re.findall(r"[$]([A-Za-z0-9_]+)", val) - for v in variables: - for b in self.cfg: - if v in self.cfg[b]: - val = val.replace("$" + v, self.cfg[b][v]) - if "default_ca" in self.cfg.get("ca", {}): - if "policy" in self.cfg.get(self.cfg["ca"]["default_ca"], {}): - if block == self.cfg[self.cfg["ca"]["default_ca"]]["policy"]: - self.dnList.append(field) - self.cfg[block][field] = val.strip() - - self.bits = int(self.cfg["req"].get("default_bits") or self.bits) - self.algoritm = self.cfg[self.cfg["ca"]["default_ca"]].get("default_md") or self.algoritm - if not self.parameters.get("CertFile"): - self.parameters["CertFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["certificate"] - self.parameters["KeyFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["private_key"] - # Read distinguished names - for k, v in self.cfg[self.cfg[self.cfg["ca"]["default_ca"]]["policy"]].items(): - oid = self.fields2oid[k] - self.parameters[oid], self.minDict[oid], self.maxDict[oid] = [], [], [] - for k in [f"{i}.{k}" for i in range(0, 5)] + [k]: - if k + "_default" in self.cfg["req"]["distinguished_name"]: - self.parameters[oid].append(self.cfg["req"]["distinguished_name"][k + "_default"]) - if k + "_min" in self.cfg["req"]["distinguished_name"]: - self.minDict[oid].append(self.cfg["req"]["distinguished_name"][k + "_min"]) - if k + "_max" in self.cfg["req"]["distinguished_name"]: - self.maxDict[oid].append(self.cfg["req"]["distinguished_name"][k + "_max"]) - if v == "supplied": - self.supplied.append(oid) - elif v == "optional": - self.optional.append(oid) - elif v == "match": - self.match.append(oid) - - def __parseDN(self, dn): - """Return DN fields - - :param str dn: DN - - :return: list -- contain tuple with positionOfField.fieldName, fieldOID, fieldValue - """ - dnInfoDict = collections.OrderedDict() - for f, v in [f.split("=") for f in dn.lstrip("/").split("/")]: - if not v: - return S_ERROR(f'No value set for "{f}"') - if f not in dnInfoDict: - dnInfoDict[f] = [v] - else: - dnInfoDict[f].append(v) - return S_OK(dnInfoDict) - - def __fillX509Name(self, field, values): - """Collect the DN attributes for the certificate subject - - :param str field: DN field name - :param list values: values of field, order important - - :return: S_OK()/S_ERROR() - """ - for value in values: - if value: - try: - self.__nameAttributes.append( - x509.NameAttribute(x509.ObjectIdentifier(self.fields2oid[field]), value) + :param str userDN: user DN + + :return: S_OK()/S_ERROR() + """ + self.log.debug("Ckecking work status of", self.parameters["ProviderName"]) + result = self.__parseDN(userDN) + if not result["OK"]: + return result + dnInfoDict = result["Value"] + + try: + userOIDs = [self.fields2oid[f.split("=")[0]] for f in userDN.lstrip("/").split("/")] + except (ValueError, KeyError) as e: + return S_ERROR(f"Unknown DN field in used DN: {e}") + oidOrder = [self.fields2oid[f] for f in self.dnList] + for index, oid in enumerate(userOIDs): + if oid not in oidOrder: + return S_ERROR( + f'"{self.oid2defField.get(oid, min(self.oid2fields[oid], key=len))}" field not found in order.' ) - except (KeyError, TypeError, ValueError) as e: - return S_ERROR(f'Cannot set "{field}" field: {e!r}.') - return S_OK() - - def __createCertificate(self): - """Create new certificate for user - - :return: S_OK(tuple)/S_ERROR() -- tuple contain certificate and private key as strings - """ - if self.algoritm not in _ALGORITHMS: - return S_ERROR(f'Unsupported signing algorithm "{self.algoritm}"') - hashAlgo = _ALGORITHMS[self.algoritm]() - - # Create user key pair - userKey = rsa.generate_private_key(public_exponent=65537, key_size=self.bits) - - # Read CA certificate and key - try: - with open(self.parameters["CertFile"], "rb") as cf: - caCert = x509.load_pem_x509_certificate(cf.read()) - with open(self.parameters["KeyFile"], "rb") as cf: - caKey = serialization.load_pem_private_key(cf.read(), password=None) - except Exception as e: - return S_ERROR(f"Cannot load CA credentials: {e!r}") - - serial = 0 - while not serial: - serial = secrets.randbits(64) - - now = datetime.datetime.now(datetime.timezone.utc) - builder = ( - x509.CertificateBuilder() - .subject_name(x509.Name(self.__nameAttributes)) - .issuer_name(caCert.subject) - .public_key(userKey.public_key()) - .serial_number(serial) - .not_valid_before(now) - .not_valid_after(now + datetime.timedelta(days=400)) - .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=False) - .add_extension(x509.ExtendedKeyUsage([x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH]), critical=True) - ) - - try: - userCert = builder.sign(caKey, hashAlgo) - except Exception as e: - return S_ERROR(f"Cannot sign the user certificate: {e!r}") - - userCertStr = userCert.public_bytes(serialization.Encoding.PEM).decode("ascii") - userKeyStr = userKey.private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.PKCS8, - serialization.NoEncryption(), - ).decode("ascii") - return S_OK((userCertStr, userKeyStr)) - - def _forceGenerateProxyForDN(self, dn, time, group=None): - """An additional helper method for creating a proxy without any substantial validation, - it can be used for a specific case(such as testing) where just need to generate a proxy - with specific DN on the fly. - - :param str dn: requested proxy DN - :param int time: expired time in a seconds - :param str group: if need to add DIRAC group - - :return: S_OK(tuple)/S_ERROR() -- contain proxy as chain and as string - """ - self.__nameAttributes = [] - result = self.__parseDN(dn) - if not result["OK"]: + if index > oidOrder.index(oid): + return S_ERROR("Bad DNs order") + for i in range(oidOrder.index(oid) - 1): + try: + if userOIDs.index(oidOrder[i]) > index: + return S_ERROR("Bad DNs order") + except (ValueError, KeyError): + continue + for i in range(oidOrder.index(oid) + 1, len(oidOrder)): + try: + if userOIDs.index(oidOrder[i]) < index: + return S_ERROR("Bad DNs order") + except (ValueError, KeyError): + continue + + for oid in self.supplied: + if oid not in [self.fields2oid[f] for f in dnInfoDict]: + return S_ERROR( + 'Current DN is invalid, "%s" field must be set.' + % self.oid2defField.get(oid, min(self.oid2fields[oid], key=len)) + ) + + for field, values in dnInfoDict.items(): + oid = self.fields2oid[field] + err = f'Current DN is invalid, "{field}" field' + if oid not in self.supplied + self.match + self.optional: + return S_ERROR(f"{err} is not found for current CA.") + if oid in self.match and not self.dnInfoDictCA[field] == values: + return S_ERROR(f"{err} must be /{field}={('/%s=' % field).joing(self.dnInfoDictCA[field])}.") + if oid in self.maxDict: + rangeMax = list(range(min(len(values), len(self.maxDict[oid])))) + if any([True if len(values[i]) > self.maxDict[oid][i] else False for i in rangeMax]): + return S_ERROR(f"{err} values must be less then {', '.join(self.maxDict[oid])}.") + if oid in self.minDict: + rangeMin = list(range(min(len(values), len(self.minDict[oid])))) + if any([True if len(values[i]) < self.minDict[oid][i] else False for i in rangeMin]): + return S_ERROR(f"{err} values must be more then {', '.join(self.minDict[oid])}.") + + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + return S_OK() + + def getProxy(self, userDN): + """Generate user proxy + + :param str userDN: user DN + + :return: S_OK(str)/S_ERROR() -- contain a proxy string + """ + self.__nameAttributes = [] + result = self.checkStatus(userDN) + if result["OK"]: + result = self.__createCertificate() + if result["OK"]: + certStr, keyStr = result["Value"] + + chain = X509Chain() + result = chain.loadChainFromString(certStr) + if result["OK"]: + result = chain.loadKeyFromString(keyStr) + if result["OK"]: + result = chain.generateProxyToString(365 * 24 * 3600) + return result - dnInfoDict = result["Value"] - for field, values in dnInfoDict.items(): - result = self.__fillX509Name(field, values) + def generateDN(self, **kwargs): + """Get DN of the user certificate that will be created + + :param dict kwargs: user description dictionary with possible fields: + - FullName or CN + - Email or emailAddress + + :return: S_OK(str)/S_ERROR() -- contain DN + """ + if kwargs.get("FullName"): + kwargs["CN"] = [kwargs["FullName"]] + if kwargs.get("Email"): + kwargs["emailAddress"] = [kwargs["Email"]] + + self.__nameAttributes = [] + self.log.info("Creating distinguished names chain") + + for oid in self.supplied: + if oid not in [self.fields2oid[f] for f in self.dnList]: + return S_ERROR( + 'DNs order list does not contain supplied DN "%s"' + % self.oid2defField.get(oid, min(self.oid2fields[oid], key=len)) + ) + + for field in self.dnList: + values = [] + oid = self.fields2oid[field] + if oid in self.match: + for field in self.oid2fields[oid]: + if field in self.dnInfoDictCA: + values = self.dnInfoDictCA[field] + if not values: + return S_ERROR(f'Not found "{field}" match DN in CA') + for field in self.oid2fields[oid]: + if kwargs.get(field): + values = kwargs[field] if isinstance(kwargs[field], list) else [kwargs[field]] + if not values and oid in self.supplied: + # Search default value + if oid not in self.oid2defField: + return S_ERROR(f'No values set for "{min(self.oid2fields[oid], key=len)}" DN') + values = self.parameters[oid] + + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + # WARN: This logic not support list of distribtes name elements + resDN = asn1_utils.nameToDN(x509.Name(self.__nameAttributes)) + + result = self.checkStatus(resDN) + if not result["OK"]: + return result + return S_OK(resDN) + + def __parseCACFG(self): + """Parse CA configuration file""" + block = "" + self.cfg = {} + self.supplied, self.optional, self.match, self.dnList = [], [], [], [] + with open(self.parameters["CAConfigFile"]) as caCFG: + for line in caCFG: + # Ignore comments + line = re.sub(r"#.*", "", line) + if re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", "")): + block = "".join(re.findall(r"\[([A-Za-z0-9_]+)\]", line.replace(" ", ""))) + if block not in self.cfg: + self.cfg[block] = {} + if not block: + continue + if len(re.findall("=", line)) == 1: + field, val = line.split("=") + field = field.strip() + variables = re.findall(r"[$]([A-Za-z0-9_]+)", val) + for v in variables: + for b in self.cfg: + if v in self.cfg[b]: + val = val.replace("$" + v, self.cfg[b][v]) + if "default_ca" in self.cfg.get("ca", {}): + if "policy" in self.cfg.get(self.cfg["ca"]["default_ca"], {}): + if block == self.cfg[self.cfg["ca"]["default_ca"]]["policy"]: + self.dnList.append(field) + self.cfg[block][field] = val.strip() + + self.bits = int(self.cfg["req"].get("default_bits") or self.bits) + self.algoritm = self.cfg[self.cfg["ca"]["default_ca"]].get("default_md") or self.algoritm + if not self.parameters.get("CertFile"): + self.parameters["CertFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["certificate"] + self.parameters["KeyFile"] = self.cfg[self.cfg["ca"]["default_ca"]]["private_key"] + # Read distinguished names + for k, v in self.cfg[self.cfg[self.cfg["ca"]["default_ca"]]["policy"]].items(): + oid = self.fields2oid[k] + self.parameters[oid], self.minDict[oid], self.maxDict[oid] = [], [], [] + for k in [f"{i}.{k}" for i in range(0, 5)] + [k]: + if k + "_default" in self.cfg["req"]["distinguished_name"]: + self.parameters[oid].append(self.cfg["req"]["distinguished_name"][k + "_default"]) + if k + "_min" in self.cfg["req"]["distinguished_name"]: + self.minDict[oid].append(self.cfg["req"]["distinguished_name"][k + "_min"]) + if k + "_max" in self.cfg["req"]["distinguished_name"]: + self.maxDict[oid].append(self.cfg["req"]["distinguished_name"][k + "_max"]) + if v == "supplied": + self.supplied.append(oid) + elif v == "optional": + self.optional.append(oid) + elif v == "match": + self.match.append(oid) + + def __parseDN(self, dn): + """Return DN fields + + :param str dn: DN + + :return: list -- contain tuple with positionOfField.fieldName, fieldOID, fieldValue + """ + dnInfoDict = collections.OrderedDict() + for f, v in [f.split("=") for f in dn.lstrip("/").split("/")]: + if not v: + return S_ERROR(f'No value set for "{f}"') + if f not in dnInfoDict: + dnInfoDict[f] = [v] + else: + dnInfoDict[f].append(v) + return S_OK(dnInfoDict) + + def __fillX509Name(self, field, values): + """Collect the DN attributes for the certificate subject + + :param str field: DN field name + :param list values: values of field, order important + + :return: S_OK()/S_ERROR() + """ + for value in values: + if value: + try: + self.__nameAttributes.append( + x509.NameAttribute(x509.ObjectIdentifier(self.fields2oid[field]), value) + ) + except (KeyError, TypeError, ValueError) as e: + return S_ERROR(f'Cannot set "{field}" field: {e!r}.') + return S_OK() + + def __createCertificate(self): + """Create new certificate for user + + :return: S_OK(tuple)/S_ERROR() -- tuple contain certificate and private key as strings + """ + if self.algoritm not in _ALGORITHMS: + return S_ERROR(f'Unsupported signing algorithm "{self.algoritm}"') + hashAlgo = _ALGORITHMS[self.algoritm]() + + # Create user key pair + userKey = rsa.generate_private_key(public_exponent=65537, key_size=self.bits) + + # Read CA certificate and key + try: + with open(self.parameters["CertFile"], "rb") as cf: + caCert = x509.load_pem_x509_certificate(cf.read()) + with open(self.parameters["KeyFile"], "rb") as cf: + caKey = serialization.load_pem_private_key(cf.read(), password=None) + except Exception as e: + return S_ERROR(f"Cannot load CA credentials: {e!r}") + + serial = 0 + while not serial: + serial = secrets.randbits(64) + + now = datetime.datetime.now(datetime.timezone.utc) + builder = ( + x509.CertificateBuilder() + .subject_name(x509.Name(self.__nameAttributes)) + .issuer_name(caCert.subject) + .public_key(userKey.public_key()) + .serial_number(serial) + .not_valid_before(now) + .not_valid_after(now + datetime.timedelta(days=400)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=False) + .add_extension(x509.ExtendedKeyUsage([x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH]), critical=True) + ) + + try: + userCert = builder.sign(caKey, hashAlgo) + except Exception as e: + return S_ERROR(f"Cannot sign the user certificate: {e!r}") + + userCertStr = userCert.public_bytes(serialization.Encoding.PEM).decode("ascii") + userKeyStr = userKey.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode("ascii") + return S_OK((userCertStr, userKeyStr)) + + def _forceGenerateProxyForDN(self, dn, time, group=None): + """An additional helper method for creating a proxy without any substantial validation, + it can be used for a specific case(such as testing) where just need to generate a proxy + with specific DN on the fly. + + :param str dn: requested proxy DN + :param int time: expired time in a seconds + :param str group: if need to add DIRAC group + + :return: S_OK(tuple)/S_ERROR() -- contain proxy as chain and as string + """ + self.__nameAttributes = [] + result = self.__parseDN(dn) if not result["OK"]: return result + dnInfoDict = result["Value"] - result = self.__createCertificate() - if result["OK"]: - certStr, keyStr = result["Value"] + for field, values in dnInfoDict.items(): + result = self.__fillX509Name(field, values) + if not result["OK"]: + return result + + result = self.__createCertificate() + if result["OK"]: + certStr, keyStr = result["Value"] + chain = X509Chain() + if chain.loadChainFromString(certStr)["OK"] and chain.loadKeyFromString(keyStr)["OK"]: + result = chain.generateProxyToString(time, diracGroup=group) + if not result["OK"]: + return result chain = X509Chain() - if chain.loadChainFromString(certStr)["OK"] and chain.loadKeyFromString(keyStr)["OK"]: - result = chain.generateProxyToString(time, diracGroup=group) - if not result["OK"]: - return result - chain = X509Chain() - chain.loadProxyFromString(result["Value"]) - return S_OK((chain, result["Value"])) + chain.loadProxyFromString(result["Value"]) + return S_OK((chain, result["Value"])) diff --git a/tests/py3CheckDirs.txt b/tests/py3CheckDirs.txt index c98c282c290..00ab41c4414 100644 --- a/tests/py3CheckDirs.txt +++ b/tests/py3CheckDirs.txt @@ -9,7 +9,7 @@ src/DIRAC/Core/Base/Client.py src/DIRAC/Core/DISET/private/BaseClient.py src/DIRAC/Core/DISET/private/Service.py src/DIRAC/Core/DISET/RequestHandler.py -src/DIRAC/Core/Security/X509Chain.py +src/DIRAC/Core/Security/pyca/X509Chain.py src/DIRAC/Core/Tornado src/DIRAC/DataManagementSystem/Service/TornadoFileCatalogHandler.py src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py From 8d3fb1d4003539a89a91225e3521f24c041adb0f Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 11 Jul 2026 00:16:23 +0200 Subject: [PATCH 6/7] test: run the X509 and SSLTransport tests against both implementations Re-add the M2Crypto variants to the implementation matrices (X509CHAINTYPES, X509REQUESTTYPES, X509CERTTYPES and TRANSPORTTESTS). Both implementations are imported through their fully qualified subpackage paths so they can be tested in the same process, whatever DIRAC_USE_M2CRYPTO says; the M2 variants are skipped when M2Crypto or pyasn1 are not installed, as they are optional dependencies. The standard library ssl transport pair is skipped on python < 3.13, where SSLSocket.get_unverified_chain is not available. --- .../Transports/test/Test_SSLTransport.py | 21 +++++++-- .../Security/test/Test_X509Certificate.py | 9 ++-- .../Core/Security/test/Test_X509Chain.py | 4 +- .../Core/Security/test/x509TestUtilities.py | 47 ++++++++++++------- 4 files changed, 58 insertions(+), 23 deletions(-) diff --git a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py index edbfeed2cc8..c80bff60f3e 100644 --- a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py @@ -1,14 +1,15 @@ """ Test the SSLTransport mechanism """ import os import selectors +import sys import threading from diraccfg import CFG -from pytest import fixture +from pytest import fixture, mark, param from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.Core.DISET.private.Transports import PlainTransport, StdSSLTransport -from DIRAC.Core.Security.test.x509TestUtilities import CERTDIR, USERCERT, getCertOption +from DIRAC.Core.Security.test.x509TestUtilities import CERTDIR, USERCERT, getCertOption, skipm2 # TODO: Expired hostcert # TODO: Expired usercert @@ -38,7 +39,16 @@ # Transports are now tested in pairs: # "Server-Client" # Each pair is defined as a string. -TRANSPORTTESTS = ("Plain-Plain", "SSL-SSL") +TRANSPORTTESTS = ( + "Plain-Plain", + # The server side of the standard library ssl transport needs + # SSLSocket.get_unverified_chain, which was added in python 3.13 + param( + "SSL-SSL", + marks=mark.skipif(sys.version_info < (3, 13), reason="needs python >= 3.13 (ssl get_unverified_chain)"), + ), + param("M2-M2", marks=skipm2), +) # https://www.ibm.com/developerworks/linux/library/l-openssl/index.html @@ -123,6 +133,11 @@ def transportByName(transport): return PlainTransport.PlainTransport elif transport.lower() == "ssl": return StdSSLTransport.SSLTransport + elif transport.lower() == "m2": + # Imported lazily as M2Crypto may not be installed + from DIRAC.Core.DISET.private.Transports import M2SSLTransport + + return M2SSLTransport.SSLTransport raise RuntimeError(f"Unknown Transport Name: {transport}") diff --git a/src/DIRAC/Core/Security/test/Test_X509Certificate.py b/src/DIRAC/Core/Security/test/Test_X509Certificate.py index 0d18daa4c4f..61090065862 100644 --- a/src/DIRAC/Core/Security/test/Test_X509Certificate.py +++ b/src/DIRAC/Core/Security/test/Test_X509Certificate.py @@ -21,15 +21,16 @@ CERTCONTENTS, getCertOption, HOSTCERT, + skipm2, VOMSPROXY, VOMS_PROXY_ATTR, ) -from pytest import mark, fixture, skip +from pytest import mark, fixture, param, skip parametrize = mark.parametrize -X509CERTTYPES = ("PYCA_X509Certificate",) +X509CERTTYPES = ("PYCA_X509Certificate", param("M2_X509Certificate", marks=skipm2)) # This fixture will return a X509Certificate class # https://docs.pytest.org/en/latest/fixture.html#automatic-grouping-of-tests-by-fixture-instances @@ -46,7 +47,9 @@ def get_X509Certificate_class(request): x509Class = request.param if x509Class == "PYCA_X509Certificate": - from DIRAC.Core.Security.X509Certificate import X509Certificate + from DIRAC.Core.Security.pyca.X509Certificate import X509Certificate + elif x509Class == "M2_X509Certificate": + from DIRAC.Core.Security.m2crypto.X509Certificate import X509Certificate else: raise NotImplementedError() diff --git a/src/DIRAC/Core/Security/test/Test_X509Chain.py b/src/DIRAC/Core/Security/test/Test_X509Chain.py index 3ab891a5e16..37e2c3abfaf 100644 --- a/src/DIRAC/Core/Security/test/Test_X509Chain.py +++ b/src/DIRAC/Core/Security/test/Test_X509Chain.py @@ -80,7 +80,9 @@ def get_proxy(request): x509Class = request.param if x509Class == "PYCA_X509Chain": - from DIRAC.Core.Security.X509Chain import X509Chain + from DIRAC.Core.Security.pyca.X509Chain import X509Chain + elif x509Class == "M2_X509Chain": + from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain else: raise NotImplementedError() diff --git a/src/DIRAC/Core/Security/test/x509TestUtilities.py b/src/DIRAC/Core/Security/test/x509TestUtilities.py index b9418329df5..cec8dd45dbb 100644 --- a/src/DIRAC/Core/Security/test/x509TestUtilities.py +++ b/src/DIRAC/Core/Security/test/x509TestUtilities.py @@ -3,7 +3,20 @@ import sys from datetime import datetime -from pytest import fixture +from pytest import fixture, mark, param + +# The M2Crypto based implementation is only tested if its (optional) +# dependencies are installed +try: + import M2Crypto # noqa pylint: disable=unused-import + import pyasn1 # noqa pylint: disable=unused-import + import pyasn1_modules # noqa pylint: disable=unused-import + + M2CRYPTO_AVAILABLE = True +except ImportError: + M2CRYPTO_AVAILABLE = False + +skipm2 = mark.skipif(not M2CRYPTO_AVAILABLE, reason="M2Crypto and/or pyasn1 are not installed") # We use certificates stored in the same folder as this test file CERTDIR = os.path.join(os.path.dirname(__file__), "certs") @@ -283,24 +296,20 @@ def getCertOption(cert, optionName): def deimportDIRAC(): """clean all what has already been imported from DIRAC. - This method is extremely fragile, but hopefully, we can get ride of all these - messy tests soon, when PyGSI has gone. + Nothing needs to be done anymore: the two X509 implementations are + imported through their fully qualified subpackage paths + (DIRAC.Core.Security.pyca and DIRAC.Core.Security.m2crypto), so they can + coexist in the same process, whatever DIRAC_USE_M2CRYPTO says. """ - if len(X509CHAINTYPES) != 1 or len(X509REQUESTTYPES) != 1: - raise NotImplementedError( - "This no longer de-imports DIRAC, if we want to test another SSL wrapper " - "we will have to find another way of doing this or run a separate pytest " - "process again" - ) # for mod in list(sys.modules): # # You should be careful with what you remove.... # if (mod == 'DIRAC' or mod.startswith('DIRAC.')) and not mod.startswith('DIRAC.Core.Security.test'): # sys.modules.pop(mod) -X509CHAINTYPES = ("PYCA_X509Chain",) +X509CHAINTYPES = ("PYCA_X509Chain", param("M2_X509Chain", marks=skipm2)) -# This fixture will return a pyGSI or M2Crypto X509Chain class +# This fixture will return a pyca/cryptography or M2Crypto X509Chain class # https://docs.pytest.org/en/latest/fixture.html#automatic-grouping-of-tests-by-fixture-instances @@ -315,7 +324,9 @@ def get_X509Chain_class(request): x509Class = request.param if x509Class == "PYCA_X509Chain": - from DIRAC.Core.Security.X509Chain import X509Chain + from DIRAC.Core.Security.pyca.X509Chain import X509Chain + elif x509Class == "M2_X509Chain": + from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain else: raise NotImplementedError() @@ -325,7 +336,7 @@ def get_X509Chain_class(request): deimportDIRAC() -X509REQUESTTYPES = ("PYCA_X509Request",) +X509REQUESTTYPES = ("PYCA_X509Request", param("M2_X509Request", marks=skipm2)) # This fixture will return a X509Request class # https://docs.pytest.org/en/latest/fixture.html#automatic-grouping-of-tests-by-fixture-instances @@ -342,7 +353,9 @@ def get_X509Request(request): x509Class = request.param if x509Class == "PYCA_X509Request": - from DIRAC.Core.Security.X509Request import X509Request + from DIRAC.Core.Security.pyca.X509Request import X509Request + elif x509Class == "M2_X509Request": + from DIRAC.Core.Security.m2crypto.X509Request import X509Request else: raise NotImplementedError() @@ -368,8 +381,10 @@ def get_X509Chain_from_X509Request(x509ReqObj): """ # In principle, we should deimport Dirac everywhere, but I am not even sure it makes any difference - if "X509Request" in x509ReqObj.__class__.__module__: - from DIRAC.Core.Security.X509Chain import X509Chain + if "m2crypto" in x509ReqObj.__class__.__module__: + from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain + elif "pyca" in x509ReqObj.__class__.__module__: + from DIRAC.Core.Security.pyca.X509Chain import X509Chain else: raise NotImplementedError() From b0795234a03c2bf7d76ed71e24f9b12e861184ec Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 11 Jul 2026 00:16:37 +0200 Subject: [PATCH 7/7] docs: document the DIRAC_USE_M2CRYPTO environment variable Describe the switch between the default pyca/cryptography based implementation and the legacy M2Crypto one, re-add the entries for the M2Crypto specific environment variables it brings back, and mention the tornado fork and tornado-m2crypto requirements of the fallback in the Tornado services guide. --- .../environment_variable_configuration.rst | 14 ++++++++++++++ .../DeveloperGuide/TornadoServices/index.rst | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst index 2f20c6aecf4..7bc6d06f0db 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst @@ -42,9 +42,23 @@ DIRAC_HTTPS_SSL_METHOD_MAX DIRAC_HTTPS_SSL_METHOD_MIN If set, overrides the lowest supported TLS version when using HTTPS. It should be a valid value of :py:class:`ssl.TLSVersion`. +DIRAC_M2CRYPTO_SPLIT_HANDSHAKE + Only relevant with ``DIRAC_USE_M2CRYPTO``. If ``true`` or ``yes`` the SSL handshake is done in a new thread (default Yes) + +DIRAC_M2CRYPTO_SSL_CIPHERS + Only relevant with ``DIRAC_USE_M2CRYPTO`` (but also accepted as an alias of ``DIRAC_SSL_CIPHERS`` by the default implementation). If set, overwrites the default SSL ciphers accepted. It should be a colon separated list. See :py:mod:`DIRAC.Core.DISET` + +DIRAC_M2CRYPTO_SSL_METHODS + Only relevant with ``DIRAC_USE_M2CRYPTO``. If set, overwrites the default SSL methods accepted. It should be a colon separated list. See :py:mod:`DIRAC.Core.DISET` + DIRAC_SSL_CIPHERS If set, overwrites the default SSL ciphers accepted by the DISET protocol. It should be an OpenSSL style colon separated list. +DIRAC_USE_M2CRYPTO + If ``true`` or ``yes``, use the legacy M2Crypto/pyasn1 based implementation of the X509 layer and of the TLS transports (DISET and HTTPS) instead of the default one, based on pyca/cryptography and the :py:mod:`ssl` module of the standard library (default No). + The implementation is selected once, at import time, so it must be set consistently for all the components of an installation. + The optional dependencies M2Crypto, pyasn1, pyasn1-modules and (for HTTPS services) tornado-m2crypto with the patched DIRACGrid tornado fork must be installed. + DIRAC_MYSQL_OPTIMIZER_TRACES_PATH If set, it should point to an existing directory, where MySQL Optimizer traces will be stored. See :py:func:`DIRAC.Core.Utilities.MySQL.captureOptimizerTraces` diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index 870c44a9356..9a738a61858 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -317,6 +317,12 @@ The standard tornado package is used: the TLS layer (including grid proxy support) relies on the ``ssl`` module of the python standard library (python >= 3.13 is required on the server side). +When the legacy M2Crypto implementation is enabled (``DIRAC_USE_M2CRYPTO=Yes``), +two special python packages are needed instead: + +* git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable : in place of the standard tornado. This adds configurable feature to tornado +* git+https://github.com/DIRACGrid/tornado_m2crypto.git: this allows to use tornado with M2Crypto + Install a service *****************