Skip to content

Commit a86407d

Browse files
committed
Merge branch 'master' into jocar-291
2 parents 4c73120 + d348f6e commit a86407d

7 files changed

Lines changed: 317 additions & 57 deletions

File tree

‎docs/usage/examples.rst‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -198,23 +198,28 @@ contains the following:
198198
</XRDS>
199199
200200
The structure of the file should be fairly self-evident. Only links with @rel="urn:oasis:names:tc:SAML:2.0:metadata"
201-
will be parsed. If a KeyInfo with a X509Certificate element (usual base64-encoded certificate format) then this
202-
certificate is used to validate the signature on the downloaded SAML metadata. Note that while 'load' supports validation
203-
based on certificate fingerprint the XRD format does not and you will have to include Base64-encoded certificates if
204-
you want validation to work.
201+
will be parsed. The X509Certificate elements (base64-encoded certificates) in a link's KeyInfo are the trusted signers
202+
of the metadata it points to: the downloaded metadata is only loaded if it is signed by one of them, so during a key
203+
rollover a link can list both the old and the new certificate. The certificate is used as such, not via its
204+
fingerprint, so this also works for metadata whose signature does not carry the signing certificate, as is common for
205+
MDQ responses.
206+
207+
.. warning::
208+
209+
A link without a certificate is loaded **without** signature verification.
205210

206211
Example 4 - Sign using a PKCS#11 module
207212
---------------------------------------
208213

209-
Fetch SWAMID metadata (and validate the signature using a certificate matching the given SHA256 fingerprint), select
214+
Fetch SWAMID metadata (and validate the signature using the SWAMID signing certificate in swamid-2.0.crt), select
210215
the Identity Providers, tidy it up a bit and sign with the key with the label 'signer' in the PKCS#11 module
211216
/usr/lib/libsofthsm.so. If a certificate is found in the same PKCS#11 object, that certificate is included in
212217
the Signature object.
213218

214219
.. code-block:: yaml
215220
216221
- load:
217-
- http://mds.swamid.se/md/swamid-2.0.xml A6:78:5A:37:C9:C9:0C:25:AD:5F:1F:69:22:EF:76:7B:C9:78:67:67:3A:AF:4F:8B:EA:A1:A7:6D:A3:A8:E5:85
222+
- http://mds.swamid.se/md/swamid-2.0.xml verify swamid-2.0.crt
218223
- select: "!//md:EntityDescriptor[md:IDPSSODescriptor]"
219224
- xslt:
220225
stylesheet: tidy.xsl

‎scripts/genkey.sh‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,18 @@
11
#!/bin/sh
22

3-
openssl genrsa 2048 > sign.key
4-
openssl req -x509 -sha256 -new -subj "/CN=Signer" -key sign.key -out sign.crt
3+
# RSA - 4096 bit
4+
openssl genrsa 4096 > sign-rsa.key
5+
openssl req -x509 -sha256 -new -subj "/CN=Signer (RSA)" -key sign-rsa.key -out sign-rsa.crt -days 10000
6+
7+
# ECDSA - NIST curve p384
8+
openssl ecparam -out sign-ecdsa-pkcs1.key -name secp384r1 -genkey
9+
openssl pkcs8 -topk8 -nocrypt -in sign-ecdsa-pkcs1.key -out sign-ecdsa-pkcs8.key
10+
openssl req -x509 -new -key sign-ecdsa-pkcs1.key -out sign-ecdsa.crt -subj "/CN=Signer (ECDSA)" -days 10000
11+
12+
# SLH-DSA - 256 bit - short
13+
openssl genpkey -algorithm SLH-DSA-SHA2-256s -out sign-slhdsa.key
14+
openssl req -x509 -new -key sign-slhdsa.key -out sign-slhdsa.crt -subj "/CN=Signer (SLH-DSA)" -days 10000
15+
16+
# ML-DSA - 87
17+
openssl genpkey -algorithm ML-DSA-87 -out sign-mldsa.key
18+
openssl req -x509 -new -key sign-mldsa.key -out sign-mldsa.crt -subj "/CN=Signer (ML-DSA)" -days 10000

‎src/pyff/api.py‎

Lines changed: 61 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import importlib
22
import threading
3+
import traceback
34
from collections.abc import Generator, Iterable, Mapping
45
from datetime import datetime, timedelta
56
from json import dumps
@@ -265,54 +266,74 @@ def _d(x: Optional[str], do_split: bool = True) -> tuple[Optional[str], Optional
265266
else:
266267
q = new_path
267268

269+
# The pipeline result - often the whole selected metadata tree - only lives in _run_plumbings' frame. Pyramid keeps
270+
# an error response's exception on the request, so an error must not reference that frame through its traceback:
271+
# request, exception and frame would form a reference cycle, freed only by a full garbage collection (issue #283).
268272
try:
269-
accepter = MediaAccept(accept)
270-
for p in request.registry.plumbings:
271-
state = {
272-
entry: True,
273-
'headers': {'Content-Type': None},
274-
'accept': accepter,
275-
'url': request.current_route_url(),
276-
'select': q,
277-
'match': match.lower() if match else match,
278-
'path': new_path,
279-
'stats': {},
280-
}
281-
282-
r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler)
283-
log.debug(f'Plumbing process result: {r}')
284-
if r is None:
285-
r = []
286-
287-
response = Response()
288-
_headers = state.get('headers', {})
289-
response.headers.update(_headers)
290-
ctype = _headers.get('Content-Type', None)
291-
if not ctype:
292-
r, t = _fmt(r, accepter)
293-
ctype = t
294-
295-
response.text = b2u(r)
296-
response.size = len(r)
297-
response.content_type = ctype
298-
cache_ttl = int(state.get('cache', 0))
299-
response.expires = datetime.now() + timedelta(seconds=cache_ttl)
300-
return response
273+
response = _run_plumbings(request, entry, accept, q, match, new_path)
274+
except exc.HTTPException as ex:
275+
# An HTTP error from processing, eg 406 from content negotiation, is the response as is. Besides the traceback,
276+
# __context__/__cause__ can reference the frame (an error raised while handling another one), so clear them too.
277+
ex.__context__ = ex.__cause__ = None
278+
raise ex.with_traceback(None)
301279
except ResourceException as ex:
302-
import traceback
303-
304280
log.debug(traceback.format_exc())
305281
log.warning(f'Exception from processing pipeline: {ex}')
306-
raise exc.exception_response(409)
282+
status = 409
307283
except BaseException as ex:
308-
import traceback
309-
310284
log.debug(traceback.format_exc())
311285
log.error(f'Exception from processing pipeline: {ex}')
312-
raise exc.exception_response(500)
286+
status = 500
287+
else:
288+
if response is None and request.method == 'GET':
289+
raise exc.exception_response(404)
290+
return response
313291

314-
if request.method == 'GET':
315-
raise exc.exception_response(404)
292+
# Raised outside the except clauses, so that it does not chain the original exception and its traceback.
293+
raise exc.exception_response(status)
294+
295+
296+
def _run_plumbings(
297+
request: Request, entry: str, accept: str, q: Optional[str], match: Optional[str], new_path: Optional[str]
298+
) -> Optional[Response]:
299+
"""
300+
Run the request through the plumbings and format the result for the accepted content type.
301+
302+
:return: the response, or None if there are no plumbings
303+
"""
304+
accepter = MediaAccept(accept)
305+
for p in request.registry.plumbings:
306+
state = {
307+
entry: True,
308+
'headers': {'Content-Type': None},
309+
'accept': accepter,
310+
'url': request.current_route_url(),
311+
'select': q,
312+
'match': match.lower() if match else match,
313+
'path': new_path,
314+
'stats': {},
315+
}
316+
317+
r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler)
318+
log.debug(f'Plumbing process result: {r}')
319+
if r is None:
320+
r = []
321+
322+
response = Response()
323+
_headers = state.get('headers', {})
324+
response.headers.update(_headers)
325+
ctype = _headers.get('Content-Type', None)
326+
if not ctype:
327+
r, t = _fmt(r, accepter)
328+
ctype = t
329+
330+
response.text = b2u(r)
331+
response.size = len(r)
332+
response.content_type = ctype
333+
cache_ttl = int(state.get('cache', 0))
334+
response.expires = datetime.now() + timedelta(seconds=cache_ttl)
335+
return response
336+
return None
316337

317338

318339
def webfinger_handler(request: Request) -> Response:

‎src/pyff/builtins.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -694,11 +694,18 @@ def load(req: Plumbing.Request, *opts):
694694
- ``as``: name the resource, e.g. to refer to it in later pipeline steps.
695695
- ``verify``: check the XML signature of the resource, using either a PEM certificate file or a SHA-1
696696
fingerprint of the signing certificate. With a fingerprint, the certificate embedded in the signature is
697-
used and the fingerprint pins it. ``verify`` may be repeated to support key rollovers; a signature by any
698-
one of the given certificates (or fingerprints) is accepted. Certificates and fingerprints cannot be mixed.
697+
used and the fingerprint pins it, so a document whose signature carries no certificate - common for MDQ
698+
responses - can only be verified with the certificate file. Other fingerprints, eg SHA-256, are not
699+
supported. ``verify`` may be repeated to support key rollovers; a signature by any one of the given
700+
certificates (or fingerprints) is accepted. Certificates and fingerprints cannot be mixed.
699701
- ``via``: name of a ``when`` clause in the same pipeline that is applied to the loaded metadata, for
700702
example to remove entities with ``fork merge remove``.
701703
- ``cleanup``: name of a ``when`` clause used to pre-process the parsed metadata before it is validated.
704+
705+
A resource may also be an XRD or eIDAS MDSL file listing metadata URLs. Each listed URL is loaded and verified
706+
with the certificates given for it in the file, as if with ``verify``; any one of them is accepted, so a key
707+
rollover can list both. An XRD link without a certificate is loaded unverified; an MDSL entry without one is
708+
skipped.
702709
"""
703710
_opts = dict(list(zip(opts[::2], opts[1::2])))
704711
_opts.setdefault('timeout', 120)

‎src/pyff/parse.py‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from pyff.resource import Resource, ResourceInfo
1212

1313
# cert_dict replaces xmlsec.crypto.CertDict (fingerprint -> cert mapping).
14-
from pyff.utils import cert_dict, find_matching_files, parse_xml, root, unicode_stream, utc_now
14+
from pyff.utils import b64_to_pem, cert_dict, find_matching_files, parse_xml, root, unicode_stream, utc_now
1515

1616
__author__ = 'leifj'
1717

@@ -120,12 +120,16 @@ def parse(self, resource: Resource, content: str) -> ParserInfo:
120120
for link in xrd.findall(".//{{{}}}Link[@rel='{}']".format(NS['xrd'], NS['md'])):
121121
link_href = link.get("href")
122122
certs = cert_dict(link)
123-
fingerprints = list(certs.keys())
124-
fp = None
125-
if len(fingerprints) > 0:
126-
fp = fingerprints[0]
127-
log.debug(f"XRD: {link_href} verified by {fp}")
128-
child_opts = resource.opts.model_copy(update={'alias': None})
123+
if certs:
124+
log.debug(f"XRD: {link_href} verified by {', '.join(certs.keys())}")
125+
else:
126+
log.debug(f"XRD: {link_href} has no certificate, not verified")
127+
# The Link's certificates are the trusted signers; all of them are accepted so that a key rollover
128+
# can list both the old and the new certificate. Without one the metadata is not verified, rather
129+
# than with the verify option of the XRD itself.
130+
child_opts = resource.opts.model_copy(
131+
update={'alias': None, 'verify': [b64_to_pem(cert_b64) for cert_b64 in certs.values()] or None}
132+
)
129133
resource.add_child(link_href, child_opts)
130134
resource.last_seen = utc_now().replace(microsecond=0)
131135
resource.expire_time = None

‎src/pyff/test/test_md_api.py‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,26 @@
1+
import gc
12
import json
23
import os
34
import shutil
45
import tempfile
56
import unittest
7+
import weakref
68
from datetime import datetime, timezone
79
from types import SimpleNamespace
10+
from unittest import mock
811
from urllib.parse import quote as urlescape
912

1013
import pytest
1114
import requests
1215
from mako.lookup import TemplateLookup
16+
from pyramid import httpexceptions as exc
1317
from pyramid.testing import DummyRequest
1418
from pyuppsala import etree
1519
from wsgi_intercept.interceptor import RequestsInterceptor, UrllibInterceptor
1620

1721
from pyff.api import mkapp, search_handler
1822
from pyff.constants import config
23+
from pyff.resource import ResourceException
1924
from pyff.samlmd import iter_entities
2025
from pyff.test import SignerTestCase
2126
from pyff.test.test_pipeline import PipeLineTest
@@ -356,3 +361,110 @@ def test_entity_id_without_trailing_slash_is_not_found(self):
356361
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
357362
assert requests.post(f'{url}/api/call/update').status_code == 200
358363
assert self._entity_ids(url, '/entities/https:/sp.example.com/saml2/metadata') == set()
364+
365+
366+
class _PipelineResult(list):
367+
"""A pipeline result that can be weakly referenced, to observe when it is freed"""
368+
369+
370+
class PyFFAPITestErrorResponses(PipeLineTest):
371+
"""
372+
Errors while processing a request must have the right status, and must not keep the pipeline result - often
373+
the whole selected metadata tree - alive after the request: cf. issue #283.
374+
"""
375+
376+
mdx = None
377+
app = None
378+
379+
@classmethod
380+
def setUpClass(cls):
381+
SignerTestCase.setUpClass()
382+
config.local_copy_dir = tempfile.mkdtemp()
383+
cls.mdx = tempfile.NamedTemporaryFile('w').name
384+
with open(cls.mdx, "w") as fd:
385+
fd.write(
386+
f"""
387+
- when update:
388+
- load:
389+
- {os.path.join(cls.datadir, 'metadata', 'test01.xml')}
390+
- when request:
391+
- select
392+
- pipe:
393+
- when accept application/xml:
394+
- emit application/xml
395+
- break
396+
"""
397+
)
398+
cls._app = mkapp(cls.mdx)
399+
cls.app = lambda *args, **kwargs: cls._app
400+
401+
@classmethod
402+
def tearDownClass(cls):
403+
SignerTestCase.tearDownClass()
404+
if os.path.exists(cls.mdx):
405+
os.unlink(cls.mdx)
406+
if os.path.exists(config.local_copy_dir):
407+
shutil.rmtree(config.local_copy_dir)
408+
409+
def _request_with_pipeline_result(self, url, fail=False, error=None):
410+
"""
411+
GET /entities as text/plain with the pipeline replaced by one producing a weakly referenceable result,
412+
then failing if ``fail`` (with a RuntimeError) or with the ``error`` factory's exception, raised while a
413+
KeyError is being handled. The garbage collector is off, so only reference counting can free the result.
414+
415+
:return: the HTTP status and whether the result was freed once the request was done
416+
"""
417+
refs = []
418+
419+
def process(*args, **kwargs):
420+
result = _PipelineResult(['<md:EntitiesDescriptor/>'])
421+
refs.append(weakref.ref(result))
422+
if error is not None:
423+
try:
424+
raise KeyError("first failure")
425+
except KeyError:
426+
raise error()
427+
if fail:
428+
raise RuntimeError("pipeline failed")
429+
return result
430+
431+
gc.collect()
432+
gc.disable()
433+
try:
434+
with mock.patch.object(self._app.registry.plumbings[0], 'process', process):
435+
status = requests.get(f'{url}/entities', headers={'Accept': 'text/plain'}).status_code
436+
return status, refs[0]() is None
437+
finally:
438+
gc.enable()
439+
440+
def test_unacceptable_type_is_406(self):
441+
"""No pipeline branch and no format matches text/plain: that is 406 Not Acceptable, not a server error"""
442+
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
443+
assert requests.post(f'{url}/api/call/update').status_code == 200
444+
r = requests.get(f'{url}/entities', headers={'Accept': 'text/plain'})
445+
assert r.status_code == 406
446+
447+
def test_406_frees_the_pipeline_result(self):
448+
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
449+
status, freed = self._request_with_pipeline_result(url, fail=False)
450+
assert status == 406
451+
assert freed, "the pipeline result outlived a 406 response"
452+
453+
def test_500_frees_the_pipeline_result(self):
454+
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
455+
status, freed = self._request_with_pipeline_result(url, fail=True)
456+
assert status == 500
457+
assert freed, "the pipeline result outlived a 500 response"
458+
459+
def test_http_error_raised_while_handling_another_frees_the_pipeline_result(self):
460+
"""An HTTP error chained to another exception must not keep the result alive through __context__"""
461+
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
462+
status, freed = self._request_with_pipeline_result(url, error=exc.HTTPNotFound)
463+
assert status == 404
464+
assert freed, "the pipeline result outlived a chained 404 response"
465+
466+
def test_409_frees_the_pipeline_result(self):
467+
with RequestsInterceptor(self.app, host='127.0.0.1', port=80) as url:
468+
status, freed = self._request_with_pipeline_result(url, error=lambda: ResourceException("conflict"))
469+
assert status == 409
470+
assert freed, "the pipeline result outlived a 409 response"

0 commit comments

Comments
 (0)