From 8298a3c0e7749a2944f05da949917e394e71b424 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 11:56:23 +0000 Subject: [PATCH 1/2] Test the Set Headers Element shared by two Pipelines The multi-pipeline elements specification says an element that keeps state for the Pipeline it is in keeps that state for each Pipeline it is added to (multi-pipeline-elements.md line 17 at 706de1f0, https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/advanced-features/multi-pipeline-elements.md?plain=1#L17). These tests add one SetHeaderElement to two Pipelines, one holding an element that gives SetHeaderBrowserAccept-CH and the other an element that gives SetHeaderPlatformCritical-CH. test_each_pipeline_gives_its_own_headers processes the Pipelines in each order, twice over, and expects each Pipeline's own headers. This fails today, because the element builds its list of SetHeader properties from the first Pipeline it processes for and keeps that list for every Pipeline. The second Pipeline then looks for an element it does not hold and gives the first Pipeline's header with an empty value, for example Accept-CH set to "" where Critical-CH is expected. test_one_pipeline_gives_the_same_headers_on_every_request processes one Pipeline three times. It passes today and guards that a Pipeline keeps giving its own headers. test_headers_are_given_once_the_properties_are_known processes a Pipeline whose element has no properties on its first request, as a cloud engine has none until the cloud service answers, and then gives the element its properties. It expects no headers and then the header. It passes today, because an empty list is built again on the next request, and guards that a list kept for each Pipeline is not an empty one kept for good. The cloud request engine specification says consumers of that metadata "MUST NOT permanently cache an empty result obtained before discovery has completed" (cloud-request-engine.md line 298 at 706de1f0, https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/pipeline-elements/cloud-request-engine.md?plain=1#L298). test_setheader_properties_holds_the_first_pipelines_properties processes the two Pipelines that share one element and checks that the public setheader_properties holds the first Pipeline's dict of data key to property names, the shape it has today. It passes today and guards that shape. --- .../tests/test_setheader.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py index a666e2e9..9f96df6a 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py @@ -28,6 +28,9 @@ from fiftyone_pipeline_core.messages import Messages from fiftyone_pipeline_core.web import * from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipeline import Pipeline from fiftyone_pipeline_core.setheaderelement import SetHeaderElement from .classes.testpipeline import TestPipeline @@ -140,3 +143,97 @@ class Pipeline(object): self.assertEqual( {"device": ["SetHeaderBrowserAccept-CH"]}, SetHeaderElement().get_setheader_properties_pipeline(Pipeline())) + + +# An element with one property for a response header, whose value is the +# header's value. +class HeaderSourceElement(FlowElement): + + def __init__(self, datakey, property_name, value): + super(HeaderSourceElement, self).__init__() + self.datakey = datakey + self.property_name = property_name + self.value = value + self.properties = { + property_name: {"name": property_name, "type": "string"}} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary( + self, + {self.property_name: AspectPropertyValue(value=self.value)})) + + +def response_headers(pipeline): + flowdata = pipeline.create_flowdata() + flowdata.process() + return flowdata["set-headers"]["responseheaderdictionary"] + + +BROWSER_HEADERS = {"Accept-CH": Constants.ACCEPTCH_BROWSER_VALUE} +PLATFORM_HEADERS = {"Critical-CH": Constants.ACCEPTCH_PLATFORM_VALUE} + + +# The multi-pipeline elements specification says an element that keeps +# state for the Pipeline it is in keeps that state for each Pipeline it is +# added to. One Set Headers Element is shared by two Pipelines whose other +# elements give different SetHeader properties. +class SetHeaderMultiPipelineTests(unittest.TestCase): + + def setUp(self): + self.set_headers = set_headers = SetHeaderElement() + # The Pipelines share an element, so neither closes its elements. + self.pipelines = { + "browser": Pipeline( + [HeaderSourceElement( + "browser", "SetHeaderBrowserAccept-CH", + Constants.ACCEPTCH_BROWSER_VALUE), + set_headers], + auto_close_elements=False), + "platform": Pipeline( + [HeaderSourceElement( + "platform", "SetHeaderPlatformCritical-CH", + Constants.ACCEPTCH_PLATFORM_VALUE), + set_headers], + auto_close_elements=False)} + self.expected = { + "browser": BROWSER_HEADERS, "platform": PLATFORM_HEADERS} + + # Each Pipeline gives the headers of its own elements, whichever + # processes first, and keeps doing so after the other has processed. + @parameterized.expand([ + ("browser_first", ["browser", "platform"]), + ("platform_first", ["platform", "browser"])]) + def test_each_pipeline_gives_its_own_headers(self, _, order): + for name in order + order: + self.assertEqual( + self.expected[name], response_headers(self.pipelines[name]), + name) + + # setheader_properties holds the SetHeader properties of the first + # Pipeline that processed, as a dict of data key to property names, for + # code that reads it. + def test_setheader_properties_holds_the_first_pipelines_properties(self): + response_headers(self.pipelines["browser"]) + response_headers(self.pipelines["platform"]) + self.assertEqual( + {"browser": ["SetHeaderBrowserAccept-CH"]}, + self.set_headers.setheader_properties) + + def test_one_pipeline_gives_the_same_headers_on_every_request(self): + for _ in range(3): + self.assertEqual( + BROWSER_HEADERS, response_headers(self.pipelines["browser"])) + + # A cloud engine has no properties until the cloud service has + # answered, so a Pipeline whose SetHeader properties are not known on + # its first request gives the headers once they are. + def test_headers_are_given_once_the_properties_are_known(self): + source = HeaderSourceElement( + "browser", "SetHeaderBrowserAccept-CH", + Constants.ACCEPTCH_BROWSER_VALUE) + properties = source.properties + source.properties = {} + pipeline = Pipeline([source, SetHeaderElement()]) + self.assertEqual({}, response_headers(pipeline)) + source.properties = properties + self.assertEqual(BROWSER_HEADERS, response_headers(pipeline)) From 598949a74e9ac1053da9f9dddaa3e38aec7e5c62 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 11:56:25 +0000 Subject: [PATCH 2/2] Keep the Set Headers Element's SetHeader properties for each Pipeline The multi-pipeline elements specification says "If this feature is enabled, then all Flow Elements that maintain any such state information MUST ensure that they have the means to keep track of isolated instances of this state for each Pipeline to which they are added and use the correct state information when processing." (multi-pipeline-elements.md line 17 at 706de1f0, https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/advanced-features/multi-pipeline-elements.md?plain=1#L17). The Set Headers Element built its list of SetHeader properties from the first Pipeline it processed for and used that list for every Pipeline. One element added to two Pipelines whose other elements give different SetHeader properties gave the second Pipeline the first one's headers, each with an empty value, and printed that the first one's element was not found. The element now keeps a list for each Pipeline in the private _setheader_properties_by_pipeline, keyed by the Pipeline, and builds each list from that Pipeline's own elements on the Pipeline's first request. Processing uses only those lists. As before, a list that is empty is built again on the next request, so the headers of a cloud engine whose properties arrive after the first request are still given. Two requests that build a list at the same time build the same one, and each reads and writes the dictionary in a single step, so no lock is needed. The public setheader_properties keeps the shape it had, a dict of data key to the names of that element's SetHeader properties. It holds a copy of the first list the element builds that is not empty, set once, so code that reads setheader_properties["device"] keeps working. As it is a copy, changing it no longer changes the headers. PipelineBuilder adds a new Set Headers Element to each Pipeline it builds, so only an application that adds one element to several Pipelines sees its headers change. --- .../setheaderelement.py | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py index 4e9f2ca6..2845b00b 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py @@ -46,7 +46,16 @@ def __init__(self): self.properties = {"responseheaderdictionary" : { "type": "dict"} } + # A copy of the SetHeader properties of the first Pipeline found to + # have any, for code that reads them. Processing does not use it. self.setheader_properties = {} + + # The SetHeader properties of each Pipeline the element processes + # for, keyed by the Pipeline, as each Pipeline holds its own + # elements. Each value is what get_setheader_properties_pipeline + # returns for that Pipeline. The element already holds each Pipeline + # it is added to in pipelines, so the keys keep none alive longer. + self._setheader_properties_by_pipeline = {} self.exclude_from_messages = True @@ -69,10 +78,25 @@ def process_internal(self, flowdata): """ - if not self.setheader_properties: - self.setheader_properties = self.get_setheader_properties_pipeline(flowdata.pipeline) - - response_headers = self.get_response_header_value(flowdata, self.setheader_properties) + # An empty list is built again on the next request, because a cloud + # engine has no properties until the cloud service answers. Requests + # that build a list at the same time build the same one, so either + # may be kept. + pipeline = flowdata.pipeline + setheader_properties = self._setheader_properties_by_pipeline.get( + pipeline) + if not setheader_properties: + setheader_properties = self.get_setheader_properties_pipeline( + pipeline) + self._setheader_properties_by_pipeline[pipeline] = ( + setheader_properties) + if setheader_properties and not self.setheader_properties: + self.setheader_properties = { + datakey: list(names) + for datakey, names in setheader_properties.items()} + + response_headers = self.get_response_header_value( + flowdata, setheader_properties) data = ElementDataDictionary(self, {"responseheaderdictionary": response_headers})