From 51b2cb63b527e30e8011145fcc286ab7c14ecfd0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 17:28:57 +0000 Subject: [PATCH 1/2] Test that each usage sharing record names the elements of its own pipeline --- .../tests/test_share_usage_pipelines.py | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py new file mode 100644 index 00000000..4aef928a --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py @@ -0,0 +1,121 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +One usage sharing element in two Pipelines. Each record names the flow +elements and packages of the Pipeline its request came from, as +multi-pipeline-elements.md asks of an element that keeps state for a +Pipeline. +""" + +import gzip +import re +import unittest +from unittest import mock + +import requests + +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + + +class FirstElement(FlowElement): + + def __init__(self): + super(FirstElement, self).__init__() + self.datakey = "first" + + def process_internal(self, flowdata): + pass + + +class SecondElement(FlowElement): + + def __init__(self): + super(SecondElement, self).__init__() + self.datakey = "second" + + def process_internal(self, flowdata): + pass + + +def sent_on_close(usage): + """Closes the element, which sends what it has queued, and returns each + record it posted. The post is answered without leaving the process.""" + sent = [] + + def post(url, headers=None, data=None, **kwargs): + sent.append(gzip.decompress(data).decode("utf8")) + return mock.Mock(status_code=200, text="") + + with mock.patch.object(requests, "post", post): + usage.close() + return [record for message in sent + for record in re.findall(r".*?", message)] + + +def process(pipeline, user_agent): + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", user_agent) + flowdata.process() + + +class ShareUsagePipelinesTests(unittest.TestCase): + + def test_each_record_names_the_elements_of_its_own_pipeline(self): + usage = ShareUsage(requested_package_size=1000, + endpoint="http://127.0.0.1:9") + first = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(FirstElement()).add(usage).build() + second = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(SecondElement()).add(usage).build() + + for _ in range(2): + process(first, "first agent") + process(second, "second agent") + records = sent_on_close(usage) + + by_agent = {} + for record in records: + agent = re.search(r'Name="user-agent">([^<]*)<', record).group(1) + by_agent.setdefault(agent, []).append( + re.findall(r"([^<]*)", record)) + self.assertEqual( + {"first agent": [["FirstElement", "ShareUsage"]], + "second agent": [["SecondElement", "ShareUsage"]]}, + by_agent) + # The usage sharing element's own package is in use in both. + for record in records: + packages = { + re.sub(r"[-_.]+", "_", name).lower() for name in + re.findall(r'', record)} + self.assertIn("fiftyone_pipeline_engines_fiftyone", packages) + + +if __name__ == "__main__": + unittest.main() From fc0c6833cce2a3180a9d974cc6512ba98c5548f4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 17:29:21 +0000 Subject: [PATCH 2/2] Name the elements of each pipeline in its usage sharing records --- .../share_usage.py | 94 +++++++++++++------ 1 file changed, 66 insertions(+), 28 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index b66ca341..24268fe5 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -276,6 +276,10 @@ def __init__( self.constant_xml = None + # The constant part of each record for each Pipeline the element + # is in, as each names its own flow elements and packages. + self._constant_xml_by_pipeline = {} + # The address of this machine for the record's ServerIP, looked up # on its own thread so that processing never waits for a slow name # lookup. It is None until the lookup ends, and records made before @@ -402,33 +406,26 @@ def _send_remaining(self, deadline, done): else: self.backoff.failed(time.monotonic()) - def get_constant_xml(self): + def get_constant_xml(self, pipeline=None): + """! + The part of a record that is the same for every request of a + Pipeline. Given the Pipeline a request came from, it names that + Pipeline's flow elements and packages, so an element in several + Pipelines writes each one's own. With no Pipeline it names those of + the one Pipeline the element is in. + """ + if pipeline is not None: + xml = self._constant_xml_by_pipeline.get(pipeline) + if xml is None: + xml = self._make_constant_xml( + self.get_flow_elements(pipeline), + list(pipeline.flow_elements)) + if self._server_ip is not None: + self._constant_xml_by_pipeline[pipeline] = xml + return xml if self.constant_xml is None: - coreVersion = version("fiftyone_pipeline_core") - osVersion = f"{ReplacedString(platform.system()).result} {ReplacedString(platform.release()).result}" - pyVersion = ReplacedString(platform.python_version()).result - - xml = "" - - # The version number of the Pipeline API - xml += f"{coreVersion}" - # Write Pipeline information - # The product name - xml += "Pipeline" - # The flow elements in the current pipeline - for flow_element in self.get_flow_elements(): - xml += f"{flow_element}" - # The release of each 51Degrees package in use - for name, package_version in fiftyone_packages( - self._pipeline_flow_elements()): - xml += record_element("Package", package_version, name) - xml += "Python" - # The software language version - xml += f"{pyVersion}" - # The address of this machine - xml += plain_element("ServerIP", self._server_ip or "") - # The OS name and version - xml += f"{osVersion}" + xml = self._make_constant_xml( + self.get_flow_elements(), self._pipeline_flow_elements()) # Kept once the lookup of the address has ended, so that # ServerIP carries the address from then on. if self._server_ip is None: @@ -436,6 +433,37 @@ def get_constant_xml(self): self.constant_xml = xml return self.constant_xml + def _make_constant_xml(self, flow_element_names, flow_elements): + """! + The constant part of a record for the flow elements of one + Pipeline. + """ + coreVersion = version("fiftyone_pipeline_core") + osVersion = f"{ReplacedString(platform.system()).result} {ReplacedString(platform.release()).result}" + pyVersion = ReplacedString(platform.python_version()).result + + xml = "" + + # The version number of the Pipeline API + xml += f"{coreVersion}" + # Write Pipeline information + # The product name + xml += "Pipeline" + # The flow elements in the Pipeline + for flow_element in flow_element_names: + xml += f"{flow_element}" + # The release of each 51Degrees package in use + for name, package_version in fiftyone_packages(flow_elements): + xml += record_element("Package", package_version, name) + xml += "Python" + # The software language version + xml += f"{pyVersion}" + # The address of this machine + xml += plain_element("ServerIP", self._server_ip or "") + # The OS name and version + xml += f"{osVersion}" + return xml + def _look_up_server_ip(self): try: self._server_ip = host_address() @@ -453,7 +481,13 @@ def _pipeline_flow_elements(self): return list(self.pipelines[0].flow_elements) return [] - def get_flow_elements(self): + def get_flow_elements(self, pipeline=None): + """! + The class names of the flow elements of the given Pipeline, or of + the one Pipeline the element is in when none is given. + """ + if pipeline is not None: + return [type(e).__name__ for e in pipeline.flow_elements] if len(self.pipelines) == 1: list = [] @@ -651,7 +685,7 @@ def add_to_share_usage(self, data): xml += "" # --- write invariant data - xml += self.get_constant_xml() + xml += self.get_constant_xml(data.pipeline) # --- write variable data # The SessionID used to track a series of requests @@ -743,6 +777,7 @@ def get_data_from_evidence(self, flow_data): provided """ data = ShareUsageData() + data.pipeline = flow_data.pipeline for key, value in self.found_evidence(flow_data).items(): # Evidence keys are compared without regard to case. @@ -969,6 +1004,9 @@ def __init__(self): self.session_id = "" self.client_ip = "" self.sequence = "" + # The Pipeline the request came from, whose flow elements the + # record names. + self.pipeline = None def try_add(self, key, value): # Get the category and field names from the evidence key.