Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,10 @@ def __init__(

self.constant_xml = None

# The constant part of each record for each Pipeline the element
# is in, as each names its own flow elements and packages.
self._constant_xml_by_pipeline = {}

# The address of this machine for the record's ServerIP, looked up
# on its own thread so that processing never waits for a slow name
# lookup. It is None until the lookup ends, and records made before
Expand Down Expand Up @@ -402,40 +406,64 @@ def _send_remaining(self, deadline, done):
else:
self.backoff.failed(time.monotonic())

def get_constant_xml(self):
def get_constant_xml(self, pipeline=None):
"""!
The part of a record that is the same for every request of a
Pipeline. Given the Pipeline a request came from, it names that
Pipeline's flow elements and packages, so an element in several
Pipelines writes each one's own. With no Pipeline it names those of
the one Pipeline the element is in.
"""
if pipeline is not None:
xml = self._constant_xml_by_pipeline.get(pipeline)
if xml is None:
xml = self._make_constant_xml(
self.get_flow_elements(pipeline),
list(pipeline.flow_elements))
if self._server_ip is not None:
self._constant_xml_by_pipeline[pipeline] = xml
return xml
if self.constant_xml is None:
coreVersion = version("fiftyone_pipeline_core")
osVersion = f"{ReplacedString(platform.system()).result} {ReplacedString(platform.release()).result}"
pyVersion = ReplacedString(platform.python_version()).result

xml = ""

# The version number of the Pipeline API
xml += f"<Version>{coreVersion}</Version>"
# Write Pipeline information
# The product name
xml += "<Product>Pipeline</Product>"
# The flow elements in the current pipeline
for flow_element in self.get_flow_elements():
xml += f"<FlowElement>{flow_element}</FlowElement>"
# The release of each 51Degrees package in use
for name, package_version in fiftyone_packages(
self._pipeline_flow_elements()):
xml += record_element("Package", package_version, name)
xml += "<Language>Python</Language>"
# The software language version
xml += f"<LanguageVersion>{pyVersion}</LanguageVersion>"
# The address of this machine
xml += plain_element("ServerIP", self._server_ip or "")
# The OS name and version
xml += f"<Platform>{osVersion}</Platform>"
xml = self._make_constant_xml(
self.get_flow_elements(), self._pipeline_flow_elements())
# Kept once the lookup of the address has ended, so that
# ServerIP carries the address from then on.
if self._server_ip is None:
return xml
self.constant_xml = xml
return self.constant_xml

def _make_constant_xml(self, flow_element_names, flow_elements):
"""!
The constant part of a record for the flow elements of one
Pipeline.
"""
coreVersion = version("fiftyone_pipeline_core")
osVersion = f"{ReplacedString(platform.system()).result} {ReplacedString(platform.release()).result}"
pyVersion = ReplacedString(platform.python_version()).result

xml = ""

# The version number of the Pipeline API
xml += f"<Version>{coreVersion}</Version>"
# Write Pipeline information
# The product name
xml += "<Product>Pipeline</Product>"
# The flow elements in the Pipeline
for flow_element in flow_element_names:
xml += f"<FlowElement>{flow_element}</FlowElement>"
# The release of each 51Degrees package in use
for name, package_version in fiftyone_packages(flow_elements):
xml += record_element("Package", package_version, name)
xml += "<Language>Python</Language>"
# The software language version
xml += f"<LanguageVersion>{pyVersion}</LanguageVersion>"
# The address of this machine
xml += plain_element("ServerIP", self._server_ip or "")
# The OS name and version
xml += f"<Platform>{osVersion}</Platform>"
return xml

def _look_up_server_ip(self):
try:
self._server_ip = host_address()
Expand All @@ -453,7 +481,13 @@ def _pipeline_flow_elements(self):
return list(self.pipelines[0].flow_elements)
return []

def get_flow_elements(self):
def get_flow_elements(self, pipeline=None):
"""!
The class names of the flow elements of the given Pipeline, or of
the one Pipeline the element is in when none is given.
"""
if pipeline is not None:
return [type(e).__name__ for e in pipeline.flow_elements]
if len(self.pipelines) == 1:
list = []

Expand Down Expand Up @@ -651,7 +685,7 @@ def add_to_share_usage(self, data):
xml += "<Device>"

# --- write invariant data
xml += self.get_constant_xml()
xml += self.get_constant_xml(data.pipeline)

# --- write variable data
# The SessionID used to track a series of requests
Expand Down Expand Up @@ -743,6 +777,7 @@ def get_data_from_evidence(self, flow_data):
provided
"""
data = ShareUsageData()
data.pipeline = flow_data.pipeline

for key, value in self.found_evidence(flow_data).items():
# Evidence keys are compared without regard to case.
Expand Down Expand Up @@ -969,6 +1004,9 @@ def __init__(self):
self.session_id = ""
self.client_ip = ""
self.sequence = ""
# The Pipeline the request came from, whose flow elements the
# record names.
self.pipeline = None

def try_add(self, key, value):
# Get the category and field names from the evidence key.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# *********************************************************************
# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
#
# This Original Work is licensed under the European Union Public Licence
# (EUPL) v.1.2 and is subject to its terms as set out below.
#
# If a copy of the EUPL was not distributed with this file, You can obtain
# one at https://opensource.org/licenses/EUPL-1.2.
#
# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
# amended by the European Commission) shall be deemed incompatible for
# the purposes of the Work and the provisions of the compatibility
# clause in Article 5 of the EUPL shall not apply.
#
# If using the Work as, or as part of, a network application, by
# including the attribution notice(s) required under Article 5 of the EUPL
# in the end user terms of the application under an appropriate heading,
# such notice(s) shall fulfill the requirements of that article.
# *********************************************************************

"""
One usage sharing element in two Pipelines. Each record names the flow
elements and packages of the Pipeline its request came from, as
multi-pipeline-elements.md asks of an element that keeps state for a
Pipeline.
"""

import gzip
import re
import unittest
from unittest import mock

import requests

from fiftyone_pipeline_core.flowelement import FlowElement
from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage

NO_DEFAULT_ELEMENTS = {
"add_javascript_builder": False,
"use_setheader_properties": False,
}


class FirstElement(FlowElement):

def __init__(self):
super(FirstElement, self).__init__()
self.datakey = "first"

def process_internal(self, flowdata):
pass


class SecondElement(FlowElement):

def __init__(self):
super(SecondElement, self).__init__()
self.datakey = "second"

def process_internal(self, flowdata):
pass


def sent_on_close(usage):
"""Closes the element, which sends what it has queued, and returns each
record it posted. The post is answered without leaving the process."""
sent = []

def post(url, headers=None, data=None, **kwargs):
sent.append(gzip.decompress(data).decode("utf8"))
return mock.Mock(status_code=200, text="")

with mock.patch.object(requests, "post", post):
usage.close()
return [record for message in sent
for record in re.findall(r"<Device>.*?</Device>", message)]


def process(pipeline, user_agent):
with pipeline.create_flowdata() as flowdata:
flowdata.evidence.add("header.user-agent", user_agent)
flowdata.process()


class ShareUsagePipelinesTests(unittest.TestCase):

def test_each_record_names_the_elements_of_its_own_pipeline(self):
usage = ShareUsage(requested_package_size=1000,
endpoint="http://127.0.0.1:9")
first = PipelineBuilder(NO_DEFAULT_ELEMENTS) \
.add(FirstElement()).add(usage).build()
second = PipelineBuilder(NO_DEFAULT_ELEMENTS) \
.add(SecondElement()).add(usage).build()

for _ in range(2):
process(first, "first agent")
process(second, "second agent")
records = sent_on_close(usage)

by_agent = {}
for record in records:
agent = re.search(r'Name="user-agent">([^<]*)<', record).group(1)
by_agent.setdefault(agent, []).append(
re.findall(r"<FlowElement>([^<]*)</FlowElement>", record))
self.assertEqual(
{"first agent": [["FirstElement", "ShareUsage"]],
"second agent": [["SecondElement", "ShareUsage"]]},
by_agent)
# The usage sharing element's own package is in use in both.
for record in records:
packages = {
re.sub(r"[-_.]+", "_", name).lower() for name in
re.findall(r'<Package Name="([^"]*)">', record)}
self.assertIn("fiftyone_pipeline_engines_fiftyone", packages)


if __name__ == "__main__":
unittest.main()
Loading