From 7c4ddef92221661f269a460cff13acf9cc138e60 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 11:51:51 +0000 Subject: [PATCH 1/9] Test that usage sharing reads its lists from comma-separated strings A configuration file gives a list of names as one comma-separated string. The usage sharing element and its evidence key filter read a string given for query_whitelist or header_blacklist one character at a time, so query_whitelist="first,second" shares neither parameter and shares a parameter named "f", and header_blacklist="x-one, x-two" leaves both headers shared. These tests fail until a string is split into its names, each trimmed and with empty entries left out, as the on-premise engines already do for restricted_properties. They cover the element built in code and from a configuration file, and the evidence key filter on its own. The test that a list given in code is used as it is passes today and is there to keep that behavior. --- .../test_share_usage_evidencekeyfilter.py | 18 ++ .../tests/test_share_usage_lists.py | 167 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py index 317ccf5a..1abc7721 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -77,6 +77,24 @@ def test_lists_ignore_case(self): self.assertTrue(keys.filter_evidence_key("query.campaign")) self.assertTrue(keys.filter_evidence_key("header.user-agent")) + def test_lists_from_comma_separated_strings(self): + """! + Check that a string given for either list, the form a + configuration file gives a list in, is read as names separated by + commas, each trimmed. + """ + + keys = ShareUsageEvidenceKeyFilter( + query_whitelist="Campaign, utm_source", + header_blacklist="X-Secret,X-Other") + + self.assertTrue(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("query.utm_source")) + self.assertFalse(keys.filter_evidence_key("query.c")) + self.assertFalse(keys.filter_evidence_key("header.x-secret")) + self.assertFalse(keys.filter_evidence_key("header.x-other")) + self.assertTrue(keys.filter_evidence_key("header.x")) + def test_no_parameter_list_shares_every_parameter(self): """! Check that None in place of the parameters to share shares every diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py new file mode 100644 index 00000000..a89c1bed --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py @@ -0,0 +1,167 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""A configuration file gives a list of names as one comma-separated string, +so the usage sharing element takes its query string parameters to share and +its headers not to share in that form, as well as in a list.""" + +import json +import unittest + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +# No JavaScript or set header elements, so the pipeline holds the usage +# sharing element alone. +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + +# The package size is never reached, so the element keeps what it would +# share and sends nothing. +QUEUE_ONLY = { + "requested_package_size": 1000, + "endpoint": "http://127.0.0.1:9", +} + +# A configuration file that gives both lists as comma-separated strings. +CONFIGURATION = """ +{ + "PipelineOptions": { + "Elements": [ + { + "elementName": "ShareUsage", + "elementPath": "fiftyone_pipeline_engines_fiftyone.share_usage", + "elementParameters": { + "requested_package_size": 1000, + "endpoint": "http://127.0.0.1:9", + "query_whitelist": "first,second", + "header_blacklist": "x-one, x-two" + } + } + ] + } +} +""" + +EVIDENCE = { + "query.first": "first value", + "query.second": "second value", + "query.third": "third value", + # Named by one letter of the parameters to share. + "query.f": "letter value", + "header.user-agent": "agent value", + "header.x-one": "one value", + "header.x-two": "two value", +} + + +def shared_by(pipeline, usage): + """Processes the evidence above and returns what the usage sharing + element queued to share.""" + + flowdata = pipeline.create_flowdata() + for key, value in EVIDENCE.items(): + flowdata.evidence.add(key, value) + flowdata.process() + return "".join(usage.share_data) + + +def shared_alone_by(usage): + """What the usage sharing element shares in a pipeline of its own.""" + + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + return shared_by(pipeline, usage) + + +class ShareUsageListTests(unittest.TestCase): + + def assert_shared(self, shared, values): + for value in values: + self.assertIn(value, shared) + + def assert_not_shared(self, shared, values): + for value in values: + self.assertNotIn(value, shared) + + def test_query_parameters_to_share_from_a_string(self): + shared = shared_alone_by( + ShareUsage(query_whitelist="first,second", **QUEUE_ONLY)) + + self.assert_shared(shared, ("first value", "second value")) + self.assert_not_shared(shared, ("third value", "letter value")) + + def test_headers_not_to_share_from_a_string(self): + shared = shared_alone_by( + ShareUsage(header_blacklist="x-one, x-two", **QUEUE_ONLY)) + + self.assert_shared(shared, ("agent value",)) + self.assert_not_shared(shared, ("one value", "two value")) + + def test_lists_from_a_configuration_file(self): + pipeline = PipelineBuilder().build_from_configuration( + json.loads(CONFIGURATION)) + usage = pipeline.get_element("shareusage") + shared = shared_by(pipeline, usage) + + self.assert_shared( + shared, ("first value", "second value", "agent value")) + self.assert_not_shared( + shared, ("third value", "letter value", "one value", "two value")) + + def test_names_are_trimmed_and_empty_entries_left_out(self): + usage = ShareUsage( + query_whitelist=" first , ,second,", + header_blacklist=",x-one,, x-two ", + **QUEUE_ONLY) + + self.assertEqual(["first", "second"], usage.query_whitelist) + self.assertEqual(["x-one", "x-two"], usage.header_blacklist) + + def test_an_empty_string_is_an_empty_list(self): + usage = ShareUsage( + query_whitelist="", header_blacklist="", **QUEUE_ONLY) + + self.assertEqual([], usage.query_whitelist) + self.assertEqual([], usage.header_blacklist) + shared = shared_alone_by(usage) + self.assert_shared(shared, ("agent value", "one value")) + self.assert_not_shared(shared, ("first value",)) + + def test_a_list_is_used_as_it_is(self): + names = ["first", "second"] + headers = ["x-one", "x-two"] + usage = ShareUsage( + query_whitelist=names, header_blacklist=headers, **QUEUE_ONLY) + + self.assertIs(names, usage.query_whitelist) + self.assertIs(headers, usage.header_blacklist) + shared = shared_alone_by(usage) + self.assert_shared( + shared, ("first value", "second value", "agent value")) + self.assert_not_shared( + shared, ("third value", "letter value", "one value", "two value")) + + +if __name__ == "__main__": + unittest.main() From ec01b6349add37493fc4f5087cceecb6ec2ef622 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 11:52:59 +0000 Subject: [PATCH 2/9] Read usage sharing's lists from a comma-separated string, as a configuration file gives them The pipeline configuration specification says "Configuration files MUST be human-readable and consistent between languages." (pipeline-configuration.md line 50 at 706de1f0, https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/features/pipeline-configuration.md?plain=1#L50), and the types it then lists include a "list of strings (represented as a comma-separated string)" (line 60, https://github.com/51Degrees/specifications/blob/706de1f0cb3edd560eb7fd7ef3bb19b93a4883dc/pipeline-specification/features/pipeline-configuration.md?plain=1#L60). ShareUsage and ShareUsageEvidenceKeyFilter read a string given for query_whitelist or header_blacklist one character at a time. A configuration file giving query_whitelist as "first,second" shared neither parameter, but did share a parameter named by any single letter of the string, and one giving header_blacklist as "x-one, x-two" left both headers shared. A string given for either list is now split on commas into its names, each trimmed, with empty entries left out, as the on-premise engines do for restricted_properties. An empty string is an empty list. A list or None is used as it is, so code that passes a list sees no change. When a string is given, the ShareUsage attributes query_whitelist and header_blacklist hold the names rather than the string. --- .../share_usage.py | 16 ++++++++---- .../share_usage_evidencekeyfilter.py | 26 ++++++++++++++++--- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 176a7111..ce7244e3 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -43,7 +43,7 @@ INITIAL_RECOVERY_DELAY_SECONDS_DEFAULT, \ MAX_RECOVERY_DELAY_SECONDS_DEFAULT, RECOVERY_MULTIPLIER_DEFAULT from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter, \ - ShareUsageTrackerEvidenceKeyFilter + ShareUsageTrackerEvidenceKeyFilter, _split_names from .share_usage_tracker import ShareUsageTracker # The key the web integration gives the web request itself. Written out @@ -141,16 +141,17 @@ def __init__( again when it comes from a new session. False by default. It needs cookie to be set, and has no effect when share_all is True, because every cookie is then shared and used to find repeats. - @type query_whitelist: list + @type query_whitelist: list or str @param query_whitelist: By default query string and HTTP form parameters are not shared unless prefixed with '51D_'. If you need to share query string parameters, a list can be - specified here. None shares every parameter. - @type header_blacklist: list + specified here, or a string of names separated by commas, which is + how a configuration file gives a list. None shares every parameter. + @type header_blacklist: list or str @param header_blacklist: By default, all HTTP headers except Cookie are shared. Individual headers can be excluded from sharing by adding them - to this list. + to this list, or to a string of names separated by commas. @type share_percentage : float @param share_percentage: approximate proportion of requests to be shared. 1 = 100%, 0.5 = 50%, etc.. @@ -182,6 +183,11 @@ def __init__( super(ShareUsage, self).__init__() + # A configuration file gives each list as one comma-separated + # string, which is split into its names. A list is kept as it is. + query_whitelist = _split_names(query_whitelist) + header_blacklist = _split_names(header_blacklist) + self.query_whitelist = query_whitelist self.header_blacklist = header_blacklist diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index dc4d4264..93da177f 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -36,6 +36,19 @@ NEVER_SHARED_KEYS = ( "id.email", "id.salt", "request-query", "request-path", "header.cookie") + +def _split_names(names): + """! + A configuration file gives a list of names as one comma-separated + string, so a string is split on commas into its names, each trimmed, + with empty entries left out. Any other value, such as a list or None, is + returned as it is. + """ + if isinstance(names, str): + return [name.strip() for name in names.split(",") if name.strip()] + return names + + class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees @@ -56,11 +69,13 @@ def __init__( Constructor for ShareUsageEvidenceKeyFilter @type cookie: string @param cookie: the name of the cookie that identifies the session - @type query_whitelist: list + @type query_whitelist: list or str @param query_whitelist: query string and form parameters to share as - well as those starting 51D_, or None to share every parameter - @type header_blacklist: list - @param header_blacklist: headers not to share + well as those starting 51D_, or None to share every parameter. A + string is read as names separated by commas. + @type header_blacklist: list or str + @param header_blacklist: headers not to share. A string is read as + names separated by commas. @type include_session: bool @param include_session: whether the session cookie is included @type share_all: bool @@ -69,6 +84,9 @@ def __init__( """ self.cookie = cookie.lower() if cookie else None + query_whitelist = _split_names(query_whitelist) + header_blacklist = _split_names(header_blacklist) + self.query_whitelist = None if query_whitelist is None else \ [parameter.lower() for parameter in query_whitelist] From 63b5fd3e0cd5ae55a78e342ca70537f751bd3d99 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 15:27:32 +0000 Subject: [PATCH 3/9] Test the usage sharing rules for query values, cookies and packages The usage sharing specification on its rc now says three things the element does not do (specifications#155 and #156, merged as 33de68a0 and 30b3741f). - "Share all query string parameters is on by default, so every query string and HTTP form parameter is shared unless a user turns it off." - "It MUST also return false, whatever the configuration and even when all Evidence is shared, for any key whose prefix is `cookie.` and whose field does not start with `51D_`." - "An xml element MUST be written for every 51Degrees package in use, being the package that holds the Pipeline, the package that holds each FlowElement in the Pipeline, and every 51Degrees package those depend on." test_share_usage_rules.py tests each, through the message the element sends: query and form values shared by default, the new share_all_query_string_parameters setting turned off in code and in a configuration file, only 51D_ cookies shared when all Evidence is, the session cookie compared but never shared whether or not all Evidence is shared, and a Package element with its version for every 51Degrees package in use and no other. The existing tests that held the old rules change with them. With no settings a query parameter such as password is now shared, sharing all Evidence no longer shares a site's cookies, the filter test of sharing all Evidence expects the same, and the tests of a list of parameters to share, including those for comma-separated lists, turn sharing every parameter off, as only then does the list decide. --- .../test_share_usage_accepts_no_evidence.py | 3 +- .../test_share_usage_evidencekeyfilter.py | 36 ++- .../tests/test_share_usage_lists.py | 14 +- .../tests/test_share_usage_rules.py | 282 ++++++++++++++++++ .../tests/test_usage.py | 22 +- 5 files changed, 342 insertions(+), 15 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py index 503abff0..4d4bad06 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py @@ -174,8 +174,7 @@ def test_the_method_is_shared_and_the_path_and_cookie_header_are_not(self): shared = "".join(usage.share_data) self.assertIn("request-method", shared) - self.assertIn('cookievalue', - shared) + self.assertNotIn("cookievalue", shared) self.assertNotIn("/page", shared) self.assertNotIn('
', shared) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py index 1abc7721..b31e3ea1 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -125,13 +125,16 @@ def test_session_cookie(self): def test_share_all(self): """! Check that sharing everything shares the evidence the other rules - leave out, apart from the evidence that is never shared. + leave out, apart from the evidence that is never shared and the + cookies that do not start 51D_. """ keys = ShareUsageEvidenceKeyFilter(share_all=True) self.assertFalse(keys.filter_evidence_key("header.cookie")) - self.assertTrue(keys.filter_evidence_key("cookie.sessionid")) + self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) + self.assertTrue( + keys.filter_evidence_key("cookie.51D_screenpixelswidth")) self.assertTrue(keys.filter_evidence_key("query.password")) self.assertTrue(keys.filter_evidence_key("query.id.usage")) self.assertTrue(keys.filter_evidence_key("server.request-method")) @@ -142,6 +145,35 @@ def test_share_all(self): self.assertFalse(keys.filter_evidence_key("server.request-query")) self.assertFalse(keys.filter_evidence_key("server.request-path")) + def test_every_query_parameter_when_sharing_them_all(self): + """! + Check that sharing every query string parameter shares one not on + the list, and that a filter made without it keeps to the list. + """ + + every = ShareUsageEvidenceKeyFilter( + query_whitelist=["campaign"], + share_all_query_string_parameters=True) + listed = ShareUsageEvidenceKeyFilter(query_whitelist=["campaign"]) + + self.assertTrue(every.filter_evidence_key("query.password")) + self.assertFalse(listed.filter_evidence_key("query.password")) + self.assertTrue(listed.filter_evidence_key("query.campaign")) + + def test_the_session_cookie_is_kept_for_the_tracker_alone(self): + """! + Check that, when all evidence is shared, the session cookie is kept + by the tracker's filter and by no other. + """ + + kept = ShareUsageEvidenceKeyFilter( + cookie="SessionId", include_session=True, share_all=True) + sent = ShareUsageEvidenceKeyFilter(cookie="SessionId", share_all=True) + + self.assertTrue(kept.filter_evidence_key("cookie.sessionid")) + self.assertFalse(sent.filter_evidence_key("cookie.sessionid")) + self.assertFalse(kept.filter_evidence_key("cookie._ga")) + def test_never_shared_in_every_mode(self): """! Check that the email address, the salt for hashed email diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py index a89c1bed..9209ae53 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py @@ -44,6 +44,10 @@ "endpoint": "http://127.0.0.1:9", } +# Shares only the query string parameters listed and those starting 51D_, +# where every one is shared by default. +LISTED_ONLY = {"share_all_query_string_parameters": False} + # A configuration file that gives both lists as comma-separated strings. CONFIGURATION = """ { @@ -55,6 +59,7 @@ "elementParameters": { "requested_package_size": 1000, "endpoint": "http://127.0.0.1:9", + "share_all_query_string_parameters": false, "query_whitelist": "first,second", "header_blacklist": "x-one, x-two" } @@ -106,7 +111,8 @@ def assert_not_shared(self, shared, values): def test_query_parameters_to_share_from_a_string(self): shared = shared_alone_by( - ShareUsage(query_whitelist="first,second", **QUEUE_ONLY)) + ShareUsage(query_whitelist="first,second", **LISTED_ONLY, + **QUEUE_ONLY)) self.assert_shared(shared, ("first value", "second value")) self.assert_not_shared(shared, ("third value", "letter value")) @@ -140,7 +146,8 @@ def test_names_are_trimmed_and_empty_entries_left_out(self): def test_an_empty_string_is_an_empty_list(self): usage = ShareUsage( - query_whitelist="", header_blacklist="", **QUEUE_ONLY) + query_whitelist="", header_blacklist="", **LISTED_ONLY, + **QUEUE_ONLY) self.assertEqual([], usage.query_whitelist) self.assertEqual([], usage.header_blacklist) @@ -152,7 +159,8 @@ def test_a_list_is_used_as_it_is(self): names = ["first", "second"] headers = ["x-one", "x-two"] usage = ShareUsage( - query_whitelist=names, header_blacklist=headers, **QUEUE_ONLY) + query_whitelist=names, header_blacklist=headers, + **LISTED_ONLY, **QUEUE_ONLY) self.assertIs(names, usage.query_whitelist) self.assertIs(headers, usage.header_blacklist) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py new file mode 100644 index 00000000..2adef573 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py @@ -0,0 +1,282 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +What usage sharing shares by default and when all Evidence is shared, and +the packages each record names. Every query string and form parameter is +shared unless that is turned off, no cookie but a 51D_ one is ever shared, +the session cookie is still compared when sessions are tracked, and each +record names every 51Degrees package in use with its version. +""" + +import gzip +import json +import re +import unittest +from importlib import metadata +from unittest import mock + +import requests +from flask import Flask, request + +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.web import webevidence +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + +# The package size is never reached, so the element keeps what it would +# share and sends nothing until it is closed. +QUEUE_ONLY = { + "requested_package_size": 1000, + "endpoint": "http://127.0.0.1:9", +} + +# The packages a Pipeline holding only usage sharing uses, being the one +# holding the Pipeline, the one holding the element and the one that +# requires. +USAGE_SHARING_PACKAGES = ( + "fiftyone_pipeline_core", + "fiftyone_pipeline_engines", + "fiftyone_pipeline_engines_fiftyone", +) + +# Turns off sharing every query string parameter, from a configuration +# file. +CONFIGURATION = """ +{ + "PipelineOptions": { + "Elements": [ + { + "elementName": "ShareUsage", + "elementPath": "fiftyone_pipeline_engines_fiftyone.share_usage", + "elementParameters": { + "requested_package_size": 1000, + "endpoint": "http://127.0.0.1:9", + "share_all_query_string_parameters": false + } + } + ] + } +} +""" + + +def canonical(name): + """A package name as the package index compares names.""" + return re.sub(r"[-_.]+", "_", name).lower() + + +def sent_on_close(usage): + """Closes the element, which sends what it has queued, and returns each + record it posted. The post is answered without leaving the process.""" + sent = [] + + def post(url, headers=None, data=None, **kwargs): + sent.append(gzip.decompress(data).decode("utf8")) + return mock.Mock(status_code=200, text="") + + with mock.patch.object(requests, "post", post): + usage.close() + return [record for message in sent + for record in re.findall(r".*?", message)] + + +def records_of(usage, requests_evidence, *elements): + """Processes each request's Evidence through a Pipeline holding the + elements and the usage sharing element, and returns the records sent.""" + builder = PipelineBuilder(NO_DEFAULT_ELEMENTS) + for element in elements: + builder.add(element) + pipeline = builder.add(usage).build() + for evidence in requests_evidence: + flowdata = pipeline.create_flowdata() + for key, value in evidence.items(): + flowdata.evidence.add(key, value) + flowdata.process() + return sent_on_close(usage) + + +def packages_in(record): + """The Package elements of a record, as name and version pairs.""" + return re.findall(r'([^<]*)', record) + + +class StarSign(FlowElement): + """An element from no installed package.""" + + def __init__(self): + super(StarSign, self).__init__() + self.datakey = "starsign" + + def process_internal(self, flow_data): + pass + + +class QueryAndFormValuesTests(unittest.TestCase): + + def test_every_query_value_is_shared_by_default(self): + records = records_of(ShareUsage(**QUEUE_ONLY), [{ + "header.user-agent": "agent value", + "query.campaign": "campaign value", + "query.password": "password value", + }]) + + self.assertEqual(1, len(records)) + self.assertIn('campaign value', + records[0]) + self.assertIn('password value', + records[0]) + + def test_every_form_value_of_the_web_request_is_shared_by_default(self): + app = Flask(__name__) + usage = ShareUsage(**QUEUE_ONLY) + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + with app.test_request_context( + "/", method="POST", data={"field": "form value"}): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.process() + records = sent_on_close(usage) + + self.assertEqual(1, len(records)) + self.assertIn('form value', records[0]) + + def test_turned_off_only_51d_and_listed_values_are_shared(self): + records = records_of(ShareUsage( + share_all_query_string_parameters=False, + query_whitelist=["campaign"], **QUEUE_ONLY), [{ + "header.user-agent": "agent value", + "query.campaign": "campaign value", + "query.51d_gethighentropyvalues": "hev", + "query.password": "password value", + }]) + + self.assertIn("campaign value", records[0]) + self.assertIn("hev", records[0]) + self.assertNotIn("password value", records[0]) + + def test_turned_off_from_a_configuration_file(self): + pipeline = PipelineBuilder().build_from_configuration( + json.loads(CONFIGURATION)) + usage = pipeline.get_element("shareusage") + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "agent value") + flowdata.evidence.add("query.password", "password value") + flowdata.process() + records = sent_on_close(usage) + + self.assertFalse(usage.share_all_query_string_parameters) + self.assertIn("agent value", records[0]) + self.assertNotIn("password value", records[0]) + + +class CookieTests(unittest.TestCase): + + def test_only_51d_cookies_are_shared_when_all_evidence_is_shared(self): + records = records_of(ShareUsage(share_all=True, **QUEUE_ONLY), [{ + "header.user-agent": "agent value", + "cookie.51d_screenpixelswidth": "1080", + "cookie.sessionid": "session value", + "cookie._ga": "analytics value", + "session.basket": "basket value", + }]) + + self.assertIn('1080', + records[0]) + self.assertNotIn("session value", records[0]) + self.assertNotIn("analytics value", records[0]) + # Sharing all Evidence still shares what it shared before, other + # than cookies. + self.assertIn("basket value", records[0]) + + def test_the_session_cookie_is_compared_but_never_shared(self): + for share_all in (False, True): + with self.subTest(share_all=share_all): + records = records_of(ShareUsage( + share_all=share_all, track_session=True, + cookie="sessionid", **QUEUE_ONLY), [ + {"header.user-agent": "agent value", + "cookie.sessionid": "first session"}, + {"header.user-agent": "agent value", + "cookie.sessionid": "first session"}, + {"header.user-agent": "agent value", + "cookie.sessionid": "second session"}, + ]) + + # The repeat in the first session is not shared, and the + # same Evidence from a new session is. + self.assertEqual(2, len(records)) + for record in records: + self.assertNotIn("first session", record) + self.assertNotIn("second session", record) + + +class PackageTests(unittest.TestCase): + + def test_each_record_names_every_51degrees_package_in_use(self): + records = records_of(ShareUsage(**QUEUE_ONLY), [ + {"header.user-agent": "first"}, + {"header.user-agent": "second"}, + ]) + + self.assertEqual(2, len(records)) + for record in records: + packages = packages_in(record) + self.assertEqual( + sorted(USAGE_SHARING_PACKAGES), + sorted(canonical(name) for name, _ in packages)) + for name, version in packages: + self.assertEqual(metadata.version(name), version, name) + + def test_packages_follow_the_flow_elements(self): + record = records_of(ShareUsage(**QUEUE_ONLY), [ + {"header.user-agent": "agent value"}])[0] + + self.assertRegex( + record, + r"(]*>[^<]*)+") + + def test_an_element_from_no_package_names_no_package(self): + record = records_of(ShareUsage(**QUEUE_ONLY), [ + {"header.user-agent": "agent value"}], StarSign())[0] + + self.assertIn("StarSign", record) + self.assertEqual( + sorted(USAGE_SHARING_PACKAGES), + sorted(canonical(name) for name, _ in packages_in(record))) + + def test_packages_that_are_not_51degrees_are_not_named(self): + record = records_of(ShareUsage(**QUEUE_ONLY), [ + {"header.user-agent": "agent value"}])[0] + + names = [canonical(name) for name, _ in packages_in(record)] + for other in ("requests", "cachetools", "flask"): + self.assertNotIn(other, names) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 95ac18ac..3cc6acd9 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -911,9 +911,9 @@ def test_escapes_xml_characters(self): def test_default_shares_only_accepted_evidence(self): """! - Check that with no options the element shares headers and the - 51Degrees cookies and parameters, and leaves out other cookies, the - Cookie header, and other query string and form parameters. + Check that with no options the element shares headers, the + 51Degrees cookies and every query string and form parameter, and + leaves out other cookies, the Cookie header and the email address. """ usage = ShareUsage( @@ -940,7 +940,7 @@ def test_default_shares_only_accepted_evidence(self): self.assertIn('hev', app.received[0]) self.assertNotIn("cookie value", app.received[0]) self.assertNotIn("header value", app.received[0]) - self.assertNotIn("password value", app.received[0]) + self.assertIn('password value', app.received[0]) self.assertNotIn("email value", app.received[0]) def test_never_shared_evidence(self): @@ -976,7 +976,8 @@ def test_never_shared_evidence(self): def test_share_all(self): """! Check that sharing all evidence shares what the other rules leave - out, but not the evidence that is never shared. + out, but not the evidence that is never shared or a cookie that + does not start 51D_. """ usage = ShareUsage( @@ -989,13 +990,15 @@ def test_share_all(self): .build() flowdata = pipeline.create_flowdata() flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("cookie.51d_screenpixelswidth", "1080") flowdata.evidence.add("query.campaign", "campaign value") flowdata.evidence.add("query.id.email", "email value") flowdata.process() self.waitForUsageThreads() self.assertEqual(len(app.received), 1) - self.assertIn('cookie value', app.received[0]) + self.assertNotIn("cookie value", app.received[0]) + self.assertIn('1080', app.received[0]) self.assertIn('campaign value', app.received[0]) self.assertNotIn("email value", app.received[0]) @@ -1008,6 +1011,7 @@ def test_parameter_names_with_dots(self): usage = ShareUsage( requested_package_size=1, endpoint=testEndpoint, + share_all_query_string_parameters=False, query_whitelist=["valid.email"]) pipeline = (PipelineBuilder())\ @@ -1025,8 +1029,9 @@ def test_parameter_names_with_dots(self): def test_empty_query_whitelist(self): """! - Check that an empty list of parameters to share shares only those - starting 51D_, and that None shares every parameter. + Check that, with sharing every parameter turned off, an empty + list of parameters to share shares only those starting 51D_, and + that None shares every parameter. """ for query_whitelist, shared in (([], False), (None, True)): @@ -1034,6 +1039,7 @@ def test_empty_query_whitelist(self): usage = ShareUsage( requested_package_size=1, endpoint=testEndpoint, + share_all_query_string_parameters=False, query_whitelist=query_whitelist, header_blacklist=["x-secret"]) From f6bf51a8421832d203f4dc94ca9cbddccb4204cf Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 15:29:15 +0000 Subject: [PATCH 4/9] Share every query value by default, no cookie but 51D_ ones, and name every package The usage sharing specification on its rc (specifications#155 and #156) says every query string and form parameter is shared unless a user turns that off, no cookie whose name does not start 51D_ is ever shared, even when all Evidence is, and each record names every 51Degrees package in use with its version. - ShareUsage takes share_all_query_string_parameters, True by default, and passes it to both of its filters. With it on, every query string and form parameter is shared. With it off, only those starting 51D_ and those in query_whitelist are, as before. None for query_whitelist still shares every parameter. - ShareUsageEvidenceKeyFilter decides a cookie before it looks at share_all, so share_all no longer shares a site's own cookies. The tracker's filter still keeps the session cookie when sessions are tracked, so track_session works the same whether or not all Evidence is shared, and the session cookie is never sent. The filter's own default for the new setting is False, so a filter made directly keeps to its list as before. - Each record holds a Package element for every 51Degrees package in use, after the FlowElement elements. These are fiftyone_pipeline_core, the package of each flow element and every 51Degrees package those require, found from the installed packages' metadata, each named once under its published name with its installed version. A package is a 51Degrees one when its author or author email names 51Degrees, as every package this repository publishes does, so an element an application defines, and a package such as requests, adds none. --- .../share_usage.py | 131 ++++++++++++++++-- .../share_usage_evidencekeyfilter.py | 46 +++--- 2 files changed, 144 insertions(+), 33 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index ce7244e3..b66ca341 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -31,6 +31,7 @@ import threading import time from concurrent.futures import ThreadPoolExecutor +from importlib import metadata from importlib.metadata import version import requests @@ -115,7 +116,8 @@ def __init__( failures_window_seconds = FAILURES_WINDOW_SECONDS_DEFAULT, initial_recovery_delay_seconds = INITIAL_RECOVERY_DELAY_SECONDS_DEFAULT, max_recovery_delay_seconds = MAX_RECOVERY_DELAY_SECONDS_DEFAULT, - recovery_multiplier = RECOVERY_MULTIPLIER_DEFAULT): + recovery_multiplier = RECOVERY_MULTIPLIER_DEFAULT, + share_all_query_string_parameters = True): """! Constructor for ShareUsage element @@ -134,19 +136,20 @@ def __init__( @type cookie: string @param cookie: The name of the cookie that holds the session id. It has no effect unless track_session is True. The cookie itself - is not shared unless share_all is True. + is never shared unless its name starts with '51D_', as no other + cookie is. @type track_session: bool @param track_session: If True, the cookie named by cookie is used to tell user sessions apart, so evidence already shared is shared again when it comes from a new session. False by default. It - needs cookie to be set, and has no effect when share_all is True, - because every cookie is then shared and used to find repeats. + needs cookie to be set, and works the same whether or not + share_all is True. @type query_whitelist: list or str - @param query_whitelist: By default query string and HTTP form - parameters are not shared unless prefixed with '51D_'. - If you need to share query string parameters, a list can be - specified here, or a string of names separated by commas, which is - how a configuration file gives a list. None shares every parameter. + @param query_whitelist: The query string and HTTP form parameters + to share, as well as those prefixed with '51D_', when + share_all_query_string_parameters is False. A string of names + separated by commas, which is how a configuration file gives a + list, is read as those names. None shares every parameter. @type header_blacklist: list or str @param header_blacklist: By default, all HTTP headers except Cookie are shared. @@ -157,8 +160,14 @@ def __init__( be shared. 1 = 100%, 0.5 = 50%, etc.. @type share_all: bool @param share_all: If True, all evidence is shared apart from the - email address, the salt used for hashed email identifiers, and the - request's query string and path. False by default. + email address, the salt used for hashed email identifiers, the + request's query string and path, and every cookie whose name does + not start with '51D_'. False by default. + @type share_all_query_string_parameters: bool + @param share_all_query_string_parameters: If True, every query + string and HTTP form parameter is shared. If False, only those + prefixed with '51D_' and those in query_whitelist are. True by + default. @type max_queue_size: int @param max_queue_size: The most entries held while waiting to be sent. When the queue is full, new entries are dropped. The default @@ -201,6 +210,8 @@ def __init__( self.endpoint = endpoint self.request_timeout = request_timeout self.share_all = share_all + self.share_all_query_string_parameters = \ + share_all_query_string_parameters self.track_session = track_session # The session id and sequence number are kept in evidence, as the @@ -212,15 +223,15 @@ def __init__( # Vary header never depend on it. It shares what it finds instead, # and these filters decide which of that is shared and which tells # a repeat apart. When sessions are tracked the tracker reads the - # session cookie, which the data sent leaves out unless all evidence - # is shared. + # session cookie, which the data sent leaves out. self.evidence_key_filter = BasicListEvidenceKeyFilter([]) self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter( - cookie, included, header_blacklist, False, share_all) + cookie, included, header_blacklist, False, share_all, + share_all_query_string_parameters) self.tracker_evidence_key_filter = \ ShareUsageTrackerEvidenceKeyFilter( cookie, included, header_blacklist, track_session, - share_all) + share_all, share_all_query_string_parameters) # Add the share usage tracker which detects when # to send up sharing data @@ -407,6 +418,10 @@ def get_constant_xml(self): # The flow elements in the current pipeline for flow_element in self.get_flow_elements(): xml += f"{flow_element}" + # The release of each 51Degrees package in use + for name, package_version in fiftyone_packages( + self._pipeline_flow_elements()): + xml += record_element("Package", package_version, name) xml += "Python" # The software language version xml += f"{pyVersion}" @@ -428,6 +443,16 @@ def _look_up_server_ip(self): self._server_ip = "" self._server_ip_found.set() + def _pipeline_flow_elements(self): + """! + The flow elements of the one Pipeline this element is in, or none + when it is in more than one, as get_flow_elements gives no names + then either. + """ + if len(self.pipelines) == 1: + return list(self.pipelines[0].flow_elements) + return [] + def get_flow_elements(self): if len(self.pipelines) == 1: list = [] @@ -801,6 +826,82 @@ def host_address(): return str(addresses[0]) if addresses else "" +# The text the author or author email of every package 51Degrees +# publishes holds. +FIFTYONE_AUTHOR = "51degrees" + +# The package that holds the Pipeline. +PIPELINE_PACKAGE = "fiftyone_pipeline_core" + + +def canonical_package_name(name): + """! + A package name as the package index compares names, so that each + package is named once however its name is spelled. + """ + return re.sub(r"[-_.]+", "_", name).lower() + + +def _fiftyone_distribution(name): + """! + The installed package of that name if 51Degrees publishes it, or None. + """ + try: + distribution = metadata.distribution(name) + except (metadata.PackageNotFoundError, ValueError): + return None + author = " ".join( + distribution.metadata.get(field) or "" + for field in ("Author", "Author-email")).lower() + return distribution if FIFTYONE_AUTHOR in author else None + + +def _required_names(distribution): + """! + The names of the packages a package requires, leaving out those it + requires only for an extra. + """ + names = [] + for requirement in distribution.requires or []: + if re.search(r"\bextra\s*==", requirement): + continue + match = re.match(r"\s*([A-Za-z0-9][A-Za-z0-9._-]*)", requirement) + if match: + names.append(match.group(1)) + return names + + +def fiftyone_packages(flow_elements): + """! + The name and version of every 51Degrees package in use, sorted by name + and each given once. These are the package that holds the Pipeline, + the package that holds each flow element and every 51Degrees package + those require. A flow element from no installed package, such as one + an application defines, adds none. + + @type flow_elements: list + @param flow_elements: the flow elements of the Pipeline + @rtype: list + @return: (name, version) pairs, with the name the package is published + under + """ + names = [PIPELINE_PACKAGE] + [ + type(element).__module__.partition(".")[0] + for element in flow_elements] + found = {} + while names: + distribution = _fiftyone_distribution(names.pop()) + if distribution is None: + continue + name = distribution.metadata["Name"] + key = canonical_package_name(name) + if key in found: + continue + found[key] = (name, distribution.version) + names.extend(_required_names(distribution)) + return [found[key] for key in sorted(found)] + + def plain_element(tag, value): """! An element of a usage sharing record that is not an Evidence entry, with diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index 93da177f..c813f627 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -52,10 +52,12 @@ def _split_names(names): class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees - ShareUsage service is sent. Headers are shared unless they are blocked, - cookies and session values only when their names start with 51D_, and - query string and form parameters only when their names start with 51D_ or - are on the list to share. Any other evidence is shared. + ShareUsage service is sent. Cookies are shared only when their names + start with 51D_, whatever the other settings. Headers are shared unless + they are blocked, session values only when their names start with 51D_, + and query string and form parameters when all of them are shared or + when their names start with 51D_ or are on the list to share. Any other + evidence is shared. """ def __init__( @@ -64,7 +66,8 @@ def __init__( query_whitelist = (), header_blacklist = (), include_session = False, - share_all = False): + share_all = False, + share_all_query_string_parameters = False): """! Constructor for ShareUsageEvidenceKeyFilter @type cookie: string @@ -80,7 +83,10 @@ def __init__( @param include_session: whether the session cookie is included @type share_all: bool @param share_all: whether all evidence is shared, apart from the - evidence that is never shared + evidence that is never shared and cookies not starting 51D_ + @type share_all_query_string_parameters: bool + @param share_all_query_string_parameters: whether every query + string and form parameter is shared, whatever query_whitelist holds """ self.cookie = cookie.lower() if cookie else None @@ -97,6 +103,9 @@ def __init__( self.share_all = share_all + self.share_all_query_string_parameters = \ + share_all_query_string_parameters + def filter_evidence_key(self, key): """! @@ -115,33 +124,34 @@ def filter_evidence_key(self, key): for never_shared in NEVER_SHARED_KEYS): return False - if self.share_all: - return True - # The prefix is everything before the first dot and the field # everything after it, so a field can itself contain dots. prefix, separator, field = key.partition(".") + # A cookie a site sets for itself is never shared, even when all + # evidence is. The tracker alone keeps the session cookie. + if separator and prefix == "cookie": + return field.startswith(FIFTYONE_PREFIX) or \ + (self.include_session and field == self.cookie) + + if self.share_all: + return True + if not separator: return True elif prefix == "header": # Add the header to the list if the header name does not # appear in the list of blocked headers return field not in self.header_blacklist - elif prefix == "cookie": - # Only add cookies that start with the 51Degrees cookie - # prefix - return field.startswith(FIFTYONE_PREFIX) or \ - (self.include_session and field == self.cookie) elif prefix == "session": # Only session values that start with the 51Degrees # cookie prefix return field.startswith(FIFTYONE_PREFIX) elif prefix == "query": - # If no query string parameter list was given then share all - # of them. Otherwise, only include query string parameters - # that start with 51d_ or that are in the list. - return self.query_whitelist is None or \ + # Share every parameter when asked to or when no list was + # given. Otherwise only those starting 51d_ or in the list. + return self.share_all_query_string_parameters or \ + self.query_whitelist is None or \ field.startswith(FIFTYONE_PREFIX) or \ field in self.query_whitelist else: From 858c373ccef38e963f939d44cf265affedbcaec2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 15 Nov 2026 15:45:46 +0000 Subject: [PATCH 5/9] Hold the web request test to the list of query parameters test_the_web_request_is_shared_by_the_same_rules checks that a query parameter of the web request that is not on the list is left out, as it would be from Evidence. Every query parameter is now shared by default, so the test turns that off to keep checking the list. --- .../tests/test_share_usage_accepts_no_evidence.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py index 4d4bad06..66761816 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py @@ -138,7 +138,11 @@ def test_a_revalidation_has_the_same_accepted_evidence(self): self.assertNotIn("header.if-none-match", accepted[1]) def test_the_web_request_is_shared_by_the_same_rules(self): - usage = usage_element(header_blacklist=["x-blocked"]) + # Sharing every query parameter is turned off, so that the query + # string of the request is held to the list as Evidence is. + usage = usage_element( + header_blacklist=["x-blocked"], + share_all_query_string_parameters=False) flowdata = pipeline_with(usage).create_flowdata() flowdata.evidence.add_from_dict(request_evidence({ "User-Agent": CHROME, From fd37a9005fb87098a48db0d7998547643b2c9b20 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 20:59:05 +0000 Subject: [PATCH 6/9] Test: write the invariant values once an hour and a process identifier on every record usage-sharing-element.md at specifications 07e1fced, lines 150 to 153: "The invariant values MUST be written to the first record the element writes. They MUST be written again to the first record the element writes once an hour or more has passed since it last wrote them. They MUST NOT be written to any other record." Lines 115 and 116: "The element MUST create a process identifier when it is built, and MUST write it to every record as the ProcessId element." The tests move the element's clock by hand rather than waiting an hour. The package test and the two Pipeline test now expect the packages and flow elements in the first record only. --- .../test_share_usage_invariant_values.py | 337 ++++++++++++++++++ .../tests/test_share_usage_pipelines.py | 25 ++ .../tests/test_share_usage_rules.py | 25 +- 3 files changed, 378 insertions(+), 9 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py new file mode 100644 index 00000000..daacae25 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py @@ -0,0 +1,337 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The values that do not change while a usage sharing element lives, written +to its first record and again once an hour, and the process identifier +written to every record. + +The rules are those of the specifications repository at commit 07e1fced, +pipeline-specification/pipeline-elements/usage-sharing-element.md. The +tests control the element's clock rather than waiting an hour. +""" + +import gzip +import os +import re +import subprocess +import sys +import threading +import unittest +from unittest import mock + +import requests + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone import share_usage +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + +# The package size is never reached, so the element keeps what it would +# share and sends nothing until it is closed. +QUEUE_ONLY = { + "requested_package_size": 1000, + "endpoint": "http://127.0.0.1:9", +} + +# usage-sharing-element.md lines 138 to 145 at 07e1fced list the invariant +# values, and lines 208 to 210 say "A record that leaves them out has no +# Version, Product, FlowElement, Package, Language, LanguageVersion, +# ServerIP or Platform element". +INVARIANT_ELEMENTS = ( + "Version", "Product", "FlowElement", "Package", "Language", + "LanguageVersion", "ServerIP", "Platform", +) + +HOUR = 3600 + +# An address from the range RFC 5737 sets aside for documentation. +SERVER_IP = "192.0.2.10" + +# How long a test waits for the lookup of the server's address it holds up. +LOOKUP_WAIT_SECONDS = 10 + +# Lines 124 and 125: "32 hexadecimal digits in groups of 8, 4, 4, 4 and 12 +# separated by hyphens", for a version 4 UUID (line 119). +UUID4 = re.compile( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}" + r"-[0-9a-fA-F]{12}\Z") + + +class Clock: + """A clock the test moves by hand, in seconds.""" + + def __init__(self): + self.now = 1000.0 + + def __call__(self): + return self.now + + +def sent_on_close(usage): + """Closes the element, which sends what it has queued, and returns each + record it posted, in the order written. The post is answered without + leaving the process.""" + sent = [] + + def post(url, headers=None, data=None, **kwargs): + sent.append(gzip.decompress(data).decode("utf8")) + return mock.Mock(status_code=200, text="") + + with mock.patch.object(requests, "post", post): + usage.close() + return [record for message in sent + for record in re.findall(r".*?", message)] + + +def process(pipeline, user_agent): + """Processes one request. Each test gives each request its own + User-Agent, so that no request repeats one already shared.""" + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", user_agent) + flowdata.process() + + +def invariants_in(record): + """The names of the invariant elements a record carries.""" + return [name for name in INVARIANT_ELEMENTS + if re.search("<" + name + r"[ >]", record)] + + +def process_ids_in(record): + return re.findall(r"([^<]*)", record) + + +def element_with_clock(wait_for_server_ip=True): + """A usage sharing element in a Pipeline of its own, with a clock the + test moves. Unless asked not to, it waits until the lookup of the + server's address has ended, as a record written before then does not + count as carrying the invariant values.""" + usage = ShareUsage(**QUEUE_ONLY) + if wait_for_server_ip: + usage._server_ip_found.wait(LOOKUP_WAIT_SECONDS) + clock = Clock() + usage.clock = clock + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + return usage, clock, pipeline + + +class InvariantValuesTests(unittest.TestCase): + + def test_the_first_record_carries_the_invariant_values(self): + # Line 150: "The invariant values MUST be written to the first + # record the element writes." + usage, clock, pipeline = element_with_clock() + process(pipeline, "first") + + records = sent_on_close(usage) + + self.assertEqual(1, len(records)) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[0])) + + def test_records_within_the_hour_leave_the_invariant_values_out(self): + # Lines 151 to 153: "They MUST be written again to the first record + # the element writes once an hour or more has passed since it last + # wrote them. They MUST NOT be written to any other record." + usage, clock, pipeline = element_with_clock() + process(pipeline, "first") + clock.now += 1 + process(pipeline, "second") + clock.now += HOUR - 1.5 + process(pipeline, "third") + + records = sent_on_close(usage) + + self.assertEqual(3, len(records)) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[0])) + self.assertEqual([], invariants_in(records[1])) + self.assertEqual([], invariants_in(records[2])) + + def test_the_invariant_values_are_written_again_once_the_hour_is_up(self): + # Lines 151 and 152, "once an hour or more has passed since it last + # wrote them", so the hour runs from the record that last carried + # them, here the one written exactly an hour after the first. + usage, clock, pipeline = element_with_clock() + process(pipeline, "first") + clock.now += HOUR + process(pipeline, "an hour later") + clock.now += 1 + process(pipeline, "a second after that") + clock.now += HOUR - 1 + process(pipeline, "an hour after the second write") + clock.now += 2 * HOUR + 5 + process(pipeline, "long after") + + records = sent_on_close(usage) + + self.assertEqual( + [True, True, False, True, True], + [invariants_in(record) == list(INVARIANT_ELEMENTS) + for record in records]) + self.assertEqual([], invariants_in(records[2])) + + def test_a_record_that_leaves_them_out_is_otherwise_the_same(self): + # Lines 208 to 210: a record that leaves the invariant values out + # "is otherwise the same". + usage, clock, pipeline = element_with_clock() + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", "first") + flowdata.evidence.add("query.51D_value", "one") + flowdata.process() + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", "second") + flowdata.evidence.add("query.51D_value", "two") + flowdata.process() + + later = sent_on_close(usage)[1] + + for name in ("ProcessId", "SessionId", "Sequence", "ClientIP", + "DateSent"): + self.assertIn("<" + name + ">", later) + self.assertIn('
second
', later) + self.assertIn('two', later) + + def test_a_record_written_before_the_server_ip_is_known_is_not_counted( + self): + # Processing does not wait for the lookup of the server's address, + # so a record written before it ends has an empty ServerIP. That + # record does not hold every invariant value, so the next record + # written once the lookup has ended carries them all, and the hour + # runs from that record. + release = threading.Event() + self.addCleanup(release.set) + + def slow(): + release.wait(LOOKUP_WAIT_SECONDS) + return SERVER_IP + + with mock.patch.object(share_usage, "host_address", slow): + usage, clock, pipeline = element_with_clock( + wait_for_server_ip=False) + process(pipeline, "before the lookup ends") + release.set() + self.assertTrue(usage._server_ip_found.wait(LOOKUP_WAIT_SECONDS)) + clock.now += 1 + process(pipeline, "after the lookup ends") + clock.now += HOUR - 2 + process(pipeline, "within the hour of that") + + records = sent_on_close(usage) + + self.assertEqual(3, len(records)) + self.assertIn("", records[0]) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[1])) + self.assertIn("" + SERVER_IP + "", records[1]) + self.assertEqual([], invariants_in(records[2])) + + +# Writes one record from a new element and prints its ProcessId, so that a +# test can compare the identifiers of two processes. +ONE_RECORD = """ +import gzip, re +from unittest import mock +import requests +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage +sent = [] +def post(url, headers=None, data=None, **kwargs): + sent.append(gzip.decompress(data).decode("utf8")) + return mock.Mock(status_code=200, text="") +usage = ShareUsage(requested_package_size=1000, endpoint="http://127.0.0.1:9") +pipeline = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}).add(usage).build() +flowdata = pipeline.create_flowdata() +flowdata.evidence.add("header.user-agent", "agent") +flowdata.process() +with mock.patch.object(requests, "post", post): + usage.close() +print(",".join(re.findall("([^<]*)", "".join(sent)))) +""" + + +class ProcessIdentifierTests(unittest.TestCase): + + def test_every_record_carries_the_same_identifier(self): + # Lines 115 and 116: "The element MUST create a process identifier + # when it is built, and MUST write it to every record as the + # ProcessId element." + usage, clock, pipeline = element_with_clock() + for index in range(5): + process(pipeline, f"agent {index}") + clock.now += HOUR / 2 + + records = sent_on_close(usage) + + self.assertEqual(5, len(records)) + identifiers = [process_ids_in(record) for record in records] + for found in identifiers: + self.assertEqual(1, len(found)) + self.assertEqual(1, len(set(found[0] for found in identifiers))) + self.assertRegex(identifiers[0][0], UUID4) + + def test_the_identifier_is_written_first_in_the_record(self): + # The snippet at lines 213 and 214 opens each Device with its + # ProcessId. + usage, clock, pipeline = element_with_clock() + process(pipeline, "first") + + record = sent_on_close(usage)[0] + + self.assertTrue(record.startswith(""), record) + + def test_a_fresh_element_has_a_new_identifier(self): + # Lines 128 to 131: a process that starts again creates a new + # identifier, and "A process that builds more than one Usage + # Sharing Element ... creates an identifier for each". + identifiers = [] + for _ in range(2): + usage, clock, pipeline = element_with_clock() + process(pipeline, "agent") + identifiers.extend(process_ids_in(sent_on_close(usage)[0])) + + self.assertEqual(2, len(identifiers)) + self.assertNotEqual(identifiers[0].lower(), identifiers[1].lower()) + + def test_a_new_process_has_a_new_identifier(self): + # Line 128: "The identifier is never stored, so a process that + # starts again creates a new one." + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join( + path for path in sys.path if path) + identifiers = [] + for _ in range(2): + output = subprocess.run( + [sys.executable, "-c", ONE_RECORD], env=environment, + capture_output=True, text=True, timeout=120, check=True) + identifiers.append(output.stdout.strip()) + + for identifier in identifiers: + self.assertRegex(identifier, UUID4) + self.assertNotEqual(identifiers[0].lower(), identifiers[1].lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py index 57722ecd..b25f7255 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_pipelines.py @@ -105,11 +105,36 @@ def test_each_record_names_the_elements_of_its_own_pipeline(self): agent = re.search(r'Name="user-agent">([^<]*)<', record).group(1) by_agent.setdefault(agent, []).append( re.findall(r"([^<]*)", record)) + # The flow elements are invariant values, so only the first record + # from each Pipeline names them (usage-sharing-element.md lines 150 + # to 153 at 07e1fced). The repeat of each request is not shared. self.assertEqual( {"first agent": [["FirstElement", "ShareUsage"]], "second agent": [["SecondElement", "ShareUsage"]]}, by_agent) + def test_only_the_first_record_from_each_pipeline_names_its_elements(self): + usage = ShareUsage(requested_package_size=1000, + endpoint="http://127.0.0.1:9") + # A record written before the lookup of the server's address ends + # does not count as carrying the flow elements, so wait for it. + usage._server_ip_found.wait(10) + first = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(FirstElement()).add(usage).build() + second = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(SecondElement()).add(usage).build() + + for index in range(2): + process(first, f"first agent {index}") + process(second, f"second agent {index}") + records = sent_on_close(usage) + + self.assertEqual( + [["FirstElement", "ShareUsage"], ["SecondElement", "ShareUsage"], + [], []], + [re.findall(r"([^<]*)", record) + for record in records]) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py index 2adef573..35d0058a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py @@ -237,20 +237,27 @@ def test_the_session_cookie_is_compared_but_never_shared(self): class PackageTests(unittest.TestCase): - def test_each_record_names_every_51degrees_package_in_use(self): - records = records_of(ShareUsage(**QUEUE_ONLY), [ + def test_the_first_record_names_every_51degrees_package_in_use(self): + # The packages are invariant values, so the record after the first, + # written within the hour, names none (usage-sharing-element.md + # lines 138 to 153 at 07e1fced). + usage = ShareUsage(**QUEUE_ONLY) + # A record written before the lookup of the server's address ends + # does not count as carrying them, so wait for it. + usage._server_ip_found.wait(10) + records = records_of(usage, [ {"header.user-agent": "first"}, {"header.user-agent": "second"}, ]) self.assertEqual(2, len(records)) - for record in records: - packages = packages_in(record) - self.assertEqual( - sorted(USAGE_SHARING_PACKAGES), - sorted(canonical(name) for name, _ in packages)) - for name, version in packages: - self.assertEqual(metadata.version(name), version, name) + packages = packages_in(records[0]) + self.assertEqual( + sorted(USAGE_SHARING_PACKAGES), + sorted(canonical(name) for name, _ in packages)) + for name, version in packages: + self.assertEqual(metadata.version(name), version, name) + self.assertEqual([], packages_in(records[1])) def test_packages_follow_the_flow_elements(self): record = records_of(ShareUsage(**QUEUE_ONLY), [ From 27b92666af6e0687dab53c155c7570883424cea4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 21:04:44 +0000 Subject: [PATCH 7/9] Fix: write the invariant values once an hour and a process identifier on every record usage-sharing-element.md at specifications 07e1fced, lines 150 to 153: "The invariant values MUST be written to the first record the element writes. They MUST be written again to the first record the element writes once an hour or more has passed since it last wrote them. They MUST NOT be written to any other record." Lines 115 and 116: "The element MUST create a process identifier when it is built, and MUST write it to every record as the ProcessId element." The element creates a version 4 UUID when it is built, as process_id, and writes it first in every record. Version, Product, the FlowElement and Package elements, Language, LanguageVersion, ServerIP and Platform go in the first record from each Pipeline and again in the first one written an hour or more after they were last written, measured by the element's clock attribute. A record written before the lookup of the server's address has ended carries an empty ServerIP, so it does not start the hour and the next record carries the values again. --- .../share_usage.py | 58 +++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 9b8e76ae..eac43aaa 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -30,6 +30,7 @@ import datetime import threading import time +import uuid from concurrent.futures import ThreadPoolExecutor from importlib import metadata from importlib.metadata import version @@ -88,6 +89,11 @@ # parts, as a timer cannot wait for any length of time on every platform. LONGEST_WAIT_SECONDS = 86400 +# The values that do not change while the element lives are written to its +# first record, and again to the first record written once this many +# seconds have passed since they were last written. +INVARIANT_VALUES_INTERVAL_SECONDS = 3600 + def devices_message(entries): """! @@ -280,6 +286,22 @@ def __init__( # is in, as each names its own flow elements. self._constant_xml_by_pipeline = {} + # Written to every record as its ProcessId. Created here and never + # stored, so an element built again, in this process or another, + # has a new one. + self.process_id = str(uuid.uuid4()) + + # The clock, in seconds, that the hour between writes of the values + # that do not change is measured by. Not a user setting, and here so + # that a test can control it. + self.clock = time.monotonic + + # When the values that do not change were last written to a record + # from each Pipeline, by the clock. A Pipeline missing here has had + # them written to none of its records. + self._invariants_written = {} + self._invariants_lock = threading.Lock() + # The address of this machine for the record's ServerIP, looked up # on its own thread so that processing never waits for a slow name # lookup. It is None until the lookup ends, and records made before @@ -409,10 +431,12 @@ def _send_remaining(self, deadline, done): def get_constant_xml(self, pipeline=None): """! The part of a record that is the same for every request of a - Pipeline. Given the Pipeline a request came from, it names that + Pipeline, being the values that do not change while the element + lives. Given the Pipeline a request came from, it names that Pipeline's flow elements, so an element in several Pipelines writes each one's own. With no Pipeline it names those of the one Pipeline - the element is in. + the element is in. A record carries it only when + _write_invariants says so. """ if pipeline is not None: xml = self._constant_xml_by_pipeline.get(pipeline) @@ -433,6 +457,27 @@ def get_constant_xml(self, pipeline=None): self.constant_xml = xml return self.constant_xml + def _write_invariants(self, pipeline): + """! + Whether the record being written for a request from the Pipeline + carries the values that do not change. They go in the first record + from each Pipeline, and again in the first one written once + INVARIANT_VALUES_INTERVAL_SECONDS have passed since they were last + written, whether or not that record is ever sent. A record written + before the lookup of the server's address has ended carries them + with an empty ServerIP, so it does not count as having written them + and the next record carries them again. + """ + with self._invariants_lock: + now = self.clock() + written = self._invariants_written.get(pipeline) + if written is not None and \ + now - written < INVARIANT_VALUES_INTERVAL_SECONDS: + return False + if self._server_ip is not None: + self._invariants_written[pipeline] = now + return True + def _make_constant_xml(self, flow_element_names, flow_elements=()): """! The constant part of a record for the flow elements of one @@ -685,8 +730,13 @@ def add_to_share_usage(self, data): xml += "" - # --- write invariant data - xml += self.get_constant_xml(data.pipeline) + # The process identifier, in every record, which ties a record that + # leaves out the values that do not change to those values. + xml += plain_element("ProcessId", self.process_id) + + # --- write invariant data, to the first record and once an hour + if self._write_invariants(data.pipeline): + xml += self.get_constant_xml(data.pipeline) # --- write variable data # The SessionID used to track a series of requests From b41d660ee8d2aeaceeb8151450a9fce928a6fe4c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 03:25:12 +0000 Subject: [PATCH 8/9] Test: write the invariant values again only in the first record once the server address is known The server address is looked up on its own thread, so the first record can be written before it is known. The usage sharing page, as specifications#189 changes it, says the element then writes all the invariant values again to the first record it writes once that value is known, and to no other record. The new test holds the lookup up, writes two records while it runs, lets it end, and then checks that only the first record and the first record after the lookup carry the values, and that the hour runs from the record that carried the address. --- .../test_share_usage_invariant_values.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py index daacae25..48143a48 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_invariant_values.py @@ -247,6 +247,54 @@ def slow(): self.assertIn("" + SERVER_IP + "", records[1]) self.assertEqual([], invariants_in(records[2])) + def test_the_values_go_again_only_in_the_first_record_once_known(self): + # usage-sharing-element.md lines 152 to 155 at 44d815f8: "When one + # of them is not yet known as the first record is written, for + # example a server address still being looked up, the element MUST + # write all of them again to the first record it writes once that + # value is known. They MUST NOT be written to any other record." + # So a second record written while the lookup is still running + # leaves them out, and the hour runs from the record that carried + # the address. + release = threading.Event() + self.addCleanup(release.set) + + def slow(): + release.wait(LOOKUP_WAIT_SECONDS) + return SERVER_IP + + with mock.patch.object(share_usage, "host_address", slow): + usage, clock, pipeline = element_with_clock( + wait_for_server_ip=False) + process(pipeline, "first, while the lookup runs") + clock.now += 1 + process(pipeline, "second, while the lookup runs") + release.set() + self.assertTrue(usage._server_ip_found.wait(LOOKUP_WAIT_SECONDS)) + clock.now += 1 + process(pipeline, "first once the address is known") + clock.now += 1 + process(pipeline, "next after that") + # An hour less a second after the record that carried the address, + # and more than an hour after the first record. + clock.now += HOUR - 2 + process(pipeline, "within the hour of the address") + clock.now += 1 + process(pipeline, "an hour after the address") + + records = sent_on_close(usage) + + self.assertEqual(6, len(records)) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[0])) + self.assertIn("", records[0]) + self.assertEqual([], invariants_in(records[1])) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[2])) + self.assertIn("" + SERVER_IP + "", records[2]) + self.assertEqual([], invariants_in(records[3])) + self.assertEqual([], invariants_in(records[4])) + self.assertEqual(list(INVARIANT_ELEMENTS), invariants_in(records[5])) + self.assertIn("" + SERVER_IP + "", records[5]) + # Writes one record from a new element and prints its ProcessId, so that a # test can compare the identifiers of two processes. From 19e18bc992784f2701916475c35c1f936df2a605 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 03:28:32 +0000 Subject: [PATCH 9/9] Fix: write the invariant values again only in the first record once the server address is known A record written while the lookup of the server address was running carried the invariant values with an empty ServerIP and was not counted, so every record until the lookup ended carried them. The usage sharing page, as specifications#189 changes it, allows only the first record, the first record once the address is known, and the first record once an hour has passed since they were last written. The element now notes the time of every record that carries them, and whether that record went without the address. The next record carries them again only when the lookup has since found the address, or the hour is up. The decision and the values are made under one lock from one reading of the address, so a record never carries an address the decision was not made on, and a part made while the lookup runs is never kept. --- .../share_usage.py | 97 +++++++++++++------ 1 file changed, 67 insertions(+), 30 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index eac43aaa..d3140cb1 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -300,6 +300,10 @@ def __init__( # from each Pipeline, by the clock. A Pipeline missing here has had # them written to none of its records. self._invariants_written = {} + # The Pipelines whose last record to carry them was written before + # the lookup of the server's address ended, so with an empty + # ServerIP. + self._invariants_without_server_ip = set() self._invariants_lock = threading.Lock() # The address of this machine for the record's ServerIP, looked up @@ -438,51 +442,81 @@ def get_constant_xml(self, pipeline=None): the element is in. A record carries it only when _write_invariants says so. """ + return self._constant_xml(pipeline, self._server_ip) + + def _constant_xml(self, pipeline, server_ip): + """! + get_constant_xml with the server's address read once, as server_ip, + which is None while the lookup is running. A part made then carries + an empty ServerIP and is not kept, so that ServerIP carries the + address once the lookup has ended. + """ if pipeline is not None: xml = self._constant_xml_by_pipeline.get(pipeline) if xml is None: xml = self._make_constant_xml( self.get_flow_elements(pipeline), - list(pipeline.flow_elements)) - if self._server_ip is not None: + list(pipeline.flow_elements), server_ip) + if server_ip is not None: self._constant_xml_by_pipeline[pipeline] = xml return xml if self.constant_xml is None: xml = self._make_constant_xml( - self.get_flow_elements(), self._pipeline_flow_elements()) - # Kept once the lookup of the address has ended, so that - # ServerIP carries the address from then on. - if self._server_ip is None: + self.get_flow_elements(), self._pipeline_flow_elements(), + server_ip) + if server_ip is None: return xml self.constant_xml = xml return self.constant_xml - def _write_invariants(self, pipeline): + def _invariant_xml(self, pipeline): + """! + The values that do not change, for the record being written for a + request from the Pipeline, or None when that record leaves them + out. The decision and the values are made under one lock from one + reading of the server's address, so that the address a record + carries is the one the decision was made on. + """ + with self._invariants_lock: + server_ip = self._server_ip + if not self._write_invariants(pipeline, server_ip): + return None + return self._constant_xml(pipeline, server_ip) + + def _write_invariants(self, pipeline, server_ip): """! Whether the record being written for a request from the Pipeline - carries the values that do not change. They go in the first record - from each Pipeline, and again in the first one written once + carries the values that do not change, given the server's address, + None while the lookup is running. They go in the first record from + each Pipeline, and again in the first one written once INVARIANT_VALUES_INTERVAL_SECONDS have passed since they were last - written, whether or not that record is ever sent. A record written - before the lookup of the server's address has ended carries them - with an empty ServerIP, so it does not count as having written them - and the next record carries them again. + written, whether or not that record is ever sent. When the last + record to carry them was written before the lookup ended, so with + an empty ServerIP, they go again in the first record written once + the lookup has found the address. No other record carries them. + Called with _invariants_lock held. """ - with self._invariants_lock: - now = self.clock() - written = self._invariants_written.get(pipeline) - if written is not None and \ - now - written < INVARIANT_VALUES_INTERVAL_SECONDS: - return False - if self._server_ip is not None: - self._invariants_written[pipeline] = now - return True - - def _make_constant_xml(self, flow_element_names, flow_elements=()): + now = self.clock() + written = self._invariants_written.get(pipeline) + found_since = bool(server_ip) and \ + pipeline in self._invariants_without_server_ip + if written is not None and not found_since and \ + now - written < INVARIANT_VALUES_INTERVAL_SECONDS: + return False + self._invariants_written[pipeline] = now + if server_ip is None: + self._invariants_without_server_ip.add(pipeline) + else: + self._invariants_without_server_ip.discard(pipeline) + return True + + def _make_constant_xml(self, flow_element_names, flow_elements=(), + server_ip=None): """! The constant part of a record for the flow elements of one - Pipeline, given their names and the elements themselves, whose - packages the record names. + Pipeline, given their names, the elements themselves, whose + packages the record names, and the server's address, written empty + when it is None. """ coreVersion = version("fiftyone_pipeline_core") osVersion = f"{ReplacedString(platform.system()).result} {ReplacedString(platform.release()).result}" @@ -505,7 +539,7 @@ def _make_constant_xml(self, flow_element_names, flow_elements=()): # The software language version xml += f"{pyVersion}" # The address of this machine - xml += plain_element("ServerIP", self._server_ip or "") + xml += plain_element("ServerIP", server_ip or "") # The OS name and version xml += f"{osVersion}" return xml @@ -734,9 +768,12 @@ def add_to_share_usage(self, data): # leaves out the values that do not change to those values. xml += plain_element("ProcessId", self.process_id) - # --- write invariant data, to the first record and once an hour - if self._write_invariants(data.pipeline): - xml += self.get_constant_xml(data.pipeline) + # --- write invariant data, to the first record, once an hour, and + # to the first record once the server's address is found when the + # last record to carry them went without it + invariants = self._invariant_xml(data.pipeline) + if invariants is not None: + xml += invariants # --- write variable data # The SessionID used to track a series of requests