diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index ce7244e3..214a0f03 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -115,7 +115,8 @@ def __init__( failures_window_seconds = FAILURES_WINDOW_SECONDS_DEFAULT, initial_recovery_delay_seconds = INITIAL_RECOVERY_DELAY_SECONDS_DEFAULT, max_recovery_delay_seconds = MAX_RECOVERY_DELAY_SECONDS_DEFAULT, - recovery_multiplier = RECOVERY_MULTIPLIER_DEFAULT): + recovery_multiplier = RECOVERY_MULTIPLIER_DEFAULT, + share_all_query_string_parameters = True): """! Constructor for ShareUsage element @@ -134,19 +135,20 @@ def __init__( @type cookie: string @param cookie: The name of the cookie that holds the session id. It has no effect unless track_session is True. The cookie itself - is not shared unless share_all is True. + is never shared unless its name starts with '51D_', as no other + cookie is. @type track_session: bool @param track_session: If True, the cookie named by cookie is used to tell user sessions apart, so evidence already shared is shared again when it comes from a new session. False by default. It - needs cookie to be set, and has no effect when share_all is True, - because every cookie is then shared and used to find repeats. + needs cookie to be set, and works the same whether or not + share_all is True. @type query_whitelist: list or str - @param query_whitelist: By default query string and HTTP form - parameters are not shared unless prefixed with '51D_'. - If you need to share query string parameters, a list can be - specified here, or a string of names separated by commas, which is - how a configuration file gives a list. None shares every parameter. + @param query_whitelist: The query string and HTTP form parameters + to share, as well as those prefixed with '51D_', when + share_all_query_string_parameters is False. A string of names + separated by commas, which is how a configuration file gives a + list, is read as those names. None shares every parameter. @type header_blacklist: list or str @param header_blacklist: By default, all HTTP headers except Cookie are shared. @@ -157,8 +159,14 @@ def __init__( be shared. 1 = 100%, 0.5 = 50%, etc.. @type share_all: bool @param share_all: If True, all evidence is shared apart from the - email address, the salt used for hashed email identifiers, and the - request's query string and path. False by default. + email address, the salt used for hashed email identifiers, the + request's query string and path, and every cookie whose name does + not start with '51D_'. False by default. + @type share_all_query_string_parameters: bool + @param share_all_query_string_parameters: If True, every query + string and HTTP form parameter is shared. If False, only those + prefixed with '51D_' and those in query_whitelist are. True by + default. @type max_queue_size: int @param max_queue_size: The most entries held while waiting to be sent. When the queue is full, new entries are dropped. The default @@ -201,6 +209,8 @@ def __init__( self.endpoint = endpoint self.request_timeout = request_timeout self.share_all = share_all + self.share_all_query_string_parameters = \ + share_all_query_string_parameters self.track_session = track_session # The session id and sequence number are kept in evidence, as the @@ -212,15 +222,15 @@ def __init__( # Vary header never depend on it. It shares what it finds instead, # and these filters decide which of that is shared and which tells # a repeat apart. When sessions are tracked the tracker reads the - # session cookie, which the data sent leaves out unless all evidence - # is shared. + # session cookie, which the data sent leaves out. self.evidence_key_filter = BasicListEvidenceKeyFilter([]) self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter( - cookie, included, header_blacklist, False, share_all) + cookie, included, header_blacklist, False, share_all, + share_all_query_string_parameters) self.tracker_evidence_key_filter = \ ShareUsageTrackerEvidenceKeyFilter( cookie, included, header_blacklist, track_session, - share_all) + share_all, share_all_query_string_parameters) # Add the share usage tracker which detects when # to send up sharing data diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index 93da177f..c813f627 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -52,10 +52,12 @@ def _split_names(names): class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees - ShareUsage service is sent. Headers are shared unless they are blocked, - cookies and session values only when their names start with 51D_, and - query string and form parameters only when their names start with 51D_ or - are on the list to share. Any other evidence is shared. + ShareUsage service is sent. Cookies are shared only when their names + start with 51D_, whatever the other settings. Headers are shared unless + they are blocked, session values only when their names start with 51D_, + and query string and form parameters when all of them are shared or + when their names start with 51D_ or are on the list to share. Any other + evidence is shared. """ def __init__( @@ -64,7 +66,8 @@ def __init__( query_whitelist = (), header_blacklist = (), include_session = False, - share_all = False): + share_all = False, + share_all_query_string_parameters = False): """! Constructor for ShareUsageEvidenceKeyFilter @type cookie: string @@ -80,7 +83,10 @@ def __init__( @param include_session: whether the session cookie is included @type share_all: bool @param share_all: whether all evidence is shared, apart from the - evidence that is never shared + evidence that is never shared and cookies not starting 51D_ + @type share_all_query_string_parameters: bool + @param share_all_query_string_parameters: whether every query + string and form parameter is shared, whatever query_whitelist holds """ self.cookie = cookie.lower() if cookie else None @@ -97,6 +103,9 @@ def __init__( self.share_all = share_all + self.share_all_query_string_parameters = \ + share_all_query_string_parameters + def filter_evidence_key(self, key): """! @@ -115,33 +124,34 @@ def filter_evidence_key(self, key): for never_shared in NEVER_SHARED_KEYS): return False - if self.share_all: - return True - # The prefix is everything before the first dot and the field # everything after it, so a field can itself contain dots. prefix, separator, field = key.partition(".") + # A cookie a site sets for itself is never shared, even when all + # evidence is. The tracker alone keeps the session cookie. + if separator and prefix == "cookie": + return field.startswith(FIFTYONE_PREFIX) or \ + (self.include_session and field == self.cookie) + + if self.share_all: + return True + if not separator: return True elif prefix == "header": # Add the header to the list if the header name does not # appear in the list of blocked headers return field not in self.header_blacklist - elif prefix == "cookie": - # Only add cookies that start with the 51Degrees cookie - # prefix - return field.startswith(FIFTYONE_PREFIX) or \ - (self.include_session and field == self.cookie) elif prefix == "session": # Only session values that start with the 51Degrees # cookie prefix return field.startswith(FIFTYONE_PREFIX) elif prefix == "query": - # If no query string parameter list was given then share all - # of them. Otherwise, only include query string parameters - # that start with 51d_ or that are in the list. - return self.query_whitelist is None or \ + # Share every parameter when asked to or when no list was + # given. Otherwise only those starting 51d_ or in the list. + return self.share_all_query_string_parameters or \ + self.query_whitelist is None or \ field.startswith(FIFTYONE_PREFIX) or \ field in self.query_whitelist else: diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py index 503abff0..66761816 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py @@ -138,7 +138,11 @@ def test_a_revalidation_has_the_same_accepted_evidence(self): self.assertNotIn("header.if-none-match", accepted[1]) def test_the_web_request_is_shared_by_the_same_rules(self): - usage = usage_element(header_blacklist=["x-blocked"]) + # Sharing every query parameter is turned off, so that the query + # string of the request is held to the list as Evidence is. + usage = usage_element( + header_blacklist=["x-blocked"], + share_all_query_string_parameters=False) flowdata = pipeline_with(usage).create_flowdata() flowdata.evidence.add_from_dict(request_evidence({ "User-Agent": CHROME, @@ -174,8 +178,7 @@ def test_the_method_is_shared_and_the_path_and_cookie_header_are_not(self): shared = "".join(usage.share_data) self.assertIn("request-method", shared) - self.assertIn('cookievalue', - shared) + self.assertNotIn("cookievalue", shared) self.assertNotIn("/page", shared) self.assertNotIn('
', shared) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py index 1abc7721..b31e3ea1 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -125,13 +125,16 @@ def test_session_cookie(self): def test_share_all(self): """! Check that sharing everything shares the evidence the other rules - leave out, apart from the evidence that is never shared. + leave out, apart from the evidence that is never shared and the + cookies that do not start 51D_. """ keys = ShareUsageEvidenceKeyFilter(share_all=True) self.assertFalse(keys.filter_evidence_key("header.cookie")) - self.assertTrue(keys.filter_evidence_key("cookie.sessionid")) + self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) + self.assertTrue( + keys.filter_evidence_key("cookie.51D_screenpixelswidth")) self.assertTrue(keys.filter_evidence_key("query.password")) self.assertTrue(keys.filter_evidence_key("query.id.usage")) self.assertTrue(keys.filter_evidence_key("server.request-method")) @@ -142,6 +145,35 @@ def test_share_all(self): self.assertFalse(keys.filter_evidence_key("server.request-query")) self.assertFalse(keys.filter_evidence_key("server.request-path")) + def test_every_query_parameter_when_sharing_them_all(self): + """! + Check that sharing every query string parameter shares one not on + the list, and that a filter made without it keeps to the list. + """ + + every = ShareUsageEvidenceKeyFilter( + query_whitelist=["campaign"], + share_all_query_string_parameters=True) + listed = ShareUsageEvidenceKeyFilter(query_whitelist=["campaign"]) + + self.assertTrue(every.filter_evidence_key("query.password")) + self.assertFalse(listed.filter_evidence_key("query.password")) + self.assertTrue(listed.filter_evidence_key("query.campaign")) + + def test_the_session_cookie_is_kept_for_the_tracker_alone(self): + """! + Check that, when all evidence is shared, the session cookie is kept + by the tracker's filter and by no other. + """ + + kept = ShareUsageEvidenceKeyFilter( + cookie="SessionId", include_session=True, share_all=True) + sent = ShareUsageEvidenceKeyFilter(cookie="SessionId", share_all=True) + + self.assertTrue(kept.filter_evidence_key("cookie.sessionid")) + self.assertFalse(sent.filter_evidence_key("cookie.sessionid")) + self.assertFalse(kept.filter_evidence_key("cookie._ga")) + def test_never_shared_in_every_mode(self): """! Check that the email address, the salt for hashed email diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py index a89c1bed..9209ae53 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_lists.py @@ -44,6 +44,10 @@ "endpoint": "http://127.0.0.1:9", } +# Shares only the query string parameters listed and those starting 51D_, +# where every one is shared by default. +LISTED_ONLY = {"share_all_query_string_parameters": False} + # A configuration file that gives both lists as comma-separated strings. CONFIGURATION = """ { @@ -55,6 +59,7 @@ "elementParameters": { "requested_package_size": 1000, "endpoint": "http://127.0.0.1:9", + "share_all_query_string_parameters": false, "query_whitelist": "first,second", "header_blacklist": "x-one, x-two" } @@ -106,7 +111,8 @@ def assert_not_shared(self, shared, values): def test_query_parameters_to_share_from_a_string(self): shared = shared_alone_by( - ShareUsage(query_whitelist="first,second", **QUEUE_ONLY)) + ShareUsage(query_whitelist="first,second", **LISTED_ONLY, + **QUEUE_ONLY)) self.assert_shared(shared, ("first value", "second value")) self.assert_not_shared(shared, ("third value", "letter value")) @@ -140,7 +146,8 @@ def test_names_are_trimmed_and_empty_entries_left_out(self): def test_an_empty_string_is_an_empty_list(self): usage = ShareUsage( - query_whitelist="", header_blacklist="", **QUEUE_ONLY) + query_whitelist="", header_blacklist="", **LISTED_ONLY, + **QUEUE_ONLY) self.assertEqual([], usage.query_whitelist) self.assertEqual([], usage.header_blacklist) @@ -152,7 +159,8 @@ def test_a_list_is_used_as_it_is(self): names = ["first", "second"] headers = ["x-one", "x-two"] usage = ShareUsage( - query_whitelist=names, header_blacklist=headers, **QUEUE_ONLY) + query_whitelist=names, header_blacklist=headers, + **LISTED_ONLY, **QUEUE_ONLY) self.assertIs(names, usage.query_whitelist) self.assertIs(headers, usage.header_blacklist) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py new file mode 100644 index 00000000..5df7e9be --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_rules.py @@ -0,0 +1,207 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +What usage sharing shares by default and when all Evidence is shared. +Every query string and form parameter is shared unless that is turned off, +no cookie but a 51D_ one is ever shared, and the session cookie is still +compared when sessions are tracked. +""" + +import gzip +import json +import re +import unittest +from unittest import mock + +import requests +from flask import Flask, request + +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.web import webevidence +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + +# The package size is never reached, so the element keeps what it would +# share and sends nothing until it is closed. +QUEUE_ONLY = { + "requested_package_size": 1000, + "endpoint": "http://127.0.0.1:9", +} + +# Turns off sharing every query string parameter, from a configuration +# file. +CONFIGURATION = """ +{ + "PipelineOptions": { + "Elements": [ + { + "elementName": "ShareUsage", + "elementPath": "fiftyone_pipeline_engines_fiftyone.share_usage", + "elementParameters": { + "requested_package_size": 1000, + "endpoint": "http://127.0.0.1:9", + "share_all_query_string_parameters": false + } + } + ] + } +} +""" + + +def sent_on_close(usage): + """Closes the element, which sends what it has queued, and returns each + record it posted. The post is answered without leaving the process.""" + sent = [] + + def post(url, headers=None, data=None, **kwargs): + sent.append(gzip.decompress(data).decode("utf8")) + return mock.Mock(status_code=200, text="") + + with mock.patch.object(requests, "post", post): + usage.close() + return [record for message in sent + for record in re.findall(r".*?", message)] + + +def records_of(usage, requests_evidence, *elements): + """Processes each request's Evidence through a Pipeline holding the + elements and the usage sharing element, and returns the records sent.""" + builder = PipelineBuilder(NO_DEFAULT_ELEMENTS) + for element in elements: + builder.add(element) + pipeline = builder.add(usage).build() + for evidence in requests_evidence: + flowdata = pipeline.create_flowdata() + for key, value in evidence.items(): + flowdata.evidence.add(key, value) + flowdata.process() + return sent_on_close(usage) + + +class QueryAndFormValuesTests(unittest.TestCase): + + def test_every_query_value_is_shared_by_default(self): + records = records_of(ShareUsage(**QUEUE_ONLY), [{ + "header.user-agent": "agent value", + "query.campaign": "campaign value", + "query.password": "password value", + }]) + + self.assertEqual(1, len(records)) + self.assertIn('campaign value', + records[0]) + self.assertIn('password value', + records[0]) + + def test_every_form_value_of_the_web_request_is_shared_by_default(self): + app = Flask(__name__) + usage = ShareUsage(**QUEUE_ONLY) + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS).add(usage).build() + with app.test_request_context( + "/", method="POST", data={"field": "form value"}): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.process() + records = sent_on_close(usage) + + self.assertEqual(1, len(records)) + self.assertIn('form value', records[0]) + + def test_turned_off_only_51d_and_listed_values_are_shared(self): + records = records_of(ShareUsage( + share_all_query_string_parameters=False, + query_whitelist=["campaign"], **QUEUE_ONLY), [{ + "header.user-agent": "agent value", + "query.campaign": "campaign value", + "query.51d_gethighentropyvalues": "hev", + "query.password": "password value", + }]) + + self.assertIn("campaign value", records[0]) + self.assertIn("hev", records[0]) + self.assertNotIn("password value", records[0]) + + def test_turned_off_from_a_configuration_file(self): + pipeline = PipelineBuilder().build_from_configuration( + json.loads(CONFIGURATION)) + usage = pipeline.get_element("shareusage") + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "agent value") + flowdata.evidence.add("query.password", "password value") + flowdata.process() + records = sent_on_close(usage) + + self.assertFalse(usage.share_all_query_string_parameters) + self.assertIn("agent value", records[0]) + self.assertNotIn("password value", records[0]) + + +class CookieTests(unittest.TestCase): + + def test_only_51d_cookies_are_shared_when_all_evidence_is_shared(self): + records = records_of(ShareUsage(share_all=True, **QUEUE_ONLY), [{ + "header.user-agent": "agent value", + "cookie.51d_screenpixelswidth": "1080", + "cookie.sessionid": "session value", + "cookie._ga": "analytics value", + "session.basket": "basket value", + }]) + + self.assertIn('1080', + records[0]) + self.assertNotIn("session value", records[0]) + self.assertNotIn("analytics value", records[0]) + # Sharing all Evidence still shares what it shared before, other + # than cookies. + self.assertIn("basket value", records[0]) + + def test_the_session_cookie_is_compared_but_never_shared(self): + for share_all in (False, True): + with self.subTest(share_all=share_all): + records = records_of(ShareUsage( + share_all=share_all, track_session=True, + cookie="sessionid", **QUEUE_ONLY), [ + {"header.user-agent": "agent value", + "cookie.sessionid": "first session"}, + {"header.user-agent": "agent value", + "cookie.sessionid": "first session"}, + {"header.user-agent": "agent value", + "cookie.sessionid": "second session"}, + ]) + + # The repeat in the first session is not shared, and the + # same Evidence from a new session is. + self.assertEqual(2, len(records)) + for record in records: + self.assertNotIn("first session", record) + self.assertNotIn("second session", record) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index f3b20313..adff0aa0 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -919,9 +919,9 @@ def test_escapes_xml_characters(self): def test_default_shares_only_accepted_evidence(self): """! - Check that with no options the element shares headers and the - 51Degrees cookies and parameters, and leaves out other cookies, the - Cookie header, and other query string and form parameters. + Check that with no options the element shares headers, the + 51Degrees cookies and every query string and form parameter, and + leaves out other cookies, the Cookie header and the email address. """ usage = ShareUsage( @@ -948,7 +948,7 @@ def test_default_shares_only_accepted_evidence(self): self.assertIn('hev', app.received[0]) self.assertNotIn("cookie value", app.received[0]) self.assertNotIn("header value", app.received[0]) - self.assertNotIn("password value", app.received[0]) + self.assertIn('password value', app.received[0]) self.assertNotIn("email value", app.received[0]) def test_never_shared_evidence(self): @@ -984,7 +984,8 @@ def test_never_shared_evidence(self): def test_share_all(self): """! Check that sharing all evidence shares what the other rules leave - out, but not the evidence that is never shared. + out, but not the evidence that is never shared or a cookie that + does not start 51D_. """ usage = ShareUsage( @@ -997,13 +998,15 @@ def test_share_all(self): .build() flowdata = pipeline.create_flowdata() flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("cookie.51d_screenpixelswidth", "1080") flowdata.evidence.add("query.campaign", "campaign value") flowdata.evidence.add("query.id.email", "email value") flowdata.process() self.waitForUsageThreads() self.assertEqual(len(app.received), 1) - self.assertIn('cookie value', app.received[0]) + self.assertNotIn("cookie value", app.received[0]) + self.assertIn('1080', app.received[0]) self.assertIn('campaign value', app.received[0]) self.assertNotIn("email value", app.received[0]) @@ -1016,6 +1019,7 @@ def test_parameter_names_with_dots(self): usage = ShareUsage( requested_package_size=1, endpoint=testEndpoint, + share_all_query_string_parameters=False, query_whitelist=["valid.email"]) pipeline = (PipelineBuilder())\ @@ -1033,8 +1037,9 @@ def test_parameter_names_with_dots(self): def test_empty_query_whitelist(self): """! - Check that an empty list of parameters to share shares only those - starting 51D_, and that None shares every parameter. + Check that, with sharing every parameter turned off, an empty + list of parameters to share shares only those starting 51D_, and + that None shares every parameter. """ for query_whitelist, shared in (([], False), (None, True)): @@ -1042,6 +1047,7 @@ def test_empty_query_whitelist(self): usage = ShareUsage( requested_package_size=1, endpoint=testEndpoint, + share_all_query_string_parameters=False, query_whitelist=query_whitelist, header_blacklist=["x-secret"])