From bae7bfb2816d2c9407e4f5a74fa66088445aaebf Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Thu, 12 Nov 2026 11:50:43 +0100 Subject: [PATCH 1/8] fix: treat failed feature flag records as unevaluated --- .sampo/changesets/fix-failed-flag-records.md | 5 + posthog/client.py | 38 +++- posthog/feature_flag_evaluations.py | 18 +- posthog/test/test_failed_flag_records.py | 193 +++++++++++++++++++ posthog/test/test_flags_v4_response.py | 11 +- posthog/types.py | 4 +- 6 files changed, 254 insertions(+), 15 deletions(-) create mode 100644 .sampo/changesets/fix-failed-flag-records.md create mode 100644 posthog/test/test_failed_flag_records.py diff --git a/.sampo/changesets/fix-failed-flag-records.md b/.sampo/changesets/fix-failed-flag-records.md new file mode 100644 index 000000000..867d7a368 --- /dev/null +++ b/.sampo/changesets/fix-failed-flag-records.md @@ -0,0 +1,5 @@ +--- +posthog: patch +--- + +Treat failed feature flag records as unevaluated, report evaluation failure diagnostics, and preserve successful local values and payloads when remote fallback fails. diff --git a/posthog/client.py b/posthog/client.py index 35380d979..de560dfef 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -4646,6 +4646,8 @@ def _get_feature_flag_result( errors.append(FeatureFlagError.ERRORS_WHILE_COMPUTING) if flag_details is None: errors.append(FeatureFlagError.FLAG_MISSING) + elif flag_details._failed: + errors.append("evaluation_failed") if errors: feature_flag_error = ",".join(errors) @@ -4677,10 +4679,11 @@ def _get_feature_flag_result( self.log.debug( f"Successfully computed flag remotely: #{key} -> #{flag_result}" ) - # A payload lookup for a caller's match_value reports that value, not - # the record's. A failure above falls back to a stale cached value. - value_from_details = ( - flag_details is not None and override_match_value is None + # A successful payload override reports the caller's value. A + # failed record still supplies diagnostics, never the override. + value_from_details = flag_details is not None and ( + override_match_value is None + or (flag_details._failed and flag_details._malformed_field is None) ) except QuotaLimitError as e: self.log.warning(f"[FEATURE FLAGS] Quota limit exceeded: {e}") @@ -5321,7 +5324,32 @@ def get_all_flags_and_payloads( flag_keys_to_evaluate=flag_keys_to_evaluate, device_id=device_id, ) - response = to_flags_and_payloads(decide_response) + remote_response = to_flags_and_payloads(decide_response) + # Only explicit failed records preserve same-context local pairs; + # successful remote records retain the existing precedence. + failed_keys = { + key + for key, detail in decide_response.get("flags", {}).items() + if detail._failed and detail._malformed_field is None + } + response = { + "featureFlags": { + **{ + k: v + for k, v in (response["featureFlags"] or {}).items() + if k in failed_keys + }, + **(remote_response["featureFlags"] or {}), + }, + "featureFlagPayloads": { + **{ + k: v + for k, v in (response["featureFlagPayloads"] or {}).items() + if k in failed_keys + }, + **(remote_response["featureFlagPayloads"] or {}), + }, + } except Exception as e: self.log.exception( f"[FEATURE FLAGS] Unable to get feature flags and payloads: {e}" diff --git a/posthog/feature_flag_evaluations.py b/posthog/feature_flag_evaluations.py index 8ddd10f49..d938b43aa 100644 --- a/posthog/feature_flag_evaluations.py +++ b/posthog/feature_flag_evaluations.py @@ -152,7 +152,14 @@ def __init__( """Internal — instances are created by the SDK via ``Client.evaluate_flags()``.""" self._host = host self._distinct_id = distinct_id - self._flags = flags + self._failed_flags = { + key: flag + for key, flag in flags.items() + if isinstance(flag, _RemoteFlagRecord) + and flag.details._failed + and flag.details._malformed_field is None + } + self._flags = {k: v for k, v in flags.items() if k not in self._failed_flags} self._groups: Dict[str, Union[str, int]] = dict(groups or {}) self._disable_geoip = disable_geoip self._request_id = request_id @@ -337,8 +344,9 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: if not self._distinct_id: return - flag = self._flags.get(key) - if flag is None: + failed = key in self._failed_flags + flag = self._flags.get(key) or self._failed_flags.get(key) + if flag is None or failed: response: Optional[FlagValue] = None elif not flag.enabled: response = False @@ -376,7 +384,9 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: errors.append("errors_while_computing_flags") if self._quota_limited: errors.append("quota_limited") - if flag is None: + if failed: + errors.append("evaluation_failed") + elif flag is None: errors.append( "local_evaluation_inconclusive" if key in self._unresolved_flags diff --git a/posthog/test/test_failed_flag_records.py b/posthog/test/test_failed_flag_records.py new file mode 100644 index 000000000..09a9fe456 --- /dev/null +++ b/posthog/test/test_failed_flag_records.py @@ -0,0 +1,193 @@ +"""Spec 211: server failed records supply diagnostics, not evaluated values.""" + +import copy +from unittest import mock + +import pytest + +from posthog import AsyncPosthog +from posthog.client import Client +from posthog.types import normalize_flags_response + + +@pytest.fixture +def client(): + client = Client("project-key", send=False) + yield client + client.shutdown() + + +def response(shape, error_bit, failed_value="remote"): + records = {} + for key, value, failed in [("failed", failed_value, True), ("off", False, False)]: + record = { + "key": key, + "failed": failed, + "reason": {"code": "dependency_error" if failed else "flag_disabled"}, + "metadata": {"id": 1, "version": 2, "payload": '{"copy":"remote"}'}, + } + if shape == "v4": + record["value"] = value + record["metadata"]["config_version"] = 2 + else: + record.update( + enabled=bool(value), variant=value if isinstance(value, str) else None + ) + records[key] = record + result = { + "flags": records, + "featureFlags": {"failed": "mirror"}, + "featureFlagPayloads": {"failed": '"mirror"'}, + } + if error_bit is not None: + result["errorsWhileComputingFlags"] = error_bit + return result + + +@pytest.mark.parametrize("shape", ["v2", "v4"]) +@pytest.mark.parametrize("error_bit", [None, False, True]) +@pytest.mark.parametrize("failed_value", [False, "remote"]) +@pytest.mark.parametrize("local_inconclusive", [False, True]) +def test_failed_direct_results_and_events( + client, shape, error_bit, failed_value, local_inconclusive +): + if local_inconclusive: + client.feature_flags = [ + { + "key": "failed", + "active": True, + "ensure_experience_continuity": True, + "filters": {"groups": []}, + } + ] + with ( + mock.patch( + "posthog.client.flags", + side_effect=lambda *args, **kwargs: copy.deepcopy( + response(shape, error_bit, failed_value) + ), + ), + mock.patch.object(client, "capture", return_value=(True, {})) as capture, + ): + assert ( + client.get_feature_flag_result( + "failed", "user", send_feature_flag_events=False + ) + is None + ) + capture.assert_not_called() + assert client.get_feature_flag_result("failed", "user") is None + assert client.get_feature_flag("failed", "user") is None + assert client.feature_enabled("failed", "user") is None + assert ( + client.get_feature_flag_payload( + "failed", "user", match_value=True, send_feature_flag_events=True + ) + is None + ) + assert capture.call_count == 1 + props = capture.call_args.kwargs["properties"] + assert props["$feature_flag_response"] is None + assert "evaluation_failed" in props["$feature_flag_error"].split(",") + assert "flag_missing" not in props["$feature_flag_error"] + assert ("errors_while_computing_flags" in props["$feature_flag_error"]) is bool( + error_bit + ) + if shape == "v4": + assert props["$feature_flag_config_version"] == 2 + assert props["$feature_flag_reason_code"] == "dependency_error" + assert "$feature_flag_payload" not in props + assert client.get_feature_flag_result("off", "user").enabled is False + assert "evaluation_failed" not in capture.call_args.kwargs["properties"].get( + "$feature_flag_error", "" + ) + combined = client.get_all_flags_and_payloads("user") + assert combined["featureFlags"] == {"off": False} + assert "failed" not in combined["featureFlagPayloads"] + + +@pytest.mark.parametrize("shape", ["v2", "v4"]) +@pytest.mark.parametrize("error_bit", [None, False, True]) +def test_failed_snapshot_defaults_diagnostics_and_enrichment(client, shape, error_bit): + with ( + mock.patch("posthog.client.flags", return_value=response(shape, error_bit)), + mock.patch.object(client, "capture", return_value=(True, {})) as capture, + ): + snapshot = client.evaluate_flags("user") + assert ( + snapshot.is_enabled("failed", True, send_feature_flag_events=False) is True + ) + capture.assert_not_called() + assert snapshot.get_flag("failed") is None + assert snapshot.get_flag_payload("failed") is None + props = capture.call_args.kwargs["properties"] + assert props["$feature_flag_response"] is None + assert "evaluation_failed" in props["$feature_flag_error"] + assert "flag_missing" not in props["$feature_flag_error"] + assert ("errors_while_computing_flags" in props["$feature_flag_error"]) is bool( + error_bit + ) + assert snapshot.is_enabled("off", True) is False + assert snapshot.keys == ["off"] + assert snapshot._get_event_properties() == {"$feature/off": False} + + +@pytest.mark.parametrize("shape", ["v2", "v4"]) +@pytest.mark.parametrize("error_bit", [None, False, True]) +def test_failed_fallback_preserves_only_successful_local_pairs( + client, shape, error_bit +): + local = { + "featureFlags": {"on": True, "off": False, "variant": "blue", "success": True}, + "featureFlagPayloads": { + k: '{"copy":"local"}' for k in ["on", "off", "variant", "success"] + }, + } + remote = response(shape, error_bit) + for key in ["on", "off", "variant"]: + remote["flags"][key] = copy.deepcopy(remote["flags"]["failed"]) + remote["flags"][key]["key"] = key + remote["flags"]["success"] = { + "key": "success", + "enabled": True, + "variant": "remote", + "metadata": {"payload": '{"copy":"success"}'}, + } + with ( + mock.patch.object( + client, + "_get_all_flags_and_payloads_locally", + return_value=(local, True, {}), + ), + mock.patch.object( + client, "_get_flags_decision", return_value=normalize_flags_response(remote) + ), + ): + result = client.get_all_flags_and_payloads("user") + assert result["featureFlags"] == { + "on": True, + "off": False, + "variant": "blue", + "success": "remote", + } + assert result["featureFlagPayloads"] == { + k: '{"copy":"success"}' if k == "success" else '{"copy":"local"}' + for k in ["on", "off", "variant", "success"] + } + + +async def test_failed_async_snapshot_honors_default(): + client = AsyncPosthog("project-key", send=False) + try: + with mock.patch( + "posthog.async_client._async_flags", + new=mock.AsyncMock(return_value=response("v4", False)), + ): + snapshot = await client.evaluate_flags("user") + assert ( + snapshot.is_enabled("failed", True, send_feature_flag_events=False) is True + ) + assert snapshot.get_flag("failed", send_feature_flag_events=False) is None + assert snapshot.keys == ["off"] + finally: + await client.shutdown() diff --git a/posthog/test/test_flags_v4_response.py b/posthog/test/test_flags_v4_response.py index 352c793aa..c730f54f5 100644 --- a/posthog/test/test_flags_v4_response.py +++ b/posthog/test/test_flags_v4_response.py @@ -211,8 +211,8 @@ def test_a_failed_record_never_renders_its_value(client): response["flags"]["garden-layout"]["value"] = "compact" snapshot = _evaluate(client, response) - assert snapshot.get_flag("garden-layout") is False - assert snapshot.is_enabled("garden-layout", True) is False + assert snapshot.get_flag("garden-layout") is None + assert snapshot.is_enabled("garden-layout", True) is True @pytest.mark.parametrize( @@ -764,7 +764,7 @@ def _failed_with_payload(shape: str) -> Dict[str, Any]: @pytest.mark.parametrize("shape", ["v4", "older"]) def test_a_failed_record_has_no_payload_on_any_read(shape, client): - # A failed flag reads as disabled, so a payload on it would surprise the caller. + # A failed flag has no evaluated result or payload. response = _failed_with_payload(shape) [flag] = normalize_flags_response(copy.deepcopy(response))["flags"].values() assert flag._failed is True @@ -772,7 +772,7 @@ def test_a_failed_record_has_no_payload_on_any_read(shape, client): with mock.patch("posthog.client.flags", return_value=copy.deepcopy(response)): result = client.get_feature_flag_result("garden-layout", DISTINCT_ID) - assert result is not None and result.payload is None + assert result is None assert client.get_feature_flag_payload("garden-layout", DISTINCT_ID) is None assert ( client.get_feature_flag_payload( @@ -951,7 +951,8 @@ def test_v1_flags_render_the_same_legacy_values_from_both_shapes(case): "flags": v4_flags, } expected = { - "featureFlags": v1_maps["featureFlags"], + # Flat legacy maps retain their interpretation, but detailed failures do not. + "featureFlags": {} if outcome["failed"] else v1_maps["featureFlags"], "featureFlagPayloads": { key: json.loads(value) for key, value in v1_maps["featureFlagPayloads"].items() diff --git a/posthog/types.py b/posthog/types.py index 229b25a16..30e788447 100644 --- a/posthog/types.py +++ b/posthog/types.py @@ -639,7 +639,7 @@ def from_flag_details( FeatureFlagResult | None: A new FeatureFlagResult instance, or None if details is None. """ - if details is None: + if details is None or (details._failed and details._malformed_field is None): return None if override_match_value is not None: @@ -720,6 +720,7 @@ def to_values(response: FlagsResponse) -> Optional[dict[str, FlagValue]]: key: value.get_value() for key, value in flags.items() if isinstance(value, FeatureFlag) + and not (value._failed and value._malformed_field is None) } @@ -731,6 +732,7 @@ def to_payloads(response: FlagsResponse) -> Optional[dict[str, str]]: key: value.metadata.payload for key, value in response.get("flags", {}).items() if isinstance(value, FeatureFlag) + and not value._failed and value.enabled and value.metadata.payload is not None } From 907ecf19944e9244dec9aa909c415087bc1f292c Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Thu, 12 Nov 2026 11:53:01 +0100 Subject: [PATCH 2/8] test: expect no result for failed flag attribution --- posthog/test/test_experiment_exposure.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/posthog/test/test_experiment_exposure.py b/posthog/test/test_experiment_exposure.py index c12152833..db59f5c66 100644 --- a/posthog/test/test_experiment_exposure.py +++ b/posthog/test/test_experiment_exposure.py @@ -517,18 +517,28 @@ def _attribution_response(source: str) -> Dict[str, Any]: "source, expected", ATTRIBUTION_CASES, ids=[c[0] for c in ATTRIBUTION_CASES] ) def test_flag_called_attribution_follows_the_record(source, expected, path): - events = _read_events(path, _attribution_response(source), key="garden-layout") + response = _attribution_response(source) + failed = response["flags"]["garden-layout"].get("failed") is True + events = _read_events(path, response, key="garden-layout") [called] = _named(events, FLAG_CALLED, "garden-layout") assert _attribution(called.properties) == expected - FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + if failed: + # The vendored schema predates the no-result sentinel for failed records. + assert called.properties["$feature_flag_response"] is None + assert set(called.properties["$feature_flag_error"].split(",")) == { + "evaluation_failed", + "errors_while_computing_flags", + } + else: + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) exposures = _named(events, EXPOSURE, "garden-layout") assert len(exposures) == (1 if "$feature_flag_variant_key" in expected else 0) # A record whose reason names no rule, or that failed, reports no rule even when it -# carries one. Neither sends an exposure. The last element says whether the record is -# a valid server outcome, which the contract's event schema then accepts. +# carries one. Neither sends an exposure. The last element says whether a successful +# record is a valid server outcome, which the contract's event schema then accepts. STRAY_RULE_CASES = [ ("no_rule_match", {}, True), ("missing_group_key", {}, True), @@ -569,7 +579,10 @@ def test_rule_fields_are_sent_only_for_reasons_that_name_a_rule( "$feature_flag_reason_code": code, } assert _named(events, EXPOSURE) == [] - if valid_outcome: + if fields.get("failed") is True: + assert called.properties["$feature_flag_response"] is None + assert called.properties["$feature_flag_error"] == "evaluation_failed" + elif valid_outcome: FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) From 3da6048117a61cd48634be14f5b6eff2cbf715bc Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Thu, 12 Nov 2026 12:04:57 +0100 Subject: [PATCH 3/8] test: update released comparisons for failed flag records --- posthog/test/test_flags_released_behaviour.py | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/posthog/test/test_flags_released_behaviour.py b/posthog/test/test_flags_released_behaviour.py index 7f5dd12b9..d646e076e 100644 --- a/posthog/test/test_flags_released_behaviour.py +++ b/posthog/test/test_flags_released_behaviour.py @@ -38,9 +38,8 @@ def test_every_legacy_getter_and_event_matches_7_82_1(request, results): assert (sparse["id"], sparse["version"]) == (0, 0) sparse["id"] = sparse["version"] = None - # A failed record reads as disabled, so it has no payload, even when the record - # carries one. 7.82.1 returned that payload from the payload reads and sent it on - # the flag's $feature_flag_called events. + # An explicit failed record has no evaluated value or payload. 7.82.1 treated + # it as disabled, included it in bulk results, and returned its payload. failed = expected["get_flags_decision"]["flags"]["server-failed"]["metadata"] assert failed["payload"] == '{"columns": 2}' failed["payload"] = None @@ -48,7 +47,15 @@ def test_every_legacy_getter_and_event_matches_7_82_1(request, results): assert expected[f"{read}:server-failed"] == {"columns": 2} expected[f"{read}:server-failed"] = None assert expected["get_feature_flag_result:server-failed"][2] == {"columns": 2} - expected["get_feature_flag_result:server-failed"][2] = None + expected["get_feature_flag_result:server-failed"] = None + for read in ["get_feature_flag", "feature_enabled", "snapshot.get_flag"]: + assert expected[f"{read}:server-failed"] is False + expected[f"{read}:server-failed"] = None + for read in ["get_feature_variants", "get_all_flags"]: + assert expected[read].pop("server-failed") is False + for read in ["get_feature_flags_and_payloads", "get_all_flags_and_payloads"]: + assert expected[read]["featureFlags"].pop("server-failed") is False + expected["snapshot.keys"].remove("server-failed") failed_events = [ properties for _, properties in expected["events"] @@ -57,6 +64,13 @@ def test_every_legacy_getter_and_event_matches_7_82_1(request, results): assert failed_events for properties in failed_events: assert properties.pop("$feature_flag_payload") == {"columns": 2} + assert properties["$feature_flag_response"] is False + properties["$feature_flag_response"] = None + properties["$feature/server-failed"] = None + properties["$feature_flag_error"] = "evaluation_failed" + for name, properties in expected["events"]: + if name in ["snapshot-event", "fetched-event"]: + assert properties.pop("$feature/server-failed") is False if request.node.callspec.params["results"] == "v4": # A $feature_flag_called for a config version 2 record carries its config @@ -108,12 +122,16 @@ def test_fallback_cache_histories_match_7_82_1(results, name): # unknown_error, no request details, and the cached value. expected = copy.deepcopy(RELEASED["cache histories"][name]) if name == "good, server failed, timeout": - # A failed record has no payload, even when the record carries one, so - # neither its read nor the cached result read after the timeout returns - # one, and their events leave it out. 7.82.1 returned it from both. - for result in expected["results"][1:]: - assert result[2] == {"theme": "dark"} - result[2] = None - for _, properties in expected["events"][1:]: - assert properties.pop("$feature_flag_payload") == {"theme": "dark"} + # Explicit failure supplies no result and cannot poison the last good + # cache entry. The later timeout still returns that successful pair. + expected["results"][1] = None + expected["results"][2] = copy.deepcopy(expected["results"][0]) + failed = expected["events"][1][1] + assert failed.pop("$feature_flag_payload") == {"theme": "dark"} + failed["$feature_flag_response"] = None + failed["$feature/cached"] = None + failed["$feature_flag_error"] = "evaluation_failed" + timeout = expected["events"][2][1] + timeout["$feature_flag_response"] = "blue" + timeout["$feature/cached"] = "blue" assert results["cache histories"][name] == expected From 0f9c9cfec49d45a2095358cf8e3aa33dbbbb6b91 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Thu, 12 Nov 2026 12:07:59 +0100 Subject: [PATCH 4/8] fix: use qualified PyPI package in flag changeset --- .sampo/changesets/fix-failed-flag-records.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sampo/changesets/fix-failed-flag-records.md b/.sampo/changesets/fix-failed-flag-records.md index 867d7a368..f1500a70e 100644 --- a/.sampo/changesets/fix-failed-flag-records.md +++ b/.sampo/changesets/fix-failed-flag-records.md @@ -1,5 +1,5 @@ --- -posthog: patch +pypi/posthog: patch --- Treat failed feature flag records as unevaluated, report evaluation failure diagnostics, and preserve successful local values and payloads when remote fallback fails. From 0621e4d20f9c782a2ddcc7010cef89993ecb105c Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Thu, 12 Nov 2026 14:18:48 +0100 Subject: [PATCH 5/8] fix(flags): preserve unresolved reasons for failed fallback --- posthog/client.py | 9 +++++- posthog/test/test_evaluate_flags.py | 47 +++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/posthog/client.py b/posthog/client.py index de560dfef..069787478 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -5563,7 +5563,14 @@ def evaluate_flags( quota_limited=quota_limited, minimal_flag_called_events=minimal_flag_called_events, unresolved_flags={ - key: reason for key, reason in unresolved.items() if key not in records + key: reason + for key, reason in unresolved.items() + if key not in records + or ( + isinstance(record := records[key], _RemoteFlagRecord) + and record.details._failed + and record.details._malformed_field is None + ) }, ) diff --git a/posthog/test/test_evaluate_flags.py b/posthog/test/test_evaluate_flags.py index dbe33583c..1382e4dd5 100644 --- a/posthog/test/test_evaluate_flags.py +++ b/posthog/test/test_evaluate_flags.py @@ -885,19 +885,60 @@ def test_dependency_resolved_locally_picks_its_condition(self, patch_flags): self.assertEqual(flags.get_flag("target"), "a") patch_flags.assert_not_called() + @parameterized.expand([(True,), (False,)]) @mock.patch("posthog.client.flags") - def test_flag_resolved_by_remote_fallback_is_not_unresolved(self, patch_flags): + def test_flag_resolved_by_remote_fallback_is_not_unresolved( + self, enabled, patch_flags + ): patch_flags.return_value = { - "flags": {"checkout": {"enabled": True, "variant": None}} + "flags": {"checkout": {"enabled": enabled, "variant": None}} } flags = self.client.evaluate_flags("user-123") - self.assertIs(flags.get_flag("checkout"), True) + self.assertIs(flags.get_flag("checkout"), enabled) self.assertEqual(sorted(flags.keys), ["beta-ui", "checkout"]) self.assertEqual(dict(flags.unresolved_flags), {}) patch_flags.assert_called_once() + @parameterized.expand([(None,), (False,), (True,)]) + @mock.patch("posthog.client.flags") + @mock.patch.object(Client, "capture") + def test_failed_remote_fallback_preserves_unresolved_reason( + self, error_bit, patch_capture, patch_flags + ): + response = { + "flags": { + key: { + "enabled": False, + "failed": True, + "metadata": {"payload": '"placeholder"'}, + } + for key in ("checkout", "remote-only") + } + } + if error_bit is not None: + response["errorsWhileComputingFlags"] = error_bit + patch_flags.return_value = response + + flags = self.client.evaluate_flags("user-123") + + self.assertEqual( + dict(flags.unresolved_flags), + {"checkout": UnresolvedFlagReason.EXPERIENCE_CONTINUITY}, + ) + self.assertEqual(flags.keys, ["beta-ui"]) + self.assertIsNone(flags.get_flag("checkout", send_feature_flag_events=False)) + self.assertTrue( + flags.is_enabled("checkout", True, send_feature_flag_events=False) + ) + self.assertIsNone(flags.get_flag_payload("checkout")) + self.assertNotIn("$feature/checkout", flags._get_event_properties()) + self.assertEqual(dict(flags.only(["checkout"]).unresolved_flags), {}) + self.assertEqual(dict(flags.only_accessed().unresolved_flags), {}) + patch_capture.assert_not_called() + patch_flags.assert_called_once() + @parameterized.expand( [ ( From 611a869bb66bd17b1baee03658d9625920ac7117 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Fri, 13 Nov 2026 09:51:01 +0100 Subject: [PATCH 6/8] fix(flags): clarify failed-record classification and regression coverage --- .sampo/changesets/fix-failed-flag-records.md | 2 +- .../openfeature-failed-flag-defaults.md | 5 ++ posthog/client.py | 55 +++++++++---------- posthog/feature_flag_evaluations.py | 26 +++++---- posthog/test/test_evaluate_flags.py | 18 ++++++ posthog/test/test_experiment_exposure.py | 2 +- posthog/test/test_failed_flag_records.py | 16 +++++- posthog/types.py | 19 +++++-- references/public_api_snapshot.txt | 1 + 9 files changed, 96 insertions(+), 48 deletions(-) create mode 100644 .sampo/changesets/openfeature-failed-flag-defaults.md diff --git a/.sampo/changesets/fix-failed-flag-records.md b/.sampo/changesets/fix-failed-flag-records.md index f1500a70e..15020ba8f 100644 --- a/.sampo/changesets/fix-failed-flag-records.md +++ b/.sampo/changesets/fix-failed-flag-records.md @@ -2,4 +2,4 @@ pypi/posthog: patch --- -Treat failed feature flag records as unevaluated, report evaluation failure diagnostics, and preserve successful local values and payloads when remote fallback fails. +Treat server-marked failed feature flag records as unevaluated: direct result, value, enabled and payload getters return None rather than a disabled False result or placeholder payload. Snapshot value and payload reads return None, enablement honors the caller's default, and failed keys are omitted from enumeration and capture enrichment. Bulk value and payload maps omit failed remote records while preserving successful locally computed values and payloads for those keys. Report evaluation_failed diagnostics and retain local unresolved reasons after failed fallback; healthy False and malformed-record compatibility are unchanged. diff --git a/.sampo/changesets/openfeature-failed-flag-defaults.md b/.sampo/changesets/openfeature-failed-flag-defaults.md new file mode 100644 index 000000000..f9f5535b8 --- /dev/null +++ b/.sampo/changesets/openfeature-failed-flag-defaults.md @@ -0,0 +1,5 @@ +--- +pypi/openfeature-provider-posthog: patch +--- + +Document failed-record default handling with the updated PostHog SDK: a server-marked failed flag supplies no result, so the provider's existing FlagNotFoundError mapping lets OpenFeature return the caller's default instead of treating the failed record as a healthy disabled flag. This behavior requires upgrading the installed posthog dependency; the provider's compatible dependency range is unchanged and this patch intent does not force that upgrade. diff --git a/posthog/client.py b/posthog/client.py index 069787478..0c8ed2ac0 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -633,18 +633,19 @@ def _capture_flag_events( Every read that sends flag events comes here, on both clients: the single-flag getters and the reads of an ``evaluate_flags()`` snapshot. Both events are built - from ``details``, the parsed ``/flags`` record that supplied the value, and the - call's ``properties``; the flag is never evaluated again. + from ``details``, the parsed ``/flags`` record supplying the value or failure + diagnostics, and the call's ``properties``; the flag is never evaluated again. ``$feature_flag_called`` is sent if the (distinct_id, flag, response, groups) tuple hasn't already been reported on this client. Group context is included so that group-scoped flags fire a separate event for each group a user is evaluated - under. A config version 2 record that supplied the reported value + under. A config version 2 record supplying the value or failed-record diagnostics (``value_from_details``) adds its attribution to the event and to that tuple, so equal-valued A/A arms, successive experiments of one flag, and holdouts are each reported. Its reason code replaces the human-readable ``$feature_flag_reason``. - A config version 1 event and tuple are unchanged, and so are those of a read - whose value the record did not supply. + A failed record keeps its config version and reason code, but no rule tuple or + experiment exposure. Config version 1 events and reads reporting a successful + caller override or stale cached value keep their existing attribution. ``has_experiment`` is the server-reported signal for whether the flag is linked to an experiment (``None`` when the server did not report it). When the @@ -677,9 +678,8 @@ def _capture_flag_events( groups_key = ( tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () ) - # Only a record that supplied the reported value attributes it. A caller's - # match_value and a stale cached value are reported as before, with no config - # version 2 properties, even when a config version 2 record came with them. + # A failed record still reports its config version and reason code. A successful + # caller override or stale cached value has no config version 2 attribution. attribution = _rule_attribution(details) if value_from_details else None # What both config version 2 events report about the read itself. Each event # pins it, so a super property cannot make the pair disagree. @@ -4561,8 +4561,8 @@ def _get_feature_flag_result( flag_result = None flag_details = None - # Whether flag_details supplied the value this read reports. Only then may the - # read report config version 2 attribution or send $experiment_exposure. + # Whether flag_details supplies the reported value or failed-record diagnostics. + # A failed record keeps its config version and reason code, but sends no exposure. value_from_details = False request_id = None evaluated_at = None @@ -4646,8 +4646,8 @@ def _get_feature_flag_result( errors.append(FeatureFlagError.ERRORS_WHILE_COMPUTING) if flag_details is None: errors.append(FeatureFlagError.FLAG_MISSING) - elif flag_details._failed: - errors.append("evaluation_failed") + elif flag_details._server_marked_failed: + errors.append(FeatureFlagError.EVALUATION_FAILED) if errors: feature_flag_error = ",".join(errors) @@ -4679,11 +4679,10 @@ def _get_feature_flag_result( self.log.debug( f"Successfully computed flag remotely: #{key} -> #{flag_result}" ) - # A successful payload override reports the caller's value. A - # failed record still supplies diagnostics, never the override. + # A successful payload override reports the caller's value. A failed + # record keeps its config version and reason code, never the override. value_from_details = flag_details is not None and ( - override_match_value is None - or (flag_details._failed and flag_details._malformed_field is None) + override_match_value is None or flag_details._server_marked_failed ) except QuotaLimitError as e: self.log.warning(f"[FEATURE FLAGS] Quota limit exceeded: {e}") @@ -4793,7 +4792,9 @@ def get_feature_flag_result( device_id: The device ID for this request. Returns: - Optional[FeatureFlagResult]: The feature flag result or None if disabled/not found. + Optional[FeatureFlagResult]: The evaluated result, including a healthy + disabled flag, or None when no result is available (for example, + a missing flag or a server-marked failure without a successful fallback). """ return self._get_feature_flag_result( key, @@ -5042,9 +5043,8 @@ def _capture_feature_flag_called( local_flag_id: Optional[int] = None, local_flag_version: Optional[int] = None, *, - # Keyword-only, and false by default: a caller that does not say the record - # supplied the value reports no config version 2 attribution and sends no - # $experiment_exposure. + # Opt in when the record supplies the value or failed-record diagnostics. + # Failed records keep config version and reason code, but send no exposure. value_from_details: bool = False, ): properties: dict[str, Any] = { @@ -5110,9 +5110,9 @@ def _capture_feature_flag_called_if_needed( Shared by the single-flag evaluation path and the reads of a ``FeatureFlagEvaluations`` snapshot, so every read reports and dedupes identically. ``details`` is the parsed ``/flags`` record of the read, or - ``None`` for a locally evaluated flag. ``value_from_details`` says whether that - record supplied the reported value; only then does the read report config - version 2 attribution or send an exposure. + ``None`` for a locally evaluated flag. ``value_from_details`` includes a + failed record supplying config-version and reason-code diagnostics despite + having no value. Only a successful experiment split sends an exposure. """ _capture_flag_events( self.capture, @@ -5325,12 +5325,12 @@ def get_all_flags_and_payloads( device_id=device_id, ) remote_response = to_flags_and_payloads(decide_response) - # Only explicit failed records preserve same-context local pairs; - # successful remote records retain the existing precedence. + # A key the server marked failed keeps its locally computed value + # and payload. Successful remote records replace local results. failed_keys = { key for key, detail in decide_response.get("flags", {}).items() - if detail._failed and detail._malformed_field is None + if detail._server_marked_failed } response = { "featureFlags": { @@ -5568,8 +5568,7 @@ def evaluate_flags( if key not in records or ( isinstance(record := records[key], _RemoteFlagRecord) - and record.details._failed - and record.details._malformed_field is None + and record.details._server_marked_failed ) }, ) diff --git a/posthog/feature_flag_evaluations.py b/posthog/feature_flag_evaluations.py index d938b43aa..78c84d536 100644 --- a/posthog/feature_flag_evaluations.py +++ b/posthog/feature_flag_evaluations.py @@ -10,6 +10,7 @@ from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union from .types import FeatureFlag as _FeatureFlag +from .types import FeatureFlagError as _FeatureFlagError from .types import FlagMetadata as _FlagMetadata from .types import FlagValue, UnresolvedFlagReason, _parse_flag_payload @@ -156,8 +157,7 @@ def __init__( key: flag for key, flag in flags.items() if isinstance(flag, _RemoteFlagRecord) - and flag.details._failed - and flag.details._malformed_field is None + and flag.details._server_marked_failed } self._flags = {k: v for k, v in flags.items() if k not in self._failed_flags} self._groups: Dict[str, Union[str, int]] = dict(groups or {}) @@ -188,9 +188,10 @@ def is_enabled( Args: key: The feature flag key. default_value: Returned when the flag has no value in this evaluation — - the key was not part of the evaluated set, or the evaluation itself - came back empty (failed ``/flags`` request, quota limit, or no - resolvable ``distinct_id``). Defaults to ``False``. + the key was not part of the evaluated set, the server marked its + record failed, or the evaluation itself came back empty (failed + ``/flags`` request, quota limit, or no resolvable ``distinct_id``). + Defaults to ``False``. send_feature_flag_events: Whether this read sends ``$feature_flag_called``. When ``False``, no event is sent and the dedupe cache is left alone, but the key still counts as accessed for :meth:`only_accessed`. @@ -209,7 +210,8 @@ def get_flag( Returns the variant string for multivariate flags, ``True`` for enabled flags without a variant, ``False`` for disabled flags, and ``None`` for flags that - were not returned by the evaluation. + were not returned or whose remote record was marked failed by the server. + An unreadable remote record retains its disabled (``False``) result. Pass ``send_feature_flag_events=False`` to skip the event for this read, as with :meth:`is_enabled`. @@ -272,8 +274,10 @@ def unresolved_flags(self) -> Mapping[str, UnresolvedFlagReason]: A flag is listed when it has a loaded local definition, was in the requested scope, was inconclusive locally, and no remote value filled the gap. Listed flags are not part of :attr:`keys`, and reading one reports the - ``local_evaluation_inconclusive`` error instead of ``flag_missing``. Reading - this mapping does not count as an access and does not fire any event. + ``local_evaluation_inconclusive`` error instead of ``flag_missing``. + Currently, a server-marked failed fallback instead reports ``evaluation_failed``; + this is an implementation exception to the snapshot inconclusive-error contract. + Reading this mapping does not count as an access and does not fire any event. Snapshots returned by :meth:`only` and :meth:`only_accessed`, and by ``AsyncClient.evaluate_flags()``, which evaluates only remotely, carry no entries. @@ -385,7 +389,7 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: if self._quota_limited: errors.append("quota_limited") if failed: - errors.append("evaluation_failed") + errors.append(_FeatureFlagError.EVALUATION_FAILED) elif flag is None: errors.append( "local_evaluation_inconclusive" @@ -404,8 +408,8 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: properties=properties, has_experiment=flag.has_experiment if flag else None, minimal_flag_called_events=self._minimal_flag_called_events, - # The record that supplied the value. A remotely evaluated experiment - # split also sends $experiment_exposure, whatever this read returned. + # A failed record still supplies config-version and reason-code diagnostics. + # Only a successful experiment split also sends $experiment_exposure. details=flag.details if isinstance(flag, _RemoteFlagRecord) else None, value_from_details=True, ) diff --git a/posthog/test/test_evaluate_flags.py b/posthog/test/test_evaluate_flags.py index 1382e4dd5..dd8fb02f5 100644 --- a/posthog/test/test_evaluate_flags.py +++ b/posthog/test/test_evaluate_flags.py @@ -901,6 +901,24 @@ def test_flag_resolved_by_remote_fallback_is_not_unresolved( self.assertEqual(dict(flags.unresolved_flags), {}) patch_flags.assert_called_once() + @mock.patch("posthog.client.flags") + def test_malformed_remote_fallback_is_not_unresolved(self, patch_flags): + patch_flags.return_value = { + "flags": { + "checkout": { + "enabled": True, + "metadata": {"version": True}, + } + } + } + + flags = self.client.evaluate_flags("user-123") + + self.assertIs(flags.get_flag("checkout", send_feature_flag_events=False), False) + self.assertEqual(sorted(flags.keys), ["beta-ui", "checkout"]) + self.assertEqual(dict(flags.unresolved_flags), {}) + patch_flags.assert_called_once() + @parameterized.expand([(None,), (False,), (True,)]) @mock.patch("posthog.client.flags") @mock.patch.object(Client, "capture") diff --git a/posthog/test/test_experiment_exposure.py b/posthog/test/test_experiment_exposure.py index db59f5c66..062f90e46 100644 --- a/posthog/test/test_experiment_exposure.py +++ b/posthog/test/test_experiment_exposure.py @@ -524,7 +524,7 @@ def test_flag_called_attribution_follows_the_record(source, expected, path): [called] = _named(events, FLAG_CALLED, "garden-layout") assert _attribution(called.properties) == expected if failed: - # The vendored schema predates the no-result sentinel for failed records. + # The vendored schema rejects the null $feature_flag_response a failed record reports. assert called.properties["$feature_flag_response"] is None assert set(called.properties["$feature_flag_error"].split(",")) == { "evaluation_failed", diff --git a/posthog/test/test_failed_flag_records.py b/posthog/test/test_failed_flag_records.py index 09a9fe456..65f550526 100644 --- a/posthog/test/test_failed_flag_records.py +++ b/posthog/test/test_failed_flag_records.py @@ -138,15 +138,26 @@ def test_failed_fallback_preserves_only_successful_local_pairs( client, shape, error_bit ): local = { - "featureFlags": {"on": True, "off": False, "variant": "blue", "success": True}, + "featureFlags": { + "on": True, + "off": False, + "variant": "blue", + "success": True, + "local-only": True, + "malformed": True, + }, "featureFlagPayloads": { - k: '{"copy":"local"}' for k in ["on", "off", "variant", "success"] + k: '{"copy":"local"}' + for k in ["on", "off", "variant", "success", "local-only", "malformed"] }, } remote = response(shape, error_bit) for key in ["on", "off", "variant"]: remote["flags"][key] = copy.deepcopy(remote["flags"]["failed"]) remote["flags"][key]["key"] = key + remote["flags"]["malformed"] = copy.deepcopy(remote["flags"]["failed"]) + remote["flags"]["malformed"]["key"] = "malformed" + remote["flags"]["malformed"]["metadata"]["version"] = True remote["flags"]["success"] = { "key": "success", "enabled": True, @@ -169,6 +180,7 @@ def test_failed_fallback_preserves_only_successful_local_pairs( "off": False, "variant": "blue", "success": "remote", + "malformed": False, } assert result["featureFlagPayloads"] == { k: '{"copy":"success"}' if k == "success" else '{"copy":"local"}' diff --git a/posthog/types.py b/posthog/types.py index 30e788447..f0bf739b5 100644 --- a/posthog/types.py +++ b/posthog/types.py @@ -275,6 +275,11 @@ class FeatureFlag: # server marks failed has ``_failed`` set and this ``None``. _malformed_field: ClassVar[Optional[str]] = None + @property + def _server_marked_failed(self) -> bool: + """Whether a readable record reports a server evaluation failure.""" + return self._failed and self._malformed_field is None + def __post_init__(self) -> None: object.__setattr__( self, @@ -634,12 +639,15 @@ def from_flag_details( details (Union[FeatureFlag, None]): The FeatureFlag object to convert. override_match_value (Optional[FlagValue]): If provided, this value will be used to populate the enabled and variant fields instead of the values from the FeatureFlag. + A server-marked failure returns None even when this override is provided. Returns: - FeatureFlagResult | None: A new FeatureFlagResult instance, or None if details is None. + FeatureFlagResult | None: A new result, or None when details is absent or + the server marked the record failed. Unreadable records are not + treated as server-marked failures. """ - if details is None or (details._failed and details._malformed_field is None): + if details is None or details._server_marked_failed: return None if override_match_value is not None: @@ -719,8 +727,7 @@ def to_values(response: FlagsResponse) -> Optional[dict[str, FlagValue]]: return { key: value.get_value() for key, value in flags.items() - if isinstance(value, FeatureFlag) - and not (value._failed and value._malformed_field is None) + if isinstance(value, FeatureFlag) and not value._server_marked_failed } @@ -732,7 +739,7 @@ def to_payloads(response: FlagsResponse) -> Optional[dict[str, str]]: key: value.metadata.payload for key, value in response.get("flags", {}).items() if isinstance(value, FeatureFlag) - and not value._failed + and not value._server_marked_failed and value.enabled and value.metadata.payload is not None } @@ -747,6 +754,7 @@ class FeatureFlagError: Error values: ERRORS_WHILE_COMPUTING: Server returned errorsWhileComputingFlags=true + EVALUATION_FAILED: Server marked the requested flag failed, with no evaluated value FLAG_MISSING: Requested flag not in API response LOCAL_EVALUATION_INCONCLUSIVE: Requested flag has a loaded definition but local evaluation was inconclusive and no remote value filled the gap @@ -760,6 +768,7 @@ class FeatureFlagError: """ ERRORS_WHILE_COMPUTING = "errors_while_computing_flags" + EVALUATION_FAILED = "evaluation_failed" FLAG_MISSING = "flag_missing" LOCAL_EVALUATION_INCONCLUSIVE = "local_evaluation_inconclusive" QUOTA_LIMITED = "quota_limited" diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index bfffd5d64..1f1f4a070 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -1178,6 +1178,7 @@ attribute posthog.types.FeatureFlag.reason: Optional[FlagReason] attribute posthog.types.FeatureFlag.variant: Optional[str] attribute posthog.types.FeatureFlagError.CONNECTION_ERROR = 'connection_error' attribute posthog.types.FeatureFlagError.ERRORS_WHILE_COMPUTING = 'errors_while_computing_flags' +attribute posthog.types.FeatureFlagError.EVALUATION_FAILED = 'evaluation_failed' attribute posthog.types.FeatureFlagError.FLAG_MISSING = 'flag_missing' attribute posthog.types.FeatureFlagError.LOCAL_EVALUATION_INCONCLUSIVE = 'local_evaluation_inconclusive' attribute posthog.types.FeatureFlagError.QUOTA_LIMITED = 'quota_limited' From 489a118646ffe76f4164c01049352692289209ea Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Fri, 13 Nov 2026 12:50:46 +0100 Subject: [PATCH 7/8] fix(flags): preserve failed snapshot diagnostics --- .sampo/changesets/fix-failed-flag-records.md | 2 +- posthog/feature_flag_evaluations.py | 28 ++++--- posthog/test/test_evaluate_flags.py | 15 ++++ posthog/test/test_failed_flag_records.py | 82 ++++++++++++++++++++ 4 files changed, 117 insertions(+), 10 deletions(-) diff --git a/.sampo/changesets/fix-failed-flag-records.md b/.sampo/changesets/fix-failed-flag-records.md index 15020ba8f..e52498056 100644 --- a/.sampo/changesets/fix-failed-flag-records.md +++ b/.sampo/changesets/fix-failed-flag-records.md @@ -2,4 +2,4 @@ pypi/posthog: patch --- -Treat server-marked failed feature flag records as unevaluated: direct result, value, enabled and payload getters return None rather than a disabled False result or placeholder payload. Snapshot value and payload reads return None, enablement honors the caller's default, and failed keys are omitted from enumeration and capture enrichment. Bulk value and payload maps omit failed remote records while preserving successful locally computed values and payloads for those keys. Report evaluation_failed diagnostics and retain local unresolved reasons after failed fallback; healthy False and malformed-record compatibility are unchanged. +Treat server-marked failed feature flag records as unevaluated: direct result, value, enabled and payload getters return None rather than a disabled False result or placeholder payload. Snapshot value and payload reads return None, enablement honors the caller's default, and failed keys are omitted from enumeration and capture enrichment. Bulk value and payload maps omit failed remote records while preserving successful locally computed values and payloads for those keys. Preserve selected or accessed failure diagnostics through snapshot filters and report local_evaluation_inconclusive alongside evaluation_failed after an unresolved failed fallback; healthy False and malformed-record compatibility are unchanged. diff --git a/posthog/feature_flag_evaluations.py b/posthog/feature_flag_evaluations.py index 78c84d536..ef32401c8 100644 --- a/posthog/feature_flag_evaluations.py +++ b/posthog/feature_flag_evaluations.py @@ -238,13 +238,23 @@ def only_accessed(self) -> "FeatureFlagEvaluations": Order-dependent: if nothing has been accessed yet, the returned snapshot is empty. The method honors its name — pre-access if you want a populated result. + Accessed failures retain private diagnostics, not values or enrichment. """ filtered = {k: self._flags[k] for k in self._accessed if k in self._flags} + filtered.update( + { + k: self._failed_flags[k] + for k in self._accessed + if k in self._failed_flags + } + ) return self._clone_with(filtered) def only(self, keys: List[str]) -> "FeatureFlagEvaluations": """Return a filtered copy containing only flags with the given keys. Keys that are not present in the evaluation are dropped and logged as a warning. + Selected failures retain private diagnostics but remain absent from values, + keys and enrichment; their existing missing-value warning is unchanged. """ filtered: Dict[str, _EvaluatedFlagRecord] = {} missing: List[str] = [] @@ -259,6 +269,9 @@ def only(self, keys: List[str]) -> "FeatureFlagEvaluations": "FeatureFlagEvaluations.only() was called with flag keys that are not in the " f"evaluation set and will be dropped: {', '.join(missing)}" ) + filtered.update( + {k: self._failed_flags[k] for k in keys if k in self._failed_flags} + ) return self._clone_with(filtered) @property @@ -275,8 +288,7 @@ def unresolved_flags(self) -> Mapping[str, UnresolvedFlagReason]: scope, was inconclusive locally, and no remote value filled the gap. Listed flags are not part of :attr:`keys`, and reading one reports the ``local_evaluation_inconclusive`` error instead of ``flag_missing``. - Currently, a server-marked failed fallback instead reports ``evaluation_failed``; - this is an implementation exception to the snapshot inconclusive-error contract. + A server-marked failed fallback additionally reports ``evaluation_failed``. Reading this mapping does not count as an access and does not fire any event. Snapshots returned by :meth:`only` and :meth:`only_accessed`, and by ``AsyncClient.evaluate_flags()``, which evaluates only remotely, carry no @@ -381,8 +393,8 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: # Build the comma-joined `$feature_flag_error` matching the single-flag path's # granularity: response-level errors (errors-while-computing, quota-limited) are - # combined with per-flag errors (flag-missing or local-evaluation-inconclusive) - # so consumers can filter by type. + # combined with per-flag errors. A failed remote fallback must not hide the + # original snapshot's local-inconclusive diagnostic. errors: List[str] = [] if self._errors_while_computing: errors.append("errors_while_computing_flags") @@ -390,12 +402,10 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: errors.append("quota_limited") if failed: errors.append(_FeatureFlagError.EVALUATION_FAILED) + if key in self._unresolved_flags: + errors.append(_FeatureFlagError.LOCAL_EVALUATION_INCONCLUSIVE) elif flag is None: - errors.append( - "local_evaluation_inconclusive" - if key in self._unresolved_flags - else "flag_missing" - ) + errors.append("flag_missing") if errors: properties["$feature_flag_error"] = ",".join(errors) diff --git a/posthog/test/test_evaluate_flags.py b/posthog/test/test_evaluate_flags.py index dd8fb02f5..448ea49a2 100644 --- a/posthog/test/test_evaluate_flags.py +++ b/posthog/test/test_evaluate_flags.py @@ -955,6 +955,21 @@ def test_failed_remote_fallback_preserves_unresolved_reason( self.assertEqual(dict(flags.only(["checkout"]).unresolved_flags), {}) self.assertEqual(dict(flags.only_accessed().unresolved_flags), {}) patch_capture.assert_not_called() + flags.is_enabled("checkout", True) + flags.get_flag("checkout") + flags.get_flag("remote-only") + properties = _flag_called_properties(patch_capture) + expected_errors = "evaluation_failed,local_evaluation_inconclusive" + remote_errors = "evaluation_failed" + if error_bit: + expected_errors = "errors_while_computing_flags," + expected_errors + remote_errors = "errors_while_computing_flags," + remote_errors + self.assertEqual(properties["checkout"]["$feature_flag_error"], expected_errors) + self.assertIsNone(properties["checkout"]["$feature_flag_response"]) + self.assertEqual( + properties["remote-only"]["$feature_flag_error"], remote_errors + ) + self.assertEqual(patch_capture.call_count, 2) patch_flags.assert_called_once() @parameterized.expand( diff --git a/posthog/test/test_failed_flag_records.py b/posthog/test/test_failed_flag_records.py index 65f550526..c6ec36582 100644 --- a/posthog/test/test_failed_flag_records.py +++ b/posthog/test/test_failed_flag_records.py @@ -132,6 +132,88 @@ def test_failed_snapshot_defaults_diagnostics_and_enrichment(client, shape, erro assert snapshot._get_event_properties() == {"$feature/off": False} +@pytest.mark.parametrize("shape", ["v2", "v4"]) +@pytest.mark.parametrize("error_bit", [None, False, True]) +@pytest.mark.parametrize("filter_method", ["only", "only_accessed"]) +def test_filtered_snapshot_preserves_selected_failure_diagnostics( + client, shape, error_bit, filter_method +): + raw = response(shape, error_bit) + raw["requestId"] = "evaluation-request" + raw["evaluatedAt"] = 12345 + raw["flags"]["failed"]["reason"]["description"] = "Evaluation failed" + raw["flags"]["failed"]["metadata"]["has_experiment"] = True + raw["flags"]["other-failed"] = copy.deepcopy(raw["flags"]["failed"]) + raw["flags"]["other-failed"]["key"] = "other-failed" + with ( + mock.patch("posthog.client.flags", return_value=raw) as remote, + mock.patch.object(client, "_enqueue", return_value=(True, {})) as enqueue, + mock.patch.object(client.log, "warning") as warning, + ): + snapshot = client.evaluate_flags("user", groups={"company": "acme"}) + assert snapshot.get_flag_payload("other-failed") is None + if filter_method == "only": + filtered = snapshot.only(["failed", "off", "missing"]) + warning.assert_called_once() + assert warning.call_args.args[0].endswith("failed, missing") + else: + assert snapshot.only_accessed().keys == [] + snapshot.get_flag("failed", send_feature_flag_events=False) + snapshot.is_enabled("off", send_feature_flag_events=False) + filtered = snapshot.only_accessed() + warning.assert_not_called() + enqueue.assert_not_called() + assert filtered.keys == ["off"] + assert dict(filtered.unresolved_flags) == {} + assert filtered.get_flag_payload("failed") is None + assert ( + filtered.is_enabled("failed", True, send_feature_flag_events=False) is True + ) + enqueue.assert_not_called() + + assert filtered.get_flag("failed") is None + assert filtered.is_enabled("failed") is False + assert enqueue.call_count == 1 + called = enqueue.call_args.args[0] + assert called["event"] == "$feature_flag_called" + props = called["properties"] + assert props["$feature_flag_response"] is None + assert props["$feature_flag_error"] == ( + "errors_while_computing_flags,evaluation_failed" + if error_bit + else "evaluation_failed" + ) + assert props["$feature_flag_id"] == 1 + assert props["$feature_flag_version"] == 2 + assert props["$feature_flag_request_id"] == "evaluation-request" + assert props["$feature_flag_evaluated_at"] == 12345 + assert props["$groups"] == {"company": "acme"} + assert "$feature_flag_payload" not in props + if shape == "v4": + assert props["$feature_flag_config_version"] == 2 + assert props["$feature_flag_reason_code"] == "dependency_error" + else: + assert props["$feature_flag_reason"] == "Evaluation failed" + + client.capture("filtered-snapshot", distinct_id="user", flags=filtered) + enriched = enqueue.call_args.args[0] + assert enriched["event"] == "filtered-snapshot" + assert enriched["properties"]["$feature/off"] is False + assert "$feature/failed" not in enriched["properties"] + assert "$feature/other-failed" not in enriched["properties"] + assert enqueue.call_count == 2 + + assert filtered.get_flag("other-failed") is None + missing = enqueue.call_args.args[0]["properties"] + assert missing["$feature_flag_error"] == ( + "errors_while_computing_flags,flag_missing" if error_bit else "flag_missing" + ) + assert "$feature_flag_id" not in missing + assert "$feature_flag_reason_code" not in missing + assert enqueue.call_count == 3 + remote.assert_called_once() + + @pytest.mark.parametrize("shape", ["v2", "v4"]) @pytest.mark.parametrize("error_bit", [None, False, True]) def test_failed_fallback_preserves_only_successful_local_pairs( From 48307c0e4231ebe6201c02d7d629e4c6a231d61e Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Fri, 13 Nov 2026 15:05:29 +0100 Subject: [PATCH 8/8] ci: upgrade SDK compliance harness to 1.31.1 --- .github/workflows/sdk-compliance.yml | 8 ++++---- sdk_compliance_adapter/CONTRIBUTING.md | 4 ++-- sdk_compliance_adapter/docker-compose.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sdk-compliance.yml b/.github/workflows/sdk-compliance.yml index 7e72514a8..ea14f9b5e 100644 --- a/.github/workflows/sdk-compliance.yml +++ b/.github/workflows/sdk-compliance.yml @@ -29,20 +29,20 @@ jobs: compliance: name: PostHog SDK compliance tests (capture v0) - uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@75fb00092c6045daef60c30acc10ac83afefbfe9 # 1.27.0 + uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@003d4162421193b458bbaa93fbb7bb0d0104bc3b # 1.31.1 with: adapter-dockerfile: "sdk_compliance_adapter/Dockerfile" adapter-context: "." - test-harness-version: "1.28.0" + test-harness-version: "1.31.1" continue-on-error: false report-name: "sdk-compliance-report-v0" compliance-v1: name: PostHog SDK compliance tests (capture v1) - uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@75fb00092c6045daef60c30acc10ac83afefbfe9 # 1.27.0 + uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@003d4162421193b458bbaa93fbb7bb0d0104bc3b # 1.31.1 with: adapter-dockerfile: "sdk_compliance_adapter/Dockerfile.v1" adapter-context: "." - test-harness-version: "1.28.0" + test-harness-version: "1.31.1" continue-on-error: false report-name: "sdk-compliance-report-v1" diff --git a/sdk_compliance_adapter/CONTRIBUTING.md b/sdk_compliance_adapter/CONTRIBUTING.md index 8514a342c..22b26b50a 100644 --- a/sdk_compliance_adapter/CONTRIBUTING.md +++ b/sdk_compliance_adapter/CONTRIBUTING.md @@ -4,7 +4,7 @@ This package contains the PostHog Python SDK compliance adapter used with the Po ## Running tests -Tests run automatically in CI via GitHub Actions against harness **1.1.1**, for both capture protocols. Both jobs opt into the local-evaluation suite and fail on compliance regressions. +Tests run automatically in CI via GitHub Actions against harness **1.31.1**, for both capture protocols. Both jobs opt into the local-evaluation suite and fail on compliance regressions. Run adapter protocol tests from the repository root in an activated virtual environment: @@ -44,7 +44,7 @@ docker run -d --name sdk-adapter --network test-network -p 8080:8080 posthog-pyt docker run --rm \ --name test-harness \ --network test-network \ - ghcr.io/posthog/sdk-test-harness:1.28.0 \ + ghcr.io/posthog/sdk-test-harness:1.31.1 \ run --adapter-url http://sdk-adapter:8080 --mock-url http://test-harness:8081 # Cleanup diff --git a/sdk_compliance_adapter/docker-compose.yml b/sdk_compliance_adapter/docker-compose.yml index dcbd677de..ead39d536 100644 --- a/sdk_compliance_adapter/docker-compose.yml +++ b/sdk_compliance_adapter/docker-compose.yml @@ -23,7 +23,7 @@ services: # Test harness test-harness: - image: ghcr.io/posthog/sdk-test-harness:1.28.0 + image: ghcr.io/posthog/sdk-test-harness:1.31.1 command: ["run", "--adapter-url", "http://sdk-adapter:8080", "--mock-url", "http://test-harness:8081"] networks: - test-network