diff --git a/.github/workflows/sdk-compliance.yml b/.github/workflows/sdk-compliance.yml index 7e72514a8..ea14f9b5e 100644 --- a/.github/workflows/sdk-compliance.yml +++ b/.github/workflows/sdk-compliance.yml @@ -29,20 +29,20 @@ jobs: compliance: name: PostHog SDK compliance tests (capture v0) - uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@75fb00092c6045daef60c30acc10ac83afefbfe9 # 1.27.0 + uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@003d4162421193b458bbaa93fbb7bb0d0104bc3b # 1.31.1 with: adapter-dockerfile: "sdk_compliance_adapter/Dockerfile" adapter-context: "." - test-harness-version: "1.28.0" + test-harness-version: "1.31.1" continue-on-error: false report-name: "sdk-compliance-report-v0" compliance-v1: name: PostHog SDK compliance tests (capture v1) - uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@75fb00092c6045daef60c30acc10ac83afefbfe9 # 1.27.0 + uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@003d4162421193b458bbaa93fbb7bb0d0104bc3b # 1.31.1 with: adapter-dockerfile: "sdk_compliance_adapter/Dockerfile.v1" adapter-context: "." - test-harness-version: "1.28.0" + test-harness-version: "1.31.1" continue-on-error: false report-name: "sdk-compliance-report-v1" diff --git a/.sampo/changesets/fix-failed-flag-records.md b/.sampo/changesets/fix-failed-flag-records.md new file mode 100644 index 000000000..e52498056 --- /dev/null +++ b/.sampo/changesets/fix-failed-flag-records.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Treat server-marked failed feature flag records as unevaluated: direct result, value, enabled and payload getters return None rather than a disabled False result or placeholder payload. Snapshot value and payload reads return None, enablement honors the caller's default, and failed keys are omitted from enumeration and capture enrichment. Bulk value and payload maps omit failed remote records while preserving successful locally computed values and payloads for those keys. Preserve selected or accessed failure diagnostics through snapshot filters and report local_evaluation_inconclusive alongside evaluation_failed after an unresolved failed fallback; healthy False and malformed-record compatibility are unchanged. diff --git a/.sampo/changesets/openfeature-failed-flag-defaults.md b/.sampo/changesets/openfeature-failed-flag-defaults.md new file mode 100644 index 000000000..f9f5535b8 --- /dev/null +++ b/.sampo/changesets/openfeature-failed-flag-defaults.md @@ -0,0 +1,5 @@ +--- +pypi/openfeature-provider-posthog: patch +--- + +Document failed-record default handling with the updated PostHog SDK: a server-marked failed flag supplies no result, so the provider's existing FlagNotFoundError mapping lets OpenFeature return the caller's default instead of treating the failed record as a healthy disabled flag. This behavior requires upgrading the installed posthog dependency; the provider's compatible dependency range is unchanged and this patch intent does not force that upgrade. diff --git a/posthog/client.py b/posthog/client.py index 73181a82c..f4b6591fc 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -666,18 +666,19 @@ def _capture_flag_events( Every read that sends flag events comes here, on both clients: the single-flag getters and the reads of an ``evaluate_flags()`` snapshot. Both events are built - from ``details``, the parsed ``/flags`` record that supplied the value, and the - call's ``properties``; the flag is never evaluated again. + from ``details``, the parsed ``/flags`` record supplying the value or failure + diagnostics, and the call's ``properties``; the flag is never evaluated again. ``$feature_flag_called`` is sent if the (distinct_id, flag, response, groups) tuple hasn't already been reported on this client. Group context is included so that group-scoped flags fire a separate event for each group a user is evaluated - under. A config version 2 record that supplied the reported value + under. A config version 2 record supplying the value or failed-record diagnostics (``value_from_details``) adds its attribution to the event and to that tuple, so equal-valued A/A arms, successive experiments of one flag, and holdouts are each reported. Its reason code replaces the human-readable ``$feature_flag_reason``. - A config version 1 event and tuple are unchanged, and so are those of a read - whose value the record did not supply. + A failed record keeps its config version and reason code, but no rule tuple or + experiment exposure. Config version 1 events and reads reporting a successful + caller override or stale cached value keep their existing attribution. ``has_experiment`` is the server-reported signal for whether the flag is linked to an experiment (``None`` when the server did not report it). When the @@ -710,9 +711,8 @@ def _capture_flag_events( groups_key = ( tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () ) - # Only a record that supplied the reported value attributes it. A caller's - # match_value and a stale cached value are reported as before, with no config - # version 2 properties, even when a config version 2 record came with them. + # A failed record still reports its config version and reason code. A successful + # caller override or stale cached value has no config version 2 attribution. attribution = _rule_attribution(details) if value_from_details else None # What both config version 2 events report about the read itself. Each event # pins it, so a super property cannot make the pair disagree. @@ -5030,10 +5030,10 @@ def _get_remote_flag_result( self.log.debug( f"Successfully computed flag remotely: #{key} -> #{read.result}" ) - # A payload lookup for a caller's match_value reports that value, not - # the record's. A failure above falls back to a stale cached value. - read.value_from_details = ( - read.details is not None and override_match_value is None + # A successful payload override reports the caller's value. A failed + # record keeps its config version and reason code, never the override. + read.value_from_details = read.details is not None and ( + override_match_value is None or read.details._server_marked_failed ) except QuotaLimitError as e: self.log.warning(f"[FEATURE FLAGS] Quota limit exceeded: {e}") @@ -5096,6 +5096,8 @@ def _remote_flag_result_error(errors_while_computing, flag_details): errors.append(FeatureFlagError.ERRORS_WHILE_COMPUTING) if flag_details is None: errors.append(FeatureFlagError.FLAG_MISSING) + elif flag_details._server_marked_failed: + errors.append(FeatureFlagError.EVALUATION_FAILED) return ",".join(errors) if errors else None def _capture_flag_read_result( @@ -5190,7 +5192,9 @@ def get_feature_flag_result( device_id: The device ID for this request. Returns: - Optional[FeatureFlagResult]: The feature flag result or None if disabled/not found. + Optional[FeatureFlagResult]: The evaluated result, including a healthy + disabled flag, or None when no result is available (for example, + a missing flag or a server-marked failure without a successful fallback). """ return self._get_feature_flag_result( key, @@ -5439,9 +5443,8 @@ def _capture_feature_flag_called( local_flag_id: Optional[int] = None, local_flag_version: Optional[int] = None, *, - # Keyword-only, and false by default: a caller that does not say the record - # supplied the value reports no config version 2 attribution and sends no - # $experiment_exposure. + # Opt in when the record supplies the value or failed-record diagnostics. + # Failed records keep config version and reason code, but send no exposure. value_from_details: bool = False, ): properties: dict[str, Any] = { @@ -5511,9 +5514,9 @@ def _capture_feature_flag_called_if_needed( Shared by the single-flag evaluation path and the reads of a ``FeatureFlagEvaluations`` snapshot, so every read reports and dedupes identically. ``details`` is the parsed ``/flags`` record of the read, or - ``None`` for a locally evaluated flag. ``value_from_details`` says whether that - record supplied the reported value; only then does the read report config - version 2 attribution or send an exposure. + ``None`` for a locally evaluated flag. ``value_from_details`` includes a + failed record supplying config-version and reason-code diagnostics despite + having no value. Only a successful experiment split sends an exposure. """ _capture_flag_events( self.capture, @@ -5725,7 +5728,32 @@ def get_all_flags_and_payloads( flag_keys_to_evaluate=flag_keys_to_evaluate, device_id=device_id, ) - response = to_flags_and_payloads(decide_response) + remote_response = to_flags_and_payloads(decide_response) + # A key the server marked failed keeps its locally computed value + # and payload. Successful remote records replace local results. + failed_keys = { + key + for key, detail in decide_response.get("flags", {}).items() + if detail._server_marked_failed + } + response = { + "featureFlags": { + **{ + k: v + for k, v in (response["featureFlags"] or {}).items() + if k in failed_keys + }, + **(remote_response["featureFlags"] or {}), + }, + "featureFlagPayloads": { + **{ + k: v + for k, v in (response["featureFlagPayloads"] or {}).items() + if k in failed_keys + }, + **(remote_response["featureFlagPayloads"] or {}), + }, + } except Exception as e: self.log.exception( f"[FEATURE FLAGS] Unable to get feature flags and payloads: {e}" @@ -5930,6 +5958,10 @@ def _evaluate_flags_with_context( key: reason for key, reason in unresolved.items() if key not in result.records + or ( + isinstance(record := result.records[key], _RemoteFlagRecord) + and record.details._server_marked_failed + ) }, ) diff --git a/posthog/feature_flag_evaluations.py b/posthog/feature_flag_evaluations.py index eb4600174..870680c6f 100644 --- a/posthog/feature_flag_evaluations.py +++ b/posthog/feature_flag_evaluations.py @@ -10,6 +10,7 @@ from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union from .types import FeatureFlag as _FeatureFlag +from .types import FeatureFlagError as _FeatureFlagError from .types import FlagMetadata as _FlagMetadata from .types import FlagValue, UnresolvedFlagReason, _parse_flag_payload @@ -152,7 +153,13 @@ def __init__( """Internal — instances are created by the SDK via ``Client.evaluate_flags()``.""" self._host = host self._distinct_id = distinct_id - self._flags = flags + self._failed_flags = { + key: flag + for key, flag in flags.items() + if isinstance(flag, _RemoteFlagRecord) + and flag.details._server_marked_failed + } + self._flags = {k: v for k, v in flags.items() if k not in self._failed_flags} self._groups: Dict[str, Union[str, int]] = dict(groups or {}) self._disable_geoip = disable_geoip self._request_id = request_id @@ -181,9 +188,10 @@ def is_enabled( Args: key: The feature flag key. default_value: Returned when the flag has no value in this evaluation — - the key was not part of the evaluated set, or the evaluation itself - came back empty (failed ``/flags`` request, quota limit, or no - resolvable ``distinct_id``). Defaults to ``False``. + the key was not part of the evaluated set, the server marked its + record failed, or the evaluation itself came back empty (failed + ``/flags`` request, quota limit, or no resolvable ``distinct_id``). + Defaults to ``False``. send_feature_flag_events: Whether this read sends ``$feature_flag_called``. When ``False``, no event is sent and the dedupe cache is left alone, but the key still counts as accessed for :meth:`only_accessed`. @@ -202,7 +210,8 @@ def get_flag( Returns the variant string for multivariate flags, ``True`` for enabled flags without a variant, ``False`` for disabled flags, and ``None`` for flags that - were not returned by the evaluation. + were not returned or whose remote record was marked failed by the server. + An unreadable remote record retains its disabled (``False``) result. Pass ``send_feature_flag_events=False`` to skip the event for this read, as with :meth:`is_enabled`. @@ -229,13 +238,23 @@ def only_accessed(self) -> "FeatureFlagEvaluations": Order-dependent: if nothing has been accessed yet, the returned snapshot is empty. The method honors its name — pre-access if you want a populated result. + Accessed failures retain private diagnostics, not values or enrichment. """ filtered = {k: self._flags[k] for k in self._accessed if k in self._flags} + filtered.update( + { + k: self._failed_flags[k] + for k in self._accessed + if k in self._failed_flags + } + ) return self._clone_with(filtered) def only(self, keys: List[str]) -> "FeatureFlagEvaluations": """Return a filtered copy containing only flags with the given keys. Keys that are not present in the evaluation are dropped and logged as a warning. + Selected failures retain private diagnostics but remain absent from values, + keys and enrichment; their existing missing-value warning is unchanged. """ filtered: Dict[str, _EvaluatedFlagRecord] = {} missing: List[str] = [] @@ -250,6 +269,9 @@ def only(self, keys: List[str]) -> "FeatureFlagEvaluations": "FeatureFlagEvaluations.only() was called with flag keys that are not in the " f"evaluation set and will be dropped: {', '.join(missing)}" ) + filtered.update( + {k: self._failed_flags[k] for k in keys if k in self._failed_flags} + ) return self._clone_with(filtered) @property @@ -265,8 +287,9 @@ def unresolved_flags(self) -> Mapping[str, UnresolvedFlagReason]: A flag is listed when it has a loaded local definition, was in the requested scope, was inconclusive locally, and no remote value filled the gap. Listed flags are not part of :attr:`keys`, and reading one reports the - ``local_evaluation_inconclusive`` error instead of ``flag_missing``. Reading - this mapping does not count as an access and does not fire any event. + ``local_evaluation_inconclusive`` error instead of ``flag_missing``. + A server-marked failed fallback additionally reports ``evaluation_failed``. + Reading this mapping does not count as an access and does not fire any event. Snapshots returned by :meth:`only` and :meth:`only_accessed`, and by ``AsyncClient.evaluate_flags()``, which evaluates only remotely, carry no entries. @@ -337,8 +360,8 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: if not self._distinct_id: return - flag = self._flags.get(key) - response = _flag_response(flag) + flag = self._flags.get(key) or self._failed_flags.get(key) + response = None if key in self._failed_flags else _flag_response(flag) properties = self._get_flag_called_properties(key, flag, response) self._host.capture_flag_called_event_if_needed( @@ -350,8 +373,8 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: properties=properties, has_experiment=flag.has_experiment if flag else None, minimal_flag_called_events=self._minimal_flag_called_events, - # The record that supplied the value. A remotely evaluated experiment - # split also sends $experiment_exposure, whatever this read returned. + # A failed record still supplies config-version and reason-code diagnostics. + # Only a successful experiment split also sends $experiment_exposure. details=flag.details if isinstance(flag, _RemoteFlagRecord) else None, value_from_details=True, ) @@ -381,19 +404,19 @@ def _get_flag_called_properties(self, key, flag, response) -> Dict[str, Any]: def _get_access_errors(self, key, flag) -> List[str]: # Build the comma-joined `$feature_flag_error` matching the single-flag path's # granularity: response-level errors (errors-while-computing, quota-limited) are - # combined with per-flag errors (flag-missing or local-evaluation-inconclusive) - # so consumers can filter by type. + # combined with per-flag errors. A failed remote fallback must not hide the + # original snapshot's local-inconclusive diagnostic. errors: List[str] = [] if self._errors_while_computing: errors.append("errors_while_computing_flags") if self._quota_limited: errors.append("quota_limited") - if flag is None: - errors.append( - "local_evaluation_inconclusive" - if key in self._unresolved_flags - else "flag_missing" - ) + if key in self._failed_flags: + errors.append(_FeatureFlagError.EVALUATION_FAILED) + if key in self._unresolved_flags: + errors.append(_FeatureFlagError.LOCAL_EVALUATION_INCONCLUSIVE) + elif flag is None: + errors.append("flag_missing") return errors diff --git a/posthog/test/test_evaluate_flags.py b/posthog/test/test_evaluate_flags.py index bb76ed85a..2546c08f9 100644 --- a/posthog/test/test_evaluate_flags.py +++ b/posthog/test/test_evaluate_flags.py @@ -885,19 +885,93 @@ def test_dependency_resolved_locally_picks_its_condition(self, patch_flags): self.assertEqual(flags.get_flag("target"), "a") patch_flags.assert_not_called() + @parameterized.expand([(True,), (False,)]) @mock.patch("posthog.client.flags") - def test_flag_resolved_by_remote_fallback_is_not_unresolved(self, patch_flags): + def test_flag_resolved_by_remote_fallback_is_not_unresolved( + self, enabled, patch_flags + ): patch_flags.return_value = { - "flags": {"checkout": {"enabled": True, "variant": None}} + "flags": {"checkout": {"enabled": enabled, "variant": None}} } flags = self.client.evaluate_flags("user-123") - self.assertIs(flags.get_flag("checkout"), True) + self.assertIs(flags.get_flag("checkout"), enabled) self.assertEqual(sorted(flags.keys), ["beta-ui", "checkout"]) self.assertEqual(dict(flags.unresolved_flags), {}) patch_flags.assert_called_once() + @mock.patch("posthog.client.flags") + def test_malformed_remote_fallback_is_not_unresolved(self, patch_flags): + patch_flags.return_value = { + "flags": { + "checkout": { + "enabled": True, + "metadata": {"version": True}, + } + } + } + + flags = self.client.evaluate_flags("user-123") + + self.assertIs(flags.get_flag("checkout", send_feature_flag_events=False), False) + self.assertEqual(sorted(flags.keys), ["beta-ui", "checkout"]) + self.assertEqual(dict(flags.unresolved_flags), {}) + patch_flags.assert_called_once() + + @parameterized.expand([(None,), (False,), (True,)]) + @mock.patch("posthog.client.flags") + @mock.patch.object(Client, "capture") + def test_failed_remote_fallback_preserves_unresolved_reason( + self, error_bit, patch_capture, patch_flags + ): + response = { + "flags": { + key: { + "enabled": False, + "failed": True, + "metadata": {"payload": '"placeholder"'}, + } + for key in ("checkout", "remote-only") + } + } + if error_bit is not None: + response["errorsWhileComputingFlags"] = error_bit + patch_flags.return_value = response + + flags = self.client.evaluate_flags("user-123") + + self.assertEqual( + dict(flags.unresolved_flags), + {"checkout": UnresolvedFlagReason.EXPERIENCE_CONTINUITY}, + ) + self.assertEqual(flags.keys, ["beta-ui"]) + self.assertIsNone(flags.get_flag("checkout", send_feature_flag_events=False)) + self.assertTrue( + flags.is_enabled("checkout", True, send_feature_flag_events=False) + ) + self.assertIsNone(flags.get_flag_payload("checkout")) + self.assertNotIn("$feature/checkout", flags._get_event_properties()) + self.assertEqual(dict(flags.only(["checkout"]).unresolved_flags), {}) + self.assertEqual(dict(flags.only_accessed().unresolved_flags), {}) + patch_capture.assert_not_called() + flags.is_enabled("checkout", True) + flags.get_flag("checkout") + flags.get_flag("remote-only") + properties = _flag_called_properties(patch_capture) + expected_errors = "evaluation_failed,local_evaluation_inconclusive" + remote_errors = "evaluation_failed" + if error_bit: + expected_errors = "errors_while_computing_flags," + expected_errors + remote_errors = "errors_while_computing_flags," + remote_errors + self.assertEqual(properties["checkout"]["$feature_flag_error"], expected_errors) + self.assertIsNone(properties["checkout"]["$feature_flag_response"]) + self.assertEqual( + properties["remote-only"]["$feature_flag_error"], remote_errors + ) + self.assertEqual(patch_capture.call_count, 2) + patch_flags.assert_called_once() + @parameterized.expand( [ ( diff --git a/posthog/test/test_experiment_exposure.py b/posthog/test/test_experiment_exposure.py index c12152833..062f90e46 100644 --- a/posthog/test/test_experiment_exposure.py +++ b/posthog/test/test_experiment_exposure.py @@ -517,18 +517,28 @@ def _attribution_response(source: str) -> Dict[str, Any]: "source, expected", ATTRIBUTION_CASES, ids=[c[0] for c in ATTRIBUTION_CASES] ) def test_flag_called_attribution_follows_the_record(source, expected, path): - events = _read_events(path, _attribution_response(source), key="garden-layout") + response = _attribution_response(source) + failed = response["flags"]["garden-layout"].get("failed") is True + events = _read_events(path, response, key="garden-layout") [called] = _named(events, FLAG_CALLED, "garden-layout") assert _attribution(called.properties) == expected - FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + if failed: + # The vendored schema rejects the null $feature_flag_response a failed record reports. + assert called.properties["$feature_flag_response"] is None + assert set(called.properties["$feature_flag_error"].split(",")) == { + "evaluation_failed", + "errors_while_computing_flags", + } + else: + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) exposures = _named(events, EXPOSURE, "garden-layout") assert len(exposures) == (1 if "$feature_flag_variant_key" in expected else 0) # A record whose reason names no rule, or that failed, reports no rule even when it -# carries one. Neither sends an exposure. The last element says whether the record is -# a valid server outcome, which the contract's event schema then accepts. +# carries one. Neither sends an exposure. The last element says whether a successful +# record is a valid server outcome, which the contract's event schema then accepts. STRAY_RULE_CASES = [ ("no_rule_match", {}, True), ("missing_group_key", {}, True), @@ -569,7 +579,10 @@ def test_rule_fields_are_sent_only_for_reasons_that_name_a_rule( "$feature_flag_reason_code": code, } assert _named(events, EXPOSURE) == [] - if valid_outcome: + if fields.get("failed") is True: + assert called.properties["$feature_flag_response"] is None + assert called.properties["$feature_flag_error"] == "evaluation_failed" + elif valid_outcome: FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) diff --git a/posthog/test/test_failed_flag_records.py b/posthog/test/test_failed_flag_records.py new file mode 100644 index 000000000..c6ec36582 --- /dev/null +++ b/posthog/test/test_failed_flag_records.py @@ -0,0 +1,287 @@ +"""Spec 211: server failed records supply diagnostics, not evaluated values.""" + +import copy +from unittest import mock + +import pytest + +from posthog import AsyncPosthog +from posthog.client import Client +from posthog.types import normalize_flags_response + + +@pytest.fixture +def client(): + client = Client("project-key", send=False) + yield client + client.shutdown() + + +def response(shape, error_bit, failed_value="remote"): + records = {} + for key, value, failed in [("failed", failed_value, True), ("off", False, False)]: + record = { + "key": key, + "failed": failed, + "reason": {"code": "dependency_error" if failed else "flag_disabled"}, + "metadata": {"id": 1, "version": 2, "payload": '{"copy":"remote"}'}, + } + if shape == "v4": + record["value"] = value + record["metadata"]["config_version"] = 2 + else: + record.update( + enabled=bool(value), variant=value if isinstance(value, str) else None + ) + records[key] = record + result = { + "flags": records, + "featureFlags": {"failed": "mirror"}, + "featureFlagPayloads": {"failed": '"mirror"'}, + } + if error_bit is not None: + result["errorsWhileComputingFlags"] = error_bit + return result + + +@pytest.mark.parametrize("shape", ["v2", "v4"]) +@pytest.mark.parametrize("error_bit", [None, False, True]) +@pytest.mark.parametrize("failed_value", [False, "remote"]) +@pytest.mark.parametrize("local_inconclusive", [False, True]) +def test_failed_direct_results_and_events( + client, shape, error_bit, failed_value, local_inconclusive +): + if local_inconclusive: + client.feature_flags = [ + { + "key": "failed", + "active": True, + "ensure_experience_continuity": True, + "filters": {"groups": []}, + } + ] + with ( + mock.patch( + "posthog.client.flags", + side_effect=lambda *args, **kwargs: copy.deepcopy( + response(shape, error_bit, failed_value) + ), + ), + mock.patch.object(client, "capture", return_value=(True, {})) as capture, + ): + assert ( + client.get_feature_flag_result( + "failed", "user", send_feature_flag_events=False + ) + is None + ) + capture.assert_not_called() + assert client.get_feature_flag_result("failed", "user") is None + assert client.get_feature_flag("failed", "user") is None + assert client.feature_enabled("failed", "user") is None + assert ( + client.get_feature_flag_payload( + "failed", "user", match_value=True, send_feature_flag_events=True + ) + is None + ) + assert capture.call_count == 1 + props = capture.call_args.kwargs["properties"] + assert props["$feature_flag_response"] is None + assert "evaluation_failed" in props["$feature_flag_error"].split(",") + assert "flag_missing" not in props["$feature_flag_error"] + assert ("errors_while_computing_flags" in props["$feature_flag_error"]) is bool( + error_bit + ) + if shape == "v4": + assert props["$feature_flag_config_version"] == 2 + assert props["$feature_flag_reason_code"] == "dependency_error" + assert "$feature_flag_payload" not in props + assert client.get_feature_flag_result("off", "user").enabled is False + assert "evaluation_failed" not in capture.call_args.kwargs["properties"].get( + "$feature_flag_error", "" + ) + combined = client.get_all_flags_and_payloads("user") + assert combined["featureFlags"] == {"off": False} + assert "failed" not in combined["featureFlagPayloads"] + + +@pytest.mark.parametrize("shape", ["v2", "v4"]) +@pytest.mark.parametrize("error_bit", [None, False, True]) +def test_failed_snapshot_defaults_diagnostics_and_enrichment(client, shape, error_bit): + with ( + mock.patch("posthog.client.flags", return_value=response(shape, error_bit)), + mock.patch.object(client, "capture", return_value=(True, {})) as capture, + ): + snapshot = client.evaluate_flags("user") + assert ( + snapshot.is_enabled("failed", True, send_feature_flag_events=False) is True + ) + capture.assert_not_called() + assert snapshot.get_flag("failed") is None + assert snapshot.get_flag_payload("failed") is None + props = capture.call_args.kwargs["properties"] + assert props["$feature_flag_response"] is None + assert "evaluation_failed" in props["$feature_flag_error"] + assert "flag_missing" not in props["$feature_flag_error"] + assert ("errors_while_computing_flags" in props["$feature_flag_error"]) is bool( + error_bit + ) + assert snapshot.is_enabled("off", True) is False + assert snapshot.keys == ["off"] + assert snapshot._get_event_properties() == {"$feature/off": False} + + +@pytest.mark.parametrize("shape", ["v2", "v4"]) +@pytest.mark.parametrize("error_bit", [None, False, True]) +@pytest.mark.parametrize("filter_method", ["only", "only_accessed"]) +def test_filtered_snapshot_preserves_selected_failure_diagnostics( + client, shape, error_bit, filter_method +): + raw = response(shape, error_bit) + raw["requestId"] = "evaluation-request" + raw["evaluatedAt"] = 12345 + raw["flags"]["failed"]["reason"]["description"] = "Evaluation failed" + raw["flags"]["failed"]["metadata"]["has_experiment"] = True + raw["flags"]["other-failed"] = copy.deepcopy(raw["flags"]["failed"]) + raw["flags"]["other-failed"]["key"] = "other-failed" + with ( + mock.patch("posthog.client.flags", return_value=raw) as remote, + mock.patch.object(client, "_enqueue", return_value=(True, {})) as enqueue, + mock.patch.object(client.log, "warning") as warning, + ): + snapshot = client.evaluate_flags("user", groups={"company": "acme"}) + assert snapshot.get_flag_payload("other-failed") is None + if filter_method == "only": + filtered = snapshot.only(["failed", "off", "missing"]) + warning.assert_called_once() + assert warning.call_args.args[0].endswith("failed, missing") + else: + assert snapshot.only_accessed().keys == [] + snapshot.get_flag("failed", send_feature_flag_events=False) + snapshot.is_enabled("off", send_feature_flag_events=False) + filtered = snapshot.only_accessed() + warning.assert_not_called() + enqueue.assert_not_called() + assert filtered.keys == ["off"] + assert dict(filtered.unresolved_flags) == {} + assert filtered.get_flag_payload("failed") is None + assert ( + filtered.is_enabled("failed", True, send_feature_flag_events=False) is True + ) + enqueue.assert_not_called() + + assert filtered.get_flag("failed") is None + assert filtered.is_enabled("failed") is False + assert enqueue.call_count == 1 + called = enqueue.call_args.args[0] + assert called["event"] == "$feature_flag_called" + props = called["properties"] + assert props["$feature_flag_response"] is None + assert props["$feature_flag_error"] == ( + "errors_while_computing_flags,evaluation_failed" + if error_bit + else "evaluation_failed" + ) + assert props["$feature_flag_id"] == 1 + assert props["$feature_flag_version"] == 2 + assert props["$feature_flag_request_id"] == "evaluation-request" + assert props["$feature_flag_evaluated_at"] == 12345 + assert props["$groups"] == {"company": "acme"} + assert "$feature_flag_payload" not in props + if shape == "v4": + assert props["$feature_flag_config_version"] == 2 + assert props["$feature_flag_reason_code"] == "dependency_error" + else: + assert props["$feature_flag_reason"] == "Evaluation failed" + + client.capture("filtered-snapshot", distinct_id="user", flags=filtered) + enriched = enqueue.call_args.args[0] + assert enriched["event"] == "filtered-snapshot" + assert enriched["properties"]["$feature/off"] is False + assert "$feature/failed" not in enriched["properties"] + assert "$feature/other-failed" not in enriched["properties"] + assert enqueue.call_count == 2 + + assert filtered.get_flag("other-failed") is None + missing = enqueue.call_args.args[0]["properties"] + assert missing["$feature_flag_error"] == ( + "errors_while_computing_flags,flag_missing" if error_bit else "flag_missing" + ) + assert "$feature_flag_id" not in missing + assert "$feature_flag_reason_code" not in missing + assert enqueue.call_count == 3 + remote.assert_called_once() + + +@pytest.mark.parametrize("shape", ["v2", "v4"]) +@pytest.mark.parametrize("error_bit", [None, False, True]) +def test_failed_fallback_preserves_only_successful_local_pairs( + client, shape, error_bit +): + local = { + "featureFlags": { + "on": True, + "off": False, + "variant": "blue", + "success": True, + "local-only": True, + "malformed": True, + }, + "featureFlagPayloads": { + k: '{"copy":"local"}' + for k in ["on", "off", "variant", "success", "local-only", "malformed"] + }, + } + remote = response(shape, error_bit) + for key in ["on", "off", "variant"]: + remote["flags"][key] = copy.deepcopy(remote["flags"]["failed"]) + remote["flags"][key]["key"] = key + remote["flags"]["malformed"] = copy.deepcopy(remote["flags"]["failed"]) + remote["flags"]["malformed"]["key"] = "malformed" + remote["flags"]["malformed"]["metadata"]["version"] = True + remote["flags"]["success"] = { + "key": "success", + "enabled": True, + "variant": "remote", + "metadata": {"payload": '{"copy":"success"}'}, + } + with ( + mock.patch.object( + client, + "_get_all_flags_and_payloads_locally", + return_value=(local, True, {}), + ), + mock.patch.object( + client, "_get_flags_decision", return_value=normalize_flags_response(remote) + ), + ): + result = client.get_all_flags_and_payloads("user") + assert result["featureFlags"] == { + "on": True, + "off": False, + "variant": "blue", + "success": "remote", + "malformed": False, + } + assert result["featureFlagPayloads"] == { + k: '{"copy":"success"}' if k == "success" else '{"copy":"local"}' + for k in ["on", "off", "variant", "success"] + } + + +async def test_failed_async_snapshot_honors_default(): + client = AsyncPosthog("project-key", send=False) + try: + with mock.patch( + "posthog.async_client._async_flags", + new=mock.AsyncMock(return_value=response("v4", False)), + ): + snapshot = await client.evaluate_flags("user") + assert ( + snapshot.is_enabled("failed", True, send_feature_flag_events=False) is True + ) + assert snapshot.get_flag("failed", send_feature_flag_events=False) is None + assert snapshot.keys == ["off"] + finally: + await client.shutdown() diff --git a/posthog/test/test_flags_released_behaviour.py b/posthog/test/test_flags_released_behaviour.py index 7f5dd12b9..d646e076e 100644 --- a/posthog/test/test_flags_released_behaviour.py +++ b/posthog/test/test_flags_released_behaviour.py @@ -38,9 +38,8 @@ def test_every_legacy_getter_and_event_matches_7_82_1(request, results): assert (sparse["id"], sparse["version"]) == (0, 0) sparse["id"] = sparse["version"] = None - # A failed record reads as disabled, so it has no payload, even when the record - # carries one. 7.82.1 returned that payload from the payload reads and sent it on - # the flag's $feature_flag_called events. + # An explicit failed record has no evaluated value or payload. 7.82.1 treated + # it as disabled, included it in bulk results, and returned its payload. failed = expected["get_flags_decision"]["flags"]["server-failed"]["metadata"] assert failed["payload"] == '{"columns": 2}' failed["payload"] = None @@ -48,7 +47,15 @@ def test_every_legacy_getter_and_event_matches_7_82_1(request, results): assert expected[f"{read}:server-failed"] == {"columns": 2} expected[f"{read}:server-failed"] = None assert expected["get_feature_flag_result:server-failed"][2] == {"columns": 2} - expected["get_feature_flag_result:server-failed"][2] = None + expected["get_feature_flag_result:server-failed"] = None + for read in ["get_feature_flag", "feature_enabled", "snapshot.get_flag"]: + assert expected[f"{read}:server-failed"] is False + expected[f"{read}:server-failed"] = None + for read in ["get_feature_variants", "get_all_flags"]: + assert expected[read].pop("server-failed") is False + for read in ["get_feature_flags_and_payloads", "get_all_flags_and_payloads"]: + assert expected[read]["featureFlags"].pop("server-failed") is False + expected["snapshot.keys"].remove("server-failed") failed_events = [ properties for _, properties in expected["events"] @@ -57,6 +64,13 @@ def test_every_legacy_getter_and_event_matches_7_82_1(request, results): assert failed_events for properties in failed_events: assert properties.pop("$feature_flag_payload") == {"columns": 2} + assert properties["$feature_flag_response"] is False + properties["$feature_flag_response"] = None + properties["$feature/server-failed"] = None + properties["$feature_flag_error"] = "evaluation_failed" + for name, properties in expected["events"]: + if name in ["snapshot-event", "fetched-event"]: + assert properties.pop("$feature/server-failed") is False if request.node.callspec.params["results"] == "v4": # A $feature_flag_called for a config version 2 record carries its config @@ -108,12 +122,16 @@ def test_fallback_cache_histories_match_7_82_1(results, name): # unknown_error, no request details, and the cached value. expected = copy.deepcopy(RELEASED["cache histories"][name]) if name == "good, server failed, timeout": - # A failed record has no payload, even when the record carries one, so - # neither its read nor the cached result read after the timeout returns - # one, and their events leave it out. 7.82.1 returned it from both. - for result in expected["results"][1:]: - assert result[2] == {"theme": "dark"} - result[2] = None - for _, properties in expected["events"][1:]: - assert properties.pop("$feature_flag_payload") == {"theme": "dark"} + # Explicit failure supplies no result and cannot poison the last good + # cache entry. The later timeout still returns that successful pair. + expected["results"][1] = None + expected["results"][2] = copy.deepcopy(expected["results"][0]) + failed = expected["events"][1][1] + assert failed.pop("$feature_flag_payload") == {"theme": "dark"} + failed["$feature_flag_response"] = None + failed["$feature/cached"] = None + failed["$feature_flag_error"] = "evaluation_failed" + timeout = expected["events"][2][1] + timeout["$feature_flag_response"] = "blue" + timeout["$feature/cached"] = "blue" assert results["cache histories"][name] == expected diff --git a/posthog/test/test_flags_v4_response.py b/posthog/test/test_flags_v4_response.py index 352c793aa..c730f54f5 100644 --- a/posthog/test/test_flags_v4_response.py +++ b/posthog/test/test_flags_v4_response.py @@ -211,8 +211,8 @@ def test_a_failed_record_never_renders_its_value(client): response["flags"]["garden-layout"]["value"] = "compact" snapshot = _evaluate(client, response) - assert snapshot.get_flag("garden-layout") is False - assert snapshot.is_enabled("garden-layout", True) is False + assert snapshot.get_flag("garden-layout") is None + assert snapshot.is_enabled("garden-layout", True) is True @pytest.mark.parametrize( @@ -764,7 +764,7 @@ def _failed_with_payload(shape: str) -> Dict[str, Any]: @pytest.mark.parametrize("shape", ["v4", "older"]) def test_a_failed_record_has_no_payload_on_any_read(shape, client): - # A failed flag reads as disabled, so a payload on it would surprise the caller. + # A failed flag has no evaluated result or payload. response = _failed_with_payload(shape) [flag] = normalize_flags_response(copy.deepcopy(response))["flags"].values() assert flag._failed is True @@ -772,7 +772,7 @@ def test_a_failed_record_has_no_payload_on_any_read(shape, client): with mock.patch("posthog.client.flags", return_value=copy.deepcopy(response)): result = client.get_feature_flag_result("garden-layout", DISTINCT_ID) - assert result is not None and result.payload is None + assert result is None assert client.get_feature_flag_payload("garden-layout", DISTINCT_ID) is None assert ( client.get_feature_flag_payload( @@ -951,7 +951,8 @@ def test_v1_flags_render_the_same_legacy_values_from_both_shapes(case): "flags": v4_flags, } expected = { - "featureFlags": v1_maps["featureFlags"], + # Flat legacy maps retain their interpretation, but detailed failures do not. + "featureFlags": {} if outcome["failed"] else v1_maps["featureFlags"], "featureFlagPayloads": { key: json.loads(value) for key, value in v1_maps["featureFlagPayloads"].items() diff --git a/posthog/types.py b/posthog/types.py index 625b5dc5a..9335274e5 100644 --- a/posthog/types.py +++ b/posthog/types.py @@ -275,6 +275,11 @@ class FeatureFlag: # server marks failed has ``_failed`` set and this ``None``. _malformed_field: ClassVar[Optional[str]] = None + @property + def _server_marked_failed(self) -> bool: + """Whether a readable record reports a server evaluation failure.""" + return self._failed and self._malformed_field is None + def __post_init__(self) -> None: object.__setattr__( self, @@ -640,12 +645,15 @@ def from_flag_details( details (Union[FeatureFlag, None]): The FeatureFlag object to convert. override_match_value (Optional[FlagValue]): If provided, this value will be used to populate the enabled and variant fields instead of the values from the FeatureFlag. + A server-marked failure returns None even when this override is provided. Returns: - FeatureFlagResult | None: A new FeatureFlagResult instance, or None if details is None. + FeatureFlagResult | None: A new result, or None when details is absent or + the server marked the record failed. Unreadable records are not + treated as server-marked failures. """ - if details is None: + if details is None or details._server_marked_failed: return None if override_match_value is not None: @@ -725,7 +733,7 @@ def to_values(response: FlagsResponse) -> Optional[dict[str, FlagValue]]: return { key: value.get_value() for key, value in flags.items() - if isinstance(value, FeatureFlag) + if isinstance(value, FeatureFlag) and not value._server_marked_failed } @@ -737,6 +745,7 @@ def to_payloads(response: FlagsResponse) -> Optional[dict[str, str]]: key: value.metadata.payload for key, value in response.get("flags", {}).items() if isinstance(value, FeatureFlag) + and not value._server_marked_failed and value.enabled and value.metadata.payload is not None } @@ -751,6 +760,7 @@ class FeatureFlagError: Error values: ERRORS_WHILE_COMPUTING: Server returned errorsWhileComputingFlags=true + EVALUATION_FAILED: Server marked the requested flag failed, with no evaluated value FLAG_MISSING: Requested flag not in API response LOCAL_EVALUATION_INCONCLUSIVE: Requested flag has a loaded definition but local evaluation was inconclusive and no remote value filled the gap @@ -764,6 +774,7 @@ class FeatureFlagError: """ ERRORS_WHILE_COMPUTING = "errors_while_computing_flags" + EVALUATION_FAILED = "evaluation_failed" FLAG_MISSING = "flag_missing" LOCAL_EVALUATION_INCONCLUSIVE = "local_evaluation_inconclusive" QUOTA_LIMITED = "quota_limited" diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index d0037bb5f..4b51f85aa 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -1216,6 +1216,7 @@ attribute posthog.types.FeatureFlag.reason: Optional[FlagReason] attribute posthog.types.FeatureFlag.variant: Optional[str] attribute posthog.types.FeatureFlagError.CONNECTION_ERROR = 'connection_error' attribute posthog.types.FeatureFlagError.ERRORS_WHILE_COMPUTING = 'errors_while_computing_flags' +attribute posthog.types.FeatureFlagError.EVALUATION_FAILED = 'evaluation_failed' attribute posthog.types.FeatureFlagError.FLAG_MISSING = 'flag_missing' attribute posthog.types.FeatureFlagError.LOCAL_EVALUATION_INCONCLUSIVE = 'local_evaluation_inconclusive' attribute posthog.types.FeatureFlagError.QUOTA_LIMITED = 'quota_limited' diff --git a/sdk_compliance_adapter/CONTRIBUTING.md b/sdk_compliance_adapter/CONTRIBUTING.md index 8514a342c..22b26b50a 100644 --- a/sdk_compliance_adapter/CONTRIBUTING.md +++ b/sdk_compliance_adapter/CONTRIBUTING.md @@ -4,7 +4,7 @@ This package contains the PostHog Python SDK compliance adapter used with the Po ## Running tests -Tests run automatically in CI via GitHub Actions against harness **1.1.1**, for both capture protocols. Both jobs opt into the local-evaluation suite and fail on compliance regressions. +Tests run automatically in CI via GitHub Actions against harness **1.31.1**, for both capture protocols. Both jobs opt into the local-evaluation suite and fail on compliance regressions. Run adapter protocol tests from the repository root in an activated virtual environment: @@ -44,7 +44,7 @@ docker run -d --name sdk-adapter --network test-network -p 8080:8080 posthog-pyt docker run --rm \ --name test-harness \ --network test-network \ - ghcr.io/posthog/sdk-test-harness:1.28.0 \ + ghcr.io/posthog/sdk-test-harness:1.31.1 \ run --adapter-url http://sdk-adapter:8080 --mock-url http://test-harness:8081 # Cleanup diff --git a/sdk_compliance_adapter/docker-compose.yml b/sdk_compliance_adapter/docker-compose.yml index dcbd677de..ead39d536 100644 --- a/sdk_compliance_adapter/docker-compose.yml +++ b/sdk_compliance_adapter/docker-compose.yml @@ -23,7 +23,7 @@ services: # Test harness test-harness: - image: ghcr.io/posthog/sdk-test-harness:1.28.0 + image: ghcr.io/posthog/sdk-test-harness:1.31.1 command: ["run", "--adapter-url", "http://sdk-adapter:8080", "--mock-url", "http://test-harness:8081"] networks: - test-network