From ee958111aa664b4d46be5de4b95e10ac1c390c2e Mon Sep 17 00:00:00 2001 From: Cal Courtney Date: Thu, 9 Jul 2026 13:57:46 +0100 Subject: [PATCH 1/5] docs: add troubleshooting section for common SRM causes and SDK mistakes Implements FT-1878. Adds a Troubleshooting category (position 11) under SDK-Documentation with three pages: - srm-and-exposure-issues.mdx: conditional exposure, audience filtering before treatment(), and not awaiting publish() before navigation - goal-undercounting.mdx: goal tracked before exposure, and metric time/property filter misconfiguration - sdk-context-best-practices.mdx: restructured as troubleshooting entries (symptom -> cause -> fix) covering multiple-context assignment conflicts and identity leakage after login/logout Review fixes applied: corrected JS SDK method name from getTreatment() to treatment() throughout (the existing docs and shared snippets use treatment() for JS; getTreatment is the Java/Go/.NET name), rewrote the conditional exposure wrong examples to actually demonstrate the one-path-skips-treatment pattern rather than showing a track-placement bug, fixed identical Python/Java publish examples to show the async variant as the wrong pattern, fixed a wrong experiment name in the Java goal example, and applied consistent style to match the existing SDK docs conventions. Co-Authored-By: Claude Fable 5 --- .../troubleshooting/_category_.json | 6 + .../troubleshooting/goal-undercounting.mdx | 102 +++++++ .../sdk-context-best-practices.mdx | 111 +++++++ .../srm-and-exposure-issues.mdx | 276 ++++++++++++++++++ 4 files changed, 495 insertions(+) create mode 100644 docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/_category_.json create mode 100644 docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx create mode 100644 docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx create mode 100644 docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/srm-and-exposure-issues.mdx diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/_category_.json b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/_category_.json new file mode 100644 index 00000000..e5e03278 --- /dev/null +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/_category_.json @@ -0,0 +1,6 @@ +{ + "label": "Troubleshooting", + "position": 11, + "collapsible": true, + "collapsed": true +} diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx new file mode 100644 index 00000000..b42d9238 --- /dev/null +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx @@ -0,0 +1,102 @@ +--- +sidebar_position: 2 +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Goal undercounting + +Your experiment results show fewer goal events than you expect, or goals that should be common look rare. The usual causes are ordering (the goal fires before exposure) and metric misconfiguration (the time or property filters on the metric exclude the events you wanted to count). + +## Goal tracked before exposure + +A goal event must be recorded after the exposure for the SDK to attribute it to the experiment. When the goal fires first, the SDK has no exposure to tie it to, and the event is not counted against the experiment. + +The most common version of this is on a page where the user lands and the goal event (a page view, a purchase, a sign-up) fires before your experiment code calls `treatment()`. + +:::tip Rule of thumb +Call `treatment()` before anything that can produce a goal event. If the goal is a click on a button, the exposure must have happened by the time the button renders. +::: + + + + +**Wrong** - the goal event fires before exposure: + +```js +analytics.track("checkout_complete", { orderId: 123 }); +// Too late: the goal already fired. +const variant = context.treatment("checkout_flow"); +``` + +**Right** - expose first, then track the goal: + +```js +const variant = context.treatment("checkout_flow"); +// Now the SDK has recorded the exposure. +context.track("checkout_complete"); +``` + + + + +**Wrong** - the goal event fires before exposure: + +```python +analytics.track("checkout_complete", {"orderId": 123}) +# Too late: the goal already fired. +variant = context.get_treatment("checkout_flow") +``` + +**Right** - expose first, then track the goal: + +```python +variant = context.get_treatment("checkout_flow") +# Now the SDK has recorded the exposure. +context.track("checkout_complete") +``` + + + + +**Wrong** - the goal event fires before exposure: + +```java +analytics.track("checkout_complete", Map.of("orderId", 123)); +// Too late: the goal already exposed. +int variant = context.getTreatment("checkout_flow"); +``` + +**Right** - expose first, then track the goal: + +```java +int variant = context.getTreatment("checkout_flow"); +// Now the SDK has recorded the exposure. +context.track("checkout_complete"); +``` + + + + +## Metric time or property filters misconfigured + +A metric is more than a goal name. When you configure a metric in the web console, you can restrict which events count by time window (relative to exposure) or by event properties. If those filters are wrong, events that you expect to count are excluded. + +### Time window too narrow + +A purchase goal with a 1-hour window will not count purchases that happen the next day. If your conversion cycle is longer than the window, undercounting is guaranteed. + +### Property filter excludes valid events + +A goal configured to only count events where `plan == "pro"` will miss every `plan == "enterprise"` conversion. The goal event fires, but the metric filter discards it. + +### How to diagnose + +1. Open the metric in the web console and check the time window and property filters against the events you expect to count. +2. Look at the raw event stream for a user you know converted. If the event is present but the metric does not count it, the filter is the cause. +3. Widen the time window or relax the property filter and re-check. + +:::info +Metric configuration lives in the web console, not the SDK. The SDK only sends the goal event; the console decides which events count. See the [metrics documentation](/docs/web-console-docs/goals-and-metrics/metrics/overview) for details. +::: diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx new file mode 100644 index 00000000..9d6bd60c --- /dev/null +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx @@ -0,0 +1,111 @@ +--- +sidebar_position: 3 +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Common SDK mistakes + +The symptoms below show up as data quality problems in the web console: SRM warnings, assignment conflicts, or participants counted under the wrong user. Each entry describes the symptom, the cause, and the fix. + +## Symptom: SRM warning with assignment conflicts + +You see an SRM alert on the experiment dashboard, and the raw event stream shows the same user assigned to different variants across requests or page loads. + +**Cause: multiple contexts for one user.** A context is scoped to one user. If you create a new context for every treatment check, the SDK records a separate exposure each time and the user can land in different variants across requests. On a server, each request gets its own context; on a client, the context should live for the session. + +**Fix: one context per user lifecycle.** Create the context once at the start of the request or app session, reuse it for every treatment check and goal track, and finalize it before the response returns or the session ends. + + + + +**Wrong** - new context per treatment check: + +```js +export function handleRequest(req, res) { + // New context each call. Two requests for the same user + // can land in different variants. + const context = sdk.createContext({ units: { user_id: req.userId } }); + const variant = context.getTreatment("checkout_flow"); + res.render(variant === 1 ? "new" : "old"); +} +``` + +**Right** - one context per request, finalized before the response: + +```js +export async function handleRequest(req, res) { + const context = sdk.createContext({ units: { user_id: req.userId } }); + const variant = context.treatment("checkout_flow"); + res.render(variant === 1 ? "new" : "old"); + await context.publish(); + context.finalize(); +} +``` + + + + +**Wrong** - new context per treatment check: + +```python +def handle_request(request): + # New context each call. Two requests for the same user + # can land in different variants. + context = sdk.create_context({"units": {"user_id": request.user_id}}) + variant = context.get_treatment("checkout_flow") + return render(variant) +``` + +**Right** - one context per request, closed before the response: + +```python +def handle_request(request): + context = sdk.create_context({"units": {"user_id": request.user_id}}) + variant = context.get_treatment("checkout_flow") + response = render(variant) + context.publish() + context.close() + return response +``` + + + + +**Wrong** - new context per treatment check: + +```java +public String handleRequest(Request request) { + // New context each call. Two requests for the same user + // can land in different variants. + Context context = sdk.createContext(Map.of("units", Map.of("user_id", request.getUserId()))); + return render(context.getTreatment("checkout_flow")); +} +``` + +**Right** - one context per request, finalized before the response: + +```java +public String handleRequest(Request request) throws Exception { + Context context = sdk.createContext(Map.of("units", Map.of("user_id", request.getUserId()))); + int variant = context.getTreatment("checkout_flow"); + String response = render(variant); + context.publish(); + context.close(); + return response; +} +``` + + + + +## Symptom: participants counted under the wrong user after login or logout + +After a user logs out or a different user logs in, the next participant is counted under the previous user's identity. + +**Cause: context not reset on user change.** The current context belongs to the old user. Reusing it leaks the old user's units and attributes into the new session. + +**Fix: finalize the old context and create a new one.** On logout, finalize the existing context and create a new one with a fresh `anonymous_id`. On login, add the `user_id` to the existing context (or create a new one if you did not keep the anonymous session). The React SDK's `resetContext` does this for you. + +For the full lifecycle (create, publish, finalize, refresh, reset), see [Managing the context lifecycle](/docs/APIs-and-SDKs/SDK-Documentation/managing-the-context-lifecycle). diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/srm-and-exposure-issues.mdx b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/srm-and-exposure-issues.mdx new file mode 100644 index 00000000..0d55296e --- /dev/null +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/srm-and-exposure-issues.mdx @@ -0,0 +1,276 @@ +--- +sidebar_position: 1 +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; +import Publish from "../_shared/publish/_publish.mdx"; + +# SRM and exposure issues + +A [Sample Ratio Mismatch (SRM)](/docs/glossary#srm-sample-ratio-mismatch) warning means the split of participants across your variants does not match the configured allocation. ABsmartly shows an SRM alert on the experiment dashboard when the mismatch is statistically significant. See [Experiment health checks](/docs/web-console-docs/experiments/Experiment-health-checks) for how the alert surfaces. + +The mistakes below are the most common causes. Each one shows the wrong code first, then the fix. + +## Conditional exposure + +Your variants call `treatment()` (or `variableValue()`) at different points in the code path, so some users are exposed and counted in one variant but never reach the call in another. + +:::tip Rule of thumb +Call `treatment()` exactly once, in the same place in the request or page lifecycle, for every user who should be in the experiment. Branch on the value you get back, not on whether to call it. +::: + + + + +**Wrong** - `treatment()` called only on one code path: + +```js +// Only users who see the new checkout call treatment(). +if (user.featureEnabled) { + const variant = context.treatment("checkout_flow"); + if (variant === 1) { + renderNewCheckout(); + } else { + renderOldCheckout(); + } +} else { + // These users never call treatment(), so they are + // never exposed or counted in the experiment. + renderOldCheckout(); +} +``` + +**Right** - call `treatment()` once, before any branching: + +```js +const variant = context.treatment("checkout_flow"); + +// Exposure is recorded for everyone at the same point. +if (variant === 1) { + renderNewCheckout(); +} else { + renderOldCheckout(); +} +``` + + + + +**Wrong** - `treatment()` called only on one code path: + +```python +# Only users who see the new checkout call treatment(). +if user.feature_enabled: + variant = context.get_treatment("checkout_flow") + if variant == 1: + render_new_checkout() + else: + render_old_checkout() +else: + # These users never call treatment(), so they are + # never exposed or counted in the experiment. + render_old_checkout() +``` + +**Right** - call `treatment()` once, before any branching: + +```python +variant = context.get_treatment("checkout_flow") + +# Exposure is recorded for everyone at the same point. +if variant == 1: + render_new_checkout() +else: + render_old_checkout() +``` + + + + +**Wrong** - `treatment()` called only on one code path: + +```java +// Only users who see the new checkout call treatment(). +if (user.isFeatureEnabled()) { + int variant = context.getTreatment("checkout_flow"); + if (variant == 1) { + renderNewCheckout(); + } else { + renderOldCheckout(); + } +} else { + // These users never call treatment(), so they are + // never exposed or counted in the experiment. + renderOldCheckout(); +} +``` + +**Right** - `treatment()` called once, before any branching: + +```java +int variant = context.getTreatment("checkout_flow"); + +// Exposure is recorded for everyone at the same point. +if (variant == 1) { + renderNewCheckout(); +} else { + renderOldCheckout(); +} +``` + + + + +## Audience filtering before treatment + +You check an attribute in your own code and skip `treatment()` for users who should not be in the experiment. But the experiment's audience is configured in the web console for a reason: when you filter in code, the users who never call `treatment()` are invisible to the analysis. If they behave differently across variants, the remaining counts skew. + +The fix is to let the SDK apply the audience. Call `treatment()` for everyone who hits the experiment surface. The SDK returns variant 0 (control) for users outside the audience, and the exposure is recorded consistently. + + + + +**Wrong** - filtering in code before calling `treatment()`: + +```js +// You filter out mobile users yourself, so they never call treatment(). +if (context.attributes().platform === "mobile") { + return renderDefault(); +} + +const variant = context.treatment("checkout_flow"); +``` + +**Right** - call `treatment()` for everyone, let the audience do the filtering: + +```js +const variant = context.treatment("checkout_flow"); + +// The SDK returns 0 for users outside the audience. +if (variant === 1) { + renderNewCheckout(); +} else { + renderOldCheckout(); +} +``` + + + + +**Wrong** - filtering in code before calling `treatment()`: + +```python +if context.attributes().get("platform") == "mobile": + return render_default() + +variant = context.get_treatment("checkout_flow") +``` + +**Right** - call `treatment()` for everyone, let the audience do the filtering: + +```python +variant = context.get_treatment("checkout_flow") + +if variant == 1: + render_new_checkout() +else: + render_old_checkout() +``` + + + + +**Wrong** - filtering in code before calling `treatment()`: + +```java +if ("mobile".equals(context.attributes().get("platform"))) { + return renderDefault(); +} + +int variant = context.getTreatment("checkout_flow"); +``` + +**Right** - call `treatment()` for everyone, let the audience do the filtering: + +```java +int variant = context.getTreatment("checkout_flow"); + +if (variant == 1) { + renderNewCheckout(); +} else { + renderOldCheckout(); +} +``` + + + + +## Not awaiting publish before navigation + +`publish()` sends exposure and goal events to the collector. If the page navigates or the server returns a response before the request completes, events are lost. The exposure was recorded locally but never delivered, so the analysis sees fewer participants than expected and the split skews. + +Always await the promise (or call the synchronous blocking variant) before redirecting or returning the response. See [Managing the context lifecycle](/docs/APIs-and-SDKs/SDK-Documentation/managing-the-context-lifecycle) for the full picture. + + + + + + +**Wrong** - fire and forget before navigation: + +```js +context.publish(); +window.location.replace("/checkout"); +// The publish request is still in flight when the page unloads. +``` + +**Right** - await the promise: + +```js +await context.publish(); +window.location.replace("/checkout"); +``` + + + + +`publish()` is synchronous and blocks until the request completes, so the mistake below does not apply to Python. If you use `publish_async()`, you must await it before the response is returned. + +**Wrong** - starting async publish without waiting: + +```python +context.publish_async() +return redirect("/checkout") +# The publish request is still in flight when the response returns. +``` + +**Right** - await the async result: + +```python +await context.publish_async() +return redirect("/checkout") +``` + + + + +`publish()` blocks until the request completes, so the mistake below does not apply to Java. If you use `publishAsync()`, you must await the future before the response is returned. + +**Wrong** - starting async publish without waiting: + +```java +context.publishAsync(); +return redirect("/checkout"); +// The publish request is still in flight when the response returns. +``` + +**Right** - await the future: + +```java +context.publishAsync().get(); +return redirect("/checkout"); +``` + + + From c4efc607880f4c373bd86ba917abe9cda1442aaf Mon Sep 17 00:00:00 2001 From: Cal Courtney Date: Thu, 9 Jul 2026 14:37:40 +0100 Subject: [PATCH 2/5] docs: fix JS API name and publish ordering in context best practices Address CodeRabbit review on PR #277: the Wrong example used getTreatment() instead of the JS SDK's treatment(), and the Right example rendered the response before awaiting publish(), which is the exact mistake the SRM page warns against. Co-Authored-By: Claude Fable 5 --- .../troubleshooting/sdk-context-best-practices.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx index 9d6bd60c..074ed97d 100644 --- a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx @@ -27,7 +27,7 @@ export function handleRequest(req, res) { // New context each call. Two requests for the same user // can land in different variants. const context = sdk.createContext({ units: { user_id: req.userId } }); - const variant = context.getTreatment("checkout_flow"); + const variant = context.treatment("checkout_flow"); res.render(variant === 1 ? "new" : "old"); } ``` @@ -38,9 +38,9 @@ export function handleRequest(req, res) { export async function handleRequest(req, res) { const context = sdk.createContext({ units: { user_id: req.userId } }); const variant = context.treatment("checkout_flow"); - res.render(variant === 1 ? "new" : "old"); await context.publish(); context.finalize(); + res.render(variant === 1 ? "new" : "old"); } ``` From 71e52b8956953fa5d948c5993dd74f4cee601fd8 Mon Sep 17 00:00:00 2001 From: Cal Courtney Date: Thu, 9 Jul 2026 14:51:49 +0100 Subject: [PATCH 3/5] docs: simplify troubleshooting examples to TypeScript Replace language tabs with a single TypeScript example per issue. The troubleshooting pages are about SDK integration mistakes rather than language-specific APIs, and a single example keeps the pages easier to scan while avoiding partial coverage across SDKs. Co-Authored-By: Claude Fable 5 --- .../troubleshooting/goal-undercounting.mdx | 57 +---- .../sdk-context-best-practices.mdx | 70 +------ .../srm-and-exposure-issues.mdx | 197 ++---------------- 3 files changed, 23 insertions(+), 301 deletions(-) diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx index b42d9238..f7f44977 100644 --- a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx @@ -2,9 +2,6 @@ sidebar_position: 2 --- -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; - # Goal undercounting Your experiment results show fewer goal events than you expect, or goals that should be common look rare. The usual causes are ordering (the goal fires before exposure) and metric misconfiguration (the time or property filters on the metric exclude the events you wanted to count). @@ -15,16 +12,9 @@ A goal event must be recorded after the exposure for the SDK to attribute it to The most common version of this is on a page where the user lands and the goal event (a page view, a purchase, a sign-up) fires before your experiment code calls `treatment()`. -:::tip Rule of thumb -Call `treatment()` before anything that can produce a goal event. If the goal is a click on a button, the exposure must have happened by the time the button renders. -::: - - - - **Wrong** - the goal event fires before exposure: -```js +```ts analytics.track("checkout_complete", { orderId: 123 }); // Too late: the goal already fired. const variant = context.treatment("checkout_flow"); @@ -32,53 +22,12 @@ const variant = context.treatment("checkout_flow"); **Right** - expose first, then track the goal: -```js +```ts const variant = context.treatment("checkout_flow"); // Now the SDK has recorded the exposure. context.track("checkout_complete"); ``` - - - -**Wrong** - the goal event fires before exposure: - -```python -analytics.track("checkout_complete", {"orderId": 123}) -# Too late: the goal already fired. -variant = context.get_treatment("checkout_flow") -``` - -**Right** - expose first, then track the goal: - -```python -variant = context.get_treatment("checkout_flow") -# Now the SDK has recorded the exposure. -context.track("checkout_complete") -``` - - - - -**Wrong** - the goal event fires before exposure: - -```java -analytics.track("checkout_complete", Map.of("orderId", 123)); -// Too late: the goal already exposed. -int variant = context.getTreatment("checkout_flow"); -``` - -**Right** - expose first, then track the goal: - -```java -int variant = context.getTreatment("checkout_flow"); -// Now the SDK has recorded the exposure. -context.track("checkout_complete"); -``` - - - - ## Metric time or property filters misconfigured A metric is more than a goal name. When you configure a metric in the web console, you can restrict which events count by time window (relative to exposure) or by event properties. If those filters are wrong, events that you expect to count are excluded. @@ -89,7 +38,7 @@ A purchase goal with a 1-hour window will not count purchases that happen the ne ### Property filter excludes valid events -A goal configured to only count events where `plan == "pro"` will miss every `plan == "enterprise"` conversion. The goal event fires, but the metric filter discards it. +A goal configured to only count events where `plan == "pro"` will miss every `plan == "enterprise"` conversion. The filter discards it even though the event fires. ### How to diagnose diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx index 074ed97d..ecf5332b 100644 --- a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx @@ -2,9 +2,6 @@ sidebar_position: 3 --- -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; - # Common SDK mistakes The symptoms below show up as data quality problems in the web console: SRM warnings, assignment conflicts, or participants counted under the wrong user. Each entry describes the symptom, the cause, and the fix. @@ -17,13 +14,10 @@ You see an SRM alert on the experiment dashboard, and the raw event stream shows **Fix: one context per user lifecycle.** Create the context once at the start of the request or app session, reuse it for every treatment check and goal track, and finalize it before the response returns or the session ends. - - - **Wrong** - new context per treatment check: -```js -export function handleRequest(req, res) { +```ts +export function handleRequest(req: Request, res: Response) { // New context each call. Two requests for the same user // can land in different variants. const context = sdk.createContext({ units: { user_id: req.userId } }); @@ -34,8 +28,8 @@ export function handleRequest(req, res) { **Right** - one context per request, finalized before the response: -```js -export async function handleRequest(req, res) { +```ts +export async function handleRequest(req: Request, res: Response) { const context = sdk.createContext({ units: { user_id: req.userId } }); const variant = context.treatment("checkout_flow"); await context.publish(); @@ -44,62 +38,6 @@ export async function handleRequest(req, res) { } ``` - - - -**Wrong** - new context per treatment check: - -```python -def handle_request(request): - # New context each call. Two requests for the same user - # can land in different variants. - context = sdk.create_context({"units": {"user_id": request.user_id}}) - variant = context.get_treatment("checkout_flow") - return render(variant) -``` - -**Right** - one context per request, closed before the response: - -```python -def handle_request(request): - context = sdk.create_context({"units": {"user_id": request.user_id}}) - variant = context.get_treatment("checkout_flow") - response = render(variant) - context.publish() - context.close() - return response -``` - - - - -**Wrong** - new context per treatment check: - -```java -public String handleRequest(Request request) { - // New context each call. Two requests for the same user - // can land in different variants. - Context context = sdk.createContext(Map.of("units", Map.of("user_id", request.getUserId()))); - return render(context.getTreatment("checkout_flow")); -} -``` - -**Right** - one context per request, finalized before the response: - -```java -public String handleRequest(Request request) throws Exception { - Context context = sdk.createContext(Map.of("units", Map.of("user_id", request.getUserId()))); - int variant = context.getTreatment("checkout_flow"); - String response = render(variant); - context.publish(); - context.close(); - return response; -} -``` - - - - ## Symptom: participants counted under the wrong user after login or logout After a user logs out or a different user logs in, the next participant is counted under the previous user's identity. diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/srm-and-exposure-issues.mdx b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/srm-and-exposure-issues.mdx index 0d55296e..7531583b 100644 --- a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/srm-and-exposure-issues.mdx +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/srm-and-exposure-issues.mdx @@ -2,10 +2,6 @@ sidebar_position: 1 --- -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; -import Publish from "../_shared/publish/_publish.mdx"; - # SRM and exposure issues A [Sample Ratio Mismatch (SRM)](/docs/glossary#srm-sample-ratio-mismatch) warning means the split of participants across your variants does not match the configured allocation. ABsmartly shows an SRM alert on the experiment dashboard when the mismatch is statistically significant. See [Experiment health checks](/docs/web-console-docs/experiments/Experiment-health-checks) for how the alert surfaces. @@ -14,18 +10,15 @@ The mistakes below are the most common causes. Each one shows the wrong code fir ## Conditional exposure -Your variants call `treatment()` (or `variableValue()`) at different points in the code path, so some users are exposed and counted in one variant but never reach the call in another. +Your variants call `treatment()` at different points in the code path, so some users are exposed and counted in one variant but never reach the call in another. :::tip Rule of thumb Call `treatment()` exactly once, in the same place in the request or page lifecycle, for every user who should be in the experiment. Branch on the value you get back, not on whether to call it. ::: - - - **Wrong** - `treatment()` called only on one code path: -```js +```ts // Only users who see the new checkout call treatment(). if (user.featureEnabled) { const variant = context.treatment("checkout_flow"); @@ -43,7 +36,7 @@ if (user.featureEnabled) { **Right** - call `treatment()` once, before any branching: -```js +```ts const variant = context.treatment("checkout_flow"); // Exposure is recorded for everyone at the same point. @@ -54,86 +47,15 @@ if (variant === 1) { } ``` - - - -**Wrong** - `treatment()` called only on one code path: - -```python -# Only users who see the new checkout call treatment(). -if user.feature_enabled: - variant = context.get_treatment("checkout_flow") - if variant == 1: - render_new_checkout() - else: - render_old_checkout() -else: - # These users never call treatment(), so they are - # never exposed or counted in the experiment. - render_old_checkout() -``` - -**Right** - call `treatment()` once, before any branching: - -```python -variant = context.get_treatment("checkout_flow") - -# Exposure is recorded for everyone at the same point. -if variant == 1: - render_new_checkout() -else: - render_old_checkout() -``` - - - - -**Wrong** - `treatment()` called only on one code path: - -```java -// Only users who see the new checkout call treatment(). -if (user.isFeatureEnabled()) { - int variant = context.getTreatment("checkout_flow"); - if (variant == 1) { - renderNewCheckout(); - } else { - renderOldCheckout(); - } -} else { - // These users never call treatment(), so they are - // never exposed or counted in the experiment. - renderOldCheckout(); -} -``` - -**Right** - `treatment()` called once, before any branching: - -```java -int variant = context.getTreatment("checkout_flow"); - -// Exposure is recorded for everyone at the same point. -if (variant == 1) { - renderNewCheckout(); -} else { - renderOldCheckout(); -} -``` - - - - ## Audience filtering before treatment You check an attribute in your own code and skip `treatment()` for users who should not be in the experiment. But the experiment's audience is configured in the web console for a reason: when you filter in code, the users who never call `treatment()` are invisible to the analysis. If they behave differently across variants, the remaining counts skew. The fix is to let the SDK apply the audience. Call `treatment()` for everyone who hits the experiment surface. The SDK returns variant 0 (control) for users outside the audience, and the exposure is recorded consistently. - - - **Wrong** - filtering in code before calling `treatment()`: -```js +```ts // You filter out mobile users yourself, so they never call treatment(). if (context.attributes().platform === "mobile") { return renderDefault(); @@ -144,7 +66,7 @@ const variant = context.treatment("checkout_flow"); **Right** - call `treatment()` for everyone, let the audience do the filtering: -```js +```ts const variant = context.treatment("checkout_flow"); // The SDK returns 0 for users outside the audience. @@ -155,71 +77,15 @@ if (variant === 1) { } ``` - - - -**Wrong** - filtering in code before calling `treatment()`: - -```python -if context.attributes().get("platform") == "mobile": - return render_default() - -variant = context.get_treatment("checkout_flow") -``` - -**Right** - call `treatment()` for everyone, let the audience do the filtering: - -```python -variant = context.get_treatment("checkout_flow") - -if variant == 1: - render_new_checkout() -else: - render_old_checkout() -``` - - - - -**Wrong** - filtering in code before calling `treatment()`: - -```java -if ("mobile".equals(context.attributes().get("platform"))) { - return renderDefault(); -} - -int variant = context.getTreatment("checkout_flow"); -``` - -**Right** - call `treatment()` for everyone, let the audience do the filtering: - -```java -int variant = context.getTreatment("checkout_flow"); - -if (variant == 1) { - renderNewCheckout(); -} else { - renderOldCheckout(); -} -``` - - - - ## Not awaiting publish before navigation `publish()` sends exposure and goal events to the collector. If the page navigates or the server returns a response before the request completes, events are lost. The exposure was recorded locally but never delivered, so the analysis sees fewer participants than expected and the split skews. Always await the promise (or call the synchronous blocking variant) before redirecting or returning the response. See [Managing the context lifecycle](/docs/APIs-and-SDKs/SDK-Documentation/managing-the-context-lifecycle) for the full picture. - - - - - **Wrong** - fire and forget before navigation: -```js +```ts context.publish(); window.location.replace("/checkout"); // The publish request is still in flight when the page unloads. @@ -227,50 +93,19 @@ window.location.replace("/checkout"); **Right** - await the promise: -```js +```ts await context.publish(); window.location.replace("/checkout"); ``` - - - -`publish()` is synchronous and blocks until the request completes, so the mistake below does not apply to Python. If you use `publish_async()`, you must await it before the response is returned. - -**Wrong** - starting async publish without waiting: - -```python -context.publish_async() -return redirect("/checkout") -# The publish request is still in flight when the response returns. -``` - -**Right** - await the async result: - -```python -await context.publish_async() -return redirect("/checkout") -``` +On the server, call `publish()` before sending the response: - - - -`publish()` blocks until the request completes, so the mistake below does not apply to Java. If you use `publishAsync()`, you must await the future before the response is returned. - -**Wrong** - starting async publish without waiting: - -```java -context.publishAsync(); -return redirect("/checkout"); -// The publish request is still in flight when the response returns. -``` - -**Right** - await the future: - -```java -context.publishAsync().get(); -return redirect("/checkout"); +```ts +export async function handleRequest(req: Request, res: Response) { + const context = sdk.createContext({ units: { user_id: req.userId } }); + const variant = context.treatment("checkout_flow"); + await context.publish(); + context.finalize(); + res.render(variant === 1 ? "new" : "old"); +} ``` - - - From cf054d41992b89a301093c0a3fa7dde6caa6657e Mon Sep 17 00:00:00 2001 From: Cal Courtney Date: Thu, 9 Jul 2026 14:54:34 +0100 Subject: [PATCH 4/5] docs: link Events > Explore and Metrics from goal undercounting diagnostics Point the diagnostic steps at the actual console pages: Events > Explore for inspecting the raw event stream, and the metrics overview for how time windows and property filters are configured. Co-Authored-By: Claude Fable 5 --- .../troubleshooting/goal-undercounting.mdx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx index f7f44977..bee6a02f 100644 --- a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/goal-undercounting.mdx @@ -42,10 +42,6 @@ A goal configured to only count events where `plan == "pro"` will miss every `pl ### How to diagnose -1. Open the metric in the web console and check the time window and property filters against the events you expect to count. -2. Look at the raw event stream for a user you know converted. If the event is present but the metric does not count it, the filter is the cause. +1. Open the metric in the web console and check the time window and property filters against the events you expect to count. See [Metrics](/docs/web-console-docs/goals-and-metrics/metrics/overview) for how metrics are configured. +2. Look at the raw event stream for a user you know converted in [Events > Explore](/docs/web-console-docs/Events/the-events-page). If the event is present but the metric does not count it, the filter is the cause. 3. Widen the time window or relax the property filter and re-check. - -:::info -Metric configuration lives in the web console, not the SDK. The SDK only sends the goal event; the console decides which events count. See the [metrics documentation](/docs/web-console-docs/goals-and-metrics/metrics/overview) for details. -::: From 055c2879c24966bdd9e4414dfb83e1404cf47f2f Mon Sep 17 00:00:00 2001 From: Cal Courtney Date: Thu, 9 Jul 2026 14:57:06 +0100 Subject: [PATCH 5/5] docs: fix multiple-contexts cause to describe duplicate exposures Variant assignment is deterministic from the unit ID and experiment name, so multiple contexts for the same user do not cause them to land in different variants. The actual symptom is duplicate exposure events that over-count the user and skew the variant split (SRM). Reworded the symptom, cause, and code comment to reflect this. Co-Authored-By: Claude Fable 5 --- .../troubleshooting/sdk-context-best-practices.mdx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx index ecf5332b..cd05cd32 100644 --- a/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx +++ b/docs/APIs-and-SDKs/SDK-Documentation/troubleshooting/sdk-context-best-practices.mdx @@ -6,11 +6,11 @@ sidebar_position: 3 The symptoms below show up as data quality problems in the web console: SRM warnings, assignment conflicts, or participants counted under the wrong user. Each entry describes the symptom, the cause, and the fix. -## Symptom: SRM warning with assignment conflicts +## Symptom: SRM warning from duplicate exposures -You see an SRM alert on the experiment dashboard, and the raw event stream shows the same user assigned to different variants across requests or page loads. +You see an SRM alert on the experiment dashboard, and the raw event stream shows the same user exposed multiple times. -**Cause: multiple contexts for one user.** A context is scoped to one user. If you create a new context for every treatment check, the SDK records a separate exposure each time and the user can land in different variants across requests. On a server, each request gets its own context; on a client, the context should live for the session. +**Cause: multiple contexts for one user.** A context is scoped to one user. If you create a new context for every treatment check, the SDK records a separate exposure each time. Variant assignment is deterministic from the unit ID and experiment name, so the user sees the same variant on every request. But each context emits its own exposure event, so the analysis over-counts that user and the variant split skews. On a server, each request gets its own context; on a client, the context should live for the session. **Fix: one context per user lifecycle.** Create the context once at the start of the request or app session, reuse it for every treatment check and goal track, and finalize it before the response returns or the session ends. @@ -18,8 +18,8 @@ You see an SRM alert on the experiment dashboard, and the raw event stream shows ```ts export function handleRequest(req: Request, res: Response) { - // New context each call. Two requests for the same user - // can land in different variants. + // New context each call. The user gets the same variant, + // but the SDK records a separate exposure each time. const context = sdk.createContext({ units: { user_id: req.userId } }); const variant = context.treatment("checkout_flow"); res.render(variant === 1 ? "new" : "old");