diff --git a/frontend/web/components/experiments/results/ExperimentDetailHeader.tsx b/frontend/web/components/experiments/results/ExperimentDetailHeader.tsx index aeb2650e2820..f9169321f607 100644 --- a/frontend/web/components/experiments/results/ExperimentDetailHeader.tsx +++ b/frontend/web/components/experiments/results/ExperimentDetailHeader.tsx @@ -284,7 +284,7 @@ const ExperimentDetailHeader: FC = ({ {renderActions()}
- {metricName && {metricName}} + {metricName && {metricName}} {[startedFact, endedFact].filter(Boolean).length > 0 && ( {metricName ? '· ' : ''} diff --git a/frontend/web/components/experiments/results/ExperimentRecommendation.tsx b/frontend/web/components/experiments/results/ExperimentRecommendation.tsx index a781c019647a..a56d860b3ab2 100644 --- a/frontend/web/components/experiments/results/ExperimentRecommendation.tsx +++ b/frontend/web/components/experiments/results/ExperimentRecommendation.tsx @@ -56,8 +56,8 @@ const ExperimentRecommendation: FC = ({ {' '} is outperforming{' '} by{' '} - {summary.liftVsControl} with {summary.chanceToBest} probability of being - the best variant. + {summary.liftValue} with {summary.chanceToBest} probability of being the + best variant.
Consider rolling out{' '} diff --git a/frontend/web/components/experiments/results/ExperimentResultsAxisChart.tsx b/frontend/web/components/experiments/results/ExperimentResultsAxisChart.tsx index 5b82c91b5efe..935fa1137dbd 100644 --- a/frontend/web/components/experiments/results/ExperimentResultsAxisChart.tsx +++ b/frontend/web/components/experiments/results/ExperimentResultsAxisChart.tsx @@ -98,10 +98,16 @@ const ExperimentResultsAxisChart: FC = ({
- {v.name} + + {v.name} +
{ - if (identity.isControl || !inference) return '—' - const pct = Math.round(inference.chance_to_win * 100) + if (chanceToWin === null) return '—' + const pct = Math.round(chanceToWin * 100) const colour = isHighest ? colorTextSuccess : colorTextSecondary return (
@@ -117,7 +118,7 @@ const renderWinProbability = ( style={{ background: colour, width: `${pct}%` }} />
- {pct}% + {formatChancePct(chanceToWin)}
) } @@ -133,95 +134,106 @@ type ExperimentResultsScorecardTableProps = { const ExperimentResultsScorecardTable: FC< ExperimentResultsScorecardTableProps -> = ({ identities, liftRange, metric, metricResult, srmBroken, winnerKey }) => ( -
-
- - - - - - - - - - - - - {identities.map((v) => { - const stats = metricResult?.variants[v.key] - const inference = metricResult?.inference[v.key] ?? null - return ( - - - - - - - - - ) - })} - -
VariantExposures - {AGGREGATION_HEADER[metric.aggregation]} - - - Delta - - - } - > - How much better or worse a variant performed compared to - control, as a percentage of the control's value. - - - - Credible Interval (95%) - - - } - > - The range we are 95% confident the true lift falls within. If it - doesn't cross zero, the result is statistically significant. - - Win Probability
- - - {v.name} - - {stats ? stats.n.toLocaleString() : '—'}{renderMetricValue(stats, metric.aggregation)} - {renderLift( - v, - inference, - metric.direction ?? 'up', - liftRange, - )} - {renderCI(v, inference)} - {renderWinProbability(v, inference, v.key === winnerKey)} -
+> = ({ identities, liftRange, metric, metricResult, srmBroken, winnerKey }) => { + const controlChance = metricResult + ? getControlChanceToWin(metricResult, identities) + : null + return ( +
+
+ + + + + + + + + + + + + {identities.map((v) => { + const stats = metricResult?.variants[v.key] + const inference = metricResult?.inference[v.key] ?? null + return ( + + + + + + + + + ) + })} + +
VariantExposures + {AGGREGATION_HEADER[metric.aggregation]} + + + Delta + + + } + > + How much better or worse a variant performed compared to + control, as a percentage of the control's value. + + + + Credible Interval (95%) + + + } + > + The range we are 95% confident the true lift falls within. If + it doesn't cross zero, the result is statistically + significant. + + Win Probability
+ + + {v.name} + + {stats ? stats.n.toLocaleString() : '—'}{renderMetricValue(stats, metric.aggregation)} + {renderLift( + v, + inference, + metric.direction ?? 'up', + liftRange, + )} + {renderCI(v, inference)} + {renderWinProbability( + v.isControl + ? controlChance + : inference && inference.chance_to_win, + v.key === winnerKey, + )} +
+
+ {srmBroken && ( +

+ Sample ratio mismatch detected — the variation split looks broken; + interpret results with caution. +

+ )}
- {srmBroken && ( -

- Sample ratio mismatch detected — the variation split looks broken; - interpret results with caution. -

- )} -
-) + ) +} export default ExperimentResultsScorecardTable diff --git a/frontend/web/components/experiments/results/ExperimentSummaryScorecard.tsx b/frontend/web/components/experiments/results/ExperimentSummaryScorecard.tsx index 1c42ac8f044a..537bdaa31d13 100644 --- a/frontend/web/components/experiments/results/ExperimentSummaryScorecard.tsx +++ b/frontend/web/components/experiments/results/ExperimentSummaryScorecard.tsx @@ -3,6 +3,7 @@ import InfoMessage from 'components/InfoMessage' import { BayesianResultsSummary, Experiment } from 'common/types/responses' import { deriveSummary } from './derive' import StatCard from './StatCard' +import VariantName from './VariantName' type ExperimentSummaryScorecardProps = { usersEnrolled: number | null @@ -25,6 +26,9 @@ const ExperimentSummaryScorecard: FC = ({ if (summary?.liftTone === 'success') liftClassName = 'text-success' if (summary?.liftTone === 'danger') liftClassName = 'text-danger' + // Give long winner names room by borrowing a column from Users enrolled. + const wideWinner = (summary?.winnerName.length ?? 0) > 15 + return ( <> {!summary && hasResults && ( @@ -34,14 +38,14 @@ const ExperimentSummaryScorecard: FC = ({ )}
-
+
-
+
= ({ - {summary.winnerName} + ) : undefined } @@ -60,16 +69,26 @@ const ExperimentSummaryScorecard: FC = ({ + {summary.chanceToBest} + + ) : undefined + } />
{summary.liftVsControl} + summary?.liftValue ? ( + {summary.liftValue} ) : undefined } /> diff --git a/frontend/web/components/experiments/results/StatCard.tsx b/frontend/web/components/experiments/results/StatCard.tsx index 6d48057eb56f..8c9b1ca3b857 100644 --- a/frontend/web/components/experiments/results/StatCard.tsx +++ b/frontend/web/components/experiments/results/StatCard.tsx @@ -1,5 +1,6 @@ import { FC, ReactNode } from 'react' import ContentCard from 'components/base/grid/ContentCard' +import './results.scss' type StatCardProps = { label: string @@ -10,7 +11,7 @@ type StatCardProps = { const StatCard: FC = ({ label, loading, value }) => (
{label}
-
+
{loading ? — : value ?? '—'}
diff --git a/frontend/web/components/experiments/results/VariantName.tsx b/frontend/web/components/experiments/results/VariantName.tsx index 0f2390e0a7d9..47571d82e002 100644 --- a/frontend/web/components/experiments/results/VariantName.tsx +++ b/frontend/web/components/experiments/results/VariantName.tsx @@ -1,21 +1,44 @@ import { FC } from 'react' import ColorSwatch from 'components/ColorSwatch' +import useFitText from 'components/hooks/useFitText' +import './results.scss' type VariantNameProps = { name: string colour: string + fontSize?: number + fit?: boolean } -const VariantName: FC = ({ colour, name }) => ( - - - {name} - -) +// fit shrinks the font until the name matches the container width, with an +// ellipsis floor; it renders as a block, which drops the prose baseline +// alignment — leave it off inside a sentence. +const VariantName: FC = ({ colour, fit, fontSize, name }) => { + const large = !!fontSize && fontSize >= 20 + const { fontSize: fittedSize, ref } = useFitText( + fit ? fontSize : undefined, + name, + ) + const appliedSize = fit ? fittedSize : fontSize + + return ( + + + {name} + + ) +} export default VariantName diff --git a/frontend/web/components/experiments/results/__tests__/derive.test.ts b/frontend/web/components/experiments/results/__tests__/derive.test.ts index 8e37fc86d67e..d7be5fc0488e 100644 --- a/frontend/web/components/experiments/results/__tests__/derive.test.ts +++ b/frontend/web/components/experiments/results/__tests__/derive.test.ts @@ -7,6 +7,8 @@ import { LiftTone, deriveSummary, formatBucketLabel, + formatChancePct, + getControlChanceToWin, getHeadlineTotal, getVariantIdentities, getVariantTotals, @@ -289,6 +291,28 @@ describe('getWinningVariant', () => { }) }) +describe('getControlChanceToWin', () => { + it('infers the control chance from the treatment chances', () => { + expect(getControlChanceToWin(losingMetricResult, identities)).toBeCloseTo( + 0.85, + ) + }) + + it('clamps the control chance at zero when treatment chances exceed one', () => { + expect(getControlChanceToWin(metricResult, identities)).toBe(0) + }) + + it.each<[string, BayesianMetricResult['inference']]>([ + ['no treatment', {}], + ['only some treatments', { b: metricResult.inference.b }], + ['a null-inference treatment', { b: metricResult.inference.b, c: null }], + ])('returns null when %s has inference data', (_, inference) => { + expect( + getControlChanceToWin({ ...metricResult, inference }, identities), + ).toBeNull() + }) +}) + describe('deriveSummary', () => { const experiment: Experiment = { created_at: '2026-06-01T00:00:00Z', @@ -325,25 +349,45 @@ describe('deriveSummary', () => { it('summarises the winning treatment with its lift vs control', () => { expect(deriveSummary(experiment, results(metricResult))).toMatchObject({ chanceToBest: '75%', + chanceToBestHigh: false, controlWins: false, + liftLabel: 'Lift vs control', liftTone: 'success', - liftVsControl: '+8.0%', + liftValue: '+8.0%', winnerName: 'b', }) }) - it('summarises control as the winner with a baseline lift', () => { + it('summarises a winning control with the best treatment lift negated', () => { expect( deriveSummary(experiment, results(losingMetricResult)), ).toMatchObject({ chanceToBest: '85%', controlWins: true, - liftTone: 'neutral', - liftVsControl: 'Baseline', + liftLabel: 'Control vs best variant', + liftTone: 'success', + liftValue: '+9.0%', winnerName: 'Control', }) }) + it('caps the winning control lead at +100% when a treatment collapses to zero', () => { + const wipedOut: BayesianMetricResult = { + ...losingMetricResult, + inference: { + b: { chance_to_win: 0.05, ci_high: -0.9, ci_low: -1, lift: -1 }, + c: { chance_to_win: 0.03, ci_high: -0.3, ci_low: -0.7, lift: -0.5 }, + }, + } + expect(deriveSummary(experiment, results(wipedOut))).toMatchObject({ + chanceToBestHigh: true, + controlWins: true, + liftLabel: 'Control vs best variant', + liftTone: 'success', + liftValue: '+100.0%', + }) + }) + it.each<[OptionalMetricDirection, LiftTone]>([ ['up', 'success'], ['down', 'danger'], @@ -361,6 +405,20 @@ describe('deriveSummary', () => { ) }) +describe('formatChancePct', () => { + it.each<[number, string]>([ + [1, '> 99%'], + [0.996, '> 99%'], + [0.994, '99%'], + [0.75, '75%'], + [0.006, '1%'], + [0.004, '< 1%'], + [0, '< 1%'], + ])('formats a %d chance as %s', (chance, expected) => { + expect(formatChancePct(chance)).toBe(expected) + }) +}) + describe('isLiftFavourable', () => { it.each<[number, MetricDirection, boolean]>([ [0.08, 'up', true], diff --git a/frontend/web/components/experiments/results/derive.ts b/frontend/web/components/experiments/results/derive.ts index a9664a78d09c..cf095c0bed21 100644 --- a/frontend/web/components/experiments/results/derive.ts +++ b/frontend/web/components/experiments/results/derive.ts @@ -141,6 +141,14 @@ export const getLiftColour = ( return isLiftFavourable(lift, direction) ? colorTextSuccess : colorTextDanger } +// Extreme rounded probabilities would read as certainties — hedge them. +export const formatChancePct = (chance: number): string => { + const pct = Math.round(chance * 100) + if (pct >= 100) return '> 99%' + if (pct <= 0) return '< 1%' + return `${pct}%` +} + export const formatLiftPct = (lift: number): string => { const pct = lift * 100 return `${pct >= 0 ? '+' : ''}${pct.toFixed(1)}%` @@ -163,17 +171,32 @@ export type WinningVariant = { // chance_to_win is pairwise — P(variant beats control) — so chances don't sum // to 1 across arms. P(control is best) = P(no variant beats it); the Bonferroni // bound 1 - Σ chance_to_win is exact for one variant, conservative otherwise. -export const getWinningVariant = ( +// The bound is only meaningful over all arms, so it stays null (and control +// can't be declared the winner) until every treatment has inference. +export const getControlChanceToWin = ( + metricResult: BayesianMetricResult, + identities: VariantIdentity[], +): number | null => { + const treatments = identities.filter((v) => !v.isControl) + if (!treatments.length) return null + let treatmentChancesTotal = 0 + for (const v of treatments) { + const inf = metricResult.inference[v.key] + if (!inf) return null + treatmentChancesTotal += inf.chance_to_win + } + return Math.max(0, 1 - treatmentChancesTotal) +} + +export const getBestTreatment = ( metricResult: BayesianMetricResult, identities: VariantIdentity[], ): WinningVariant | null => { let best: WinningVariant | null = null - let treatmentChancesTotal = 0 for (const v of identities) { if (v.isControl) continue const inf = metricResult.inference[v.key] if (!inf) continue - treatmentChancesTotal += inf.chance_to_win if (!best || inf.chance_to_win > best.chanceToWin) { best = { chanceToWin: inf.chance_to_win, @@ -184,10 +207,18 @@ export const getWinningVariant = ( } } } + return best +} + +export const getWinningVariant = ( + metricResult: BayesianMetricResult, + identities: VariantIdentity[], + best: WinningVariant | null = getBestTreatment(metricResult, identities), +): WinningVariant | null => { if (!best) return null const control = identities.find((v) => v.isControl) - const controlChance = Math.max(0, 1 - treatmentChancesTotal) - if (control && controlChance > best.chanceToWin) { + const controlChance = getControlChanceToWin(metricResult, identities) + if (control && controlChance !== null && controlChance > best.chanceToWin) { return { chanceToWin: controlChance, inference: null, @@ -207,7 +238,9 @@ export type SummaryStats = { controlColour: string controlWins: boolean chanceToBest: string - liftVsControl: string + chanceToBestHigh: boolean + liftLabel: string + liftValue: string liftTone: LiftTone } @@ -221,28 +254,34 @@ export const deriveSummary = ( if (!metricResult) return null const identities = getVariantIdentities(experiment.feature) - const winner = getWinningVariant(metricResult, identities) + const bestTreatment = getBestTreatment(metricResult, identities) + const winner = getWinningVariant(metricResult, identities, bestTreatment) if (!winner) return null const winnerIdentity = identities.find((v) => v.key === winner.key) const controlIdentity = identities.find((v) => v.isControl) const direction = metric.direction ?? 'up' + let lift: number | null = winner.inference?.lift ?? null + if (winner.isControl) { + // Control's lead, kept in control's terms: the best treatment's lift + // negated, so the magnitude matches the delta column and axis chart. + lift = bestTreatment?.inference ? -bestTreatment.inference.lift : null + } + let liftTone: LiftTone = 'neutral' - if (winner.inference && direction !== 'informational') { - liftTone = isLiftFavourable(winner.inference.lift, direction) - ? 'success' - : 'danger' + if (lift !== null && direction !== 'informational') { + liftTone = isLiftFavourable(lift, direction) ? 'success' : 'danger' } return { - chanceToBest: `${Math.round(winner.chanceToWin * 100)}%`, + chanceToBest: formatChancePct(winner.chanceToWin), + chanceToBestHigh: winner.chanceToWin > 0.9, controlColour: controlIdentity?.colour ?? '', controlWins: winner.isControl, + liftLabel: winner.isControl ? 'Control vs best variant' : 'Lift vs control', liftTone, - liftVsControl: winner.inference - ? formatLiftPct(winner.inference.lift) - : 'Baseline', + liftValue: lift !== null ? formatLiftPct(lift) : 'Baseline', winnerColour: winnerIdentity?.colour ?? '', winnerName: winner.name, } diff --git a/frontend/web/components/experiments/results/results.scss b/frontend/web/components/experiments/results/results.scss index 0866ca5d0bc5..096ed509b671 100644 --- a/frontend/web/components/experiments/results/results.scss +++ b/frontend/web/components/experiments/results/results.scss @@ -17,6 +17,27 @@ color: var(--color-text-secondary); } +// Block display keeps the element's width pinned to the container so the +// fit observer sees resizes; ellipsis is the floor when the font can't +// shrink further. (Bootstrap's .text-truncate helper isn't in this build.) +.variant-name--fit { + display: block; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +// Caps' optical centre sits ~0.35em above the baseline, so lift the dot's +// bottom edge by that minus half its height. Text stays on the baseline. +.variant-name__swatch { + vertical-align: calc(0.35em - 4px); + + &--lg { + vertical-align: calc(0.35em - 6px); + } +} + .experiment-results { &__legend-row { display: flex; @@ -56,9 +77,13 @@ position: relative; flex: 1; height: 12px; - background: var(--color-surface-muted); + background: var(--color-surface-emphasis); border-radius: var(--radius-full); overflow: hidden; + + @media (max-width: 1263px) { + display: none; + } } &__lift-zero { @@ -172,6 +197,11 @@ padding-right: 6px; } + &__axis-row-label-text { + overflow: hidden; + text-overflow: ellipsis; + } + &__axis-tracks { position: relative; padding: 12px 0; @@ -213,7 +243,7 @@ &__win-prob-track { width: 100px; height: 6px; - background: var(--color-surface-muted); + background: var(--color-surface-emphasis); border-radius: var(--radius-full); overflow: hidden; flex-shrink: 0; @@ -295,6 +325,14 @@ white-space: nowrap; } + // h4 typography without the heading semantics — stat values shouldn't + // appear in the document outline. + &__stat-value { + font-size: 24px; + line-height: 32px; + font-weight: var(--font-weight-bold); + } + &__refresh { display: flex; align-items: center; diff --git a/frontend/web/components/hooks/useFitText.ts b/frontend/web/components/hooks/useFitText.ts new file mode 100644 index 000000000000..2bd7e789cc30 --- /dev/null +++ b/frontend/web/components/hooks/useFitText.ts @@ -0,0 +1,41 @@ +import { useLayoutEffect, useRef, useState } from 'react' + +const MIN_FONT_SIZE = 14 + +// Shrinks the font from maxFontSize (down to MIN_FONT_SIZE) until the +// element's content fits its width, refitting when the element is resized. +// Pass undefined to leave the element alone. +export default function useFitText( + maxFontSize: number | undefined, + text: string, +) { + const ref = useRef(null) + const [fontSize, setFontSize] = useState(maxFontSize) + + useLayoutEffect(() => { + const el = ref.current + if (!el || !maxFontSize) return + const fit = () => { + let size = maxFontSize + el.style.fontSize = `${size}px` + while (size > MIN_FONT_SIZE && el.scrollWidth > el.clientWidth) { + size -= 1 + el.style.fontSize = `${size}px` + } + setFontSize(size) + } + fit() + // Fitting changes the element's height, so react to width changes only — + // refitting on our own mutations would loop the observer. + let lastWidth = el.clientWidth + const observer = new ResizeObserver(() => { + if (el.clientWidth === lastWidth) return + lastWidth = el.clientWidth + fit() + }) + observer.observe(el) + return () => observer.disconnect() + }, [maxFontSize, text]) + + return { fontSize, ref } +}