feat(aws/ladder): estimate stable usage for the EC2 Instance SP base layer

CHANGELOG.md

@@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/).
 
 ### Added
 
+- **`ladder`/`aws`:** `AWSLadder.WithBufferFraction` and a stable-usage estimator: when the fraction is set, `GetUsageBaseline` fills `StableUSDPerHour` with the lowest of the low-water mark, the 10th percentile and the last 3 days, times `1 - fraction`. It stays nil when the option is unset, the series has fewer than 14 days, or the 10th percentile is under half the median (#31).
 - **AWS/Redshift:** when the `AttemptJournal` also implements `common.ClaimReader`, the reserved-node dedupe skips nodes another purchase recorded as bought, before and after the send. A claim read error refuses before the send or leaves the outcome unknown after it; if the skip leaves one of several matches and the token has no sending or outcome_unknown record, the purchase is refused as ambiguous (#700).
 - **`pkg`:** optional `common.ClaimReader` (`ClaimedByOthers(ctx, token, ids)`), which an `AttemptJournal` may implement so a provider can skip commitments another purchase recorded as bought (#700).
 - **`pkg`:** `common.Commitment.CostKnown` reports whether `Cost` holds a price the provider reported. The zero value means unknown, so a `Cost` of 0 without `CostKnown` must not be read as free. Producers set it in a follow-up (#74).

pkg/ladder/types.go

@@ -453,7 +453,8 @@ type UsageBaseline struct {
 	// percentile (e.g., the 5th percentile of hourly spend).
 	LowWaterUSDPerHour *float64
 	// StableUSDPerHour is the estimated stable portion after applying the
-	// buffer fraction to LowWaterUSDPerHour.
+	// buffer fraction to LowWaterUSDPerHour. It is nil when the provider cannot
+	// estimate it (short or volatile series, or no buffer fraction configured).
 	StableUSDPerHour *float64
 	// Basis says what the series measures: total demand (the default, empty)
 	// or residual uncovered usage already net of existing commitments. See

providers/aws/ladder/baseline.go

@@ -52,13 +52,11 @@ const maxSeriesAgeDays = 3
 //     covers EC2 on-demand cost only, not RDS/ElastiCache/etc.).
 //   - LowWaterUSDPerHour is the nearest-rank percentile of the daily series
 //     (see nearestRankPercentile).
-//   - StableUSDPerHour is nil: per the pkg/ladder contract (types.go), Stable
-//     is the post-buffer-fraction estimate — a producer obligation this
-//     implementation cannot yet meet (no stable-usage estimator exists for
-//     AWS). Returning nil triggers the engine's documented degradation
-//     ("stable usage unknown; routing all core gap to flex"), which is honest
-//     and conservative. Do NOT alias it to LowWater: the engine consumes
-//     Stable verbatim as the base-layer cap and would over-commit the base.
+//   - StableUSDPerHour is nil unless WithBufferFraction was called and the
+//     series is long and steady enough (see estimateStableUSDPerHour). When set
+//     it is the post-buffer-fraction estimate the pkg/ladder contract requires.
+//     Do NOT alias it to LowWater: the engine consumes Stable verbatim as the
+//     base-layer cap and would over-commit the base.
 //
 // Error conditions:
 //   - Series empty: hard error (no data from the coverage source).
@@ -115,19 +113,14 @@ func (a *AWSLadder) GetUsageBaseline(ctx context.Context, scope ladder.Scope, lo
 		return ladder.UsageBaseline{}, fmt.Errorf("GetUsageBaseline: %w", vErr)
 	}
 
-	lowWater, err := nearestRankPercentile(series, percentile)
+	lowWater, stable, err := a.baselineStats(series, percentile)
 	if err != nil {
-		return ladder.UsageBaseline{}, fmt.Errorf("GetUsageBaseline: percentile computation failed: %w", err)
+		return ladder.UsageBaseline{}, err
 	}
 
-	// StableUSDPerHour is intentionally nil: the pkg/ladder contract defines
-	// Stable as the post-buffer-fraction estimate (a producer obligation), and
-	// no stable-usage estimator exists for AWS yet. nil triggers the engine's
-	// documented "stable usage unknown; routing all core gap to flex"
-	// degradation — honest and conservative until a real estimator lands.
 	return ladder.UsageBaseline{
 		LowWaterUSDPerHour: ptr(lowWater),
-		StableUSDPerHour:   nil,
+		StableUSDPerHour:   stable,
 		Basis:              ladder.BaselineBasisResidual,
 		Series:             series,
 		LookbackDays:       lookbackDays,
@@ -302,3 +295,21 @@ func nearestRankPercentile(data []float64, p float64) (float64, error) {
 	}
 	return sorted[rank-1], nil
 }
+
+// baselineStats derives the low-water mark and, when a buffer fraction was
+// configured, the stable estimate. Errors are returned fully prefixed. It is
+// split out so GetUsageBaseline stays within the cyclomatic-complexity limit.
+func (a *AWSLadder) baselineStats(series []float64, percentile float64) (lowWater float64, stable *float64, err error) {
+	lowWater, err = nearestRankPercentile(series, percentile)
+	if err != nil {
+		return 0, nil, fmt.Errorf("GetUsageBaseline: percentile computation failed: %w", err)
+	}
+	if a.bufferFraction == nil {
+		return lowWater, nil, nil
+	}
+	stable, err = estimateStableUSDPerHour(series, lowWater, *a.bufferFraction)
+	if err != nil {
+		return 0, nil, fmt.Errorf("GetUsageBaseline: stable usage estimate: %w", err)
+	}
+	return lowWater, stable, nil
+}

providers/aws/ladder/baseline_sdk_test.go

@@ -120,6 +120,15 @@ func TestScopedBaselineParentCompatible(t *testing.T) {
 			if mode == "account" || mode == "reverse" {
 				assert.Equal(t, int32(2), calls.Load())
 			}
+			if mode == "account" {
+				// 28 days at 10/h then 2 days at 2/h: P10 = 10, min3 = 2, so 2 * (1 - 0.1).
+				_, err = a.WithBufferFraction(0.1)
+				require.NoError(t, err)
+				bl, err = a.GetUsageBaseline(context.Background(), scope, 30, 5)
+				require.NoError(t, err)
+				require.NotNil(t, bl.StableUSDPerHour)
+				assert.InDelta(t, 1.8, *bl.StableUSDPerHour, 1e-9)
+			}
 		})
 	}
 }

providers/aws/ladder/ladder.go

@@ -3,6 +3,7 @@ package ladder
 import (
 	"errors"
 	"fmt"
+	"math"
 
 	"github.com/LeanerCloud/cloud-commitments-go/pkg/common"
 	"github.com/LeanerCloud/cloud-commitments-go/pkg/ladder"
@@ -95,7 +96,10 @@ type AWSLadder struct {
 	exchange    exchangeRunner      // write side; nil until WithWriteSide is called
 	spRates     spCommitmentRateSource
 	observed    observedRateSource
-	cfg         Config
+	// bufferFraction is nil until WithBufferFraction is called; nil keeps the
+	// stable-usage estimate off, so callers that never opt in see no change.
+	bufferFraction *float64
+	cfg            Config
 }
 
 // New constructs an AWSLadder. The five required read-side interfaces must be
@@ -188,3 +192,17 @@ func (a *AWSLadder) WithWriteSide(riP riPurchaser, spP spPurchaser, ex exchangeR
 	a.exchange = ex
 	return a, nil
 }
+
+// WithBufferFraction sets the buffer fraction used to haircut the stable-usage
+// estimate and returns the same instance for chaining. f must be finite and in
+// [0, 1). A later call overwrites an earlier one, and a failed call leaves the
+// stored value unchanged.
+func (a *AWSLadder) WithBufferFraction(f float64) (*AWSLadder, error) {
+	// Same rule as LadderConfig.Validate in pkg/ladder, which is inline there and
+	// cannot be imported; the negated form also rejects NaN and +Inf.
+	if math.IsNaN(f) || !(f >= 0 && f < 1) {
+		return nil, fmt.Errorf("AWSLadder.WithBufferFraction: buffer_fraction %g must be in [0, 1)", f)
+	}
+	a.bufferFraction = &f
+	return a, nil
+}

providers/aws/ladder/ladder_test.go

@@ -778,10 +778,10 @@ func TestGetUsageBaseline_SingleDaySeries_ReturnsThatValue(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, bl.LowWaterUSDPerHour)
 	assert.InDelta(t, 3.0, *bl.LowWaterUSDPerHour, 1e-9)
-	// StableUSDPerHour must be nil: no stable-usage estimator exists yet, and
-	// the pkg/ladder contract defines Stable as post-buffer-fraction (aliasing
-	// it to LowWater would make the engine over-commit the base layer). nil
-	// triggers the engine's documented "route all core gap to flex" degradation.
+	// StableUSDPerHour must be nil without WithBufferFraction: the pkg/ladder
+	// contract defines Stable as post-buffer-fraction (aliasing it to LowWater
+	// would make the engine over-commit the base layer). nil triggers the
+	// engine's documented "route all core gap to flex" degradation.
 	assert.Nil(t, bl.StableUSDPerHour)
 	assert.Equal(t, 7, bl.LookbackDays)
 	assert.InDelta(t, 5.0, bl.Percentile, 1e-9)
@@ -1140,3 +1140,59 @@ func makeConstantPoints(n int, value float64) []DailyPoint {
 	}
 	return out
 }
+
+// ---------------------------------------------------------------------------
+// WithBufferFraction
+// ---------------------------------------------------------------------------
+
+func TestWithBufferFraction_Validation(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name    string
+		f       float64
+		wantErr string
+	}{
+		{"NaN", math.NaN(), "AWSLadder.WithBufferFraction: buffer_fraction NaN must be in [0, 1)"},
+		{"+Inf", math.Inf(1), "AWSLadder.WithBufferFraction: buffer_fraction +Inf must be in [0, 1)"},
+		{"negative", -0.1, "AWSLadder.WithBufferFraction: buffer_fraction -0.1 must be in [0, 1)"},
+		{"one", 1.0, "AWSLadder.WithBufferFraction: buffer_fraction 1 must be in [0, 1)"},
+		{"above one", 1.5, "AWSLadder.WithBufferFraction: buffer_fraction 1.5 must be in [0, 1)"},
+		{"zero accepted", 0, ""},
+		{"just below one accepted", 0.999, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			a := newTestLadder(t, &fakeRILister{}, &fakeSPLister{}, &fakeCoverageSource{}, &fakeUtilizationSource{})
+			prev := 0.25
+			_, err := a.WithBufferFraction(prev)
+			require.NoError(t, err)
+
+			got, err := a.WithBufferFraction(tt.f)
+			if tt.wantErr != "" {
+				require.EqualError(t, err, tt.wantErr)
+				assert.Nil(t, got)
+				require.NotNil(t, a.bufferFraction)
+				assert.Equal(t, prev, *a.bufferFraction, "failed call must leave the stored value unchanged")
+				return
+			}
+			require.NoError(t, err)
+			assert.Same(t, a, got)
+			require.NotNil(t, a.bufferFraction)
+			assert.Equal(t, tt.f, *a.bufferFraction)
+		})
+	}
+}
+
+func TestWithBufferFraction_UnsetAndLastCallWins(t *testing.T) {
+	t.Parallel()
+	a := newTestLadder(t, &fakeRILister{}, &fakeSPLister{}, &fakeCoverageSource{}, &fakeUtilizationSource{})
+	assert.Nil(t, a.bufferFraction)
+
+	_, err := a.WithBufferFraction(0.1)
+	require.NoError(t, err)
+	_, err = a.WithBufferFraction(0.3)
+	require.NoError(t, err)
+	require.NotNil(t, a.bufferFraction)
+	assert.Equal(t, 0.3, *a.bufferFraction)
+}

providers/aws/ladder/stable_usage.go

@@ -0,0 +1,90 @@
+package ladder
+
+import (
+	"math"
+	"slices"
+)
+
+const (
+	// minStableSeriesDays is two full weekly cycles, so weekday/weekend
+	// seasonality is visible before a base layer is sized from the series.
+	minStableSeriesDays = 14
+	// stableFloorPercentile is independent of the configured low-water
+	// percentile, which can be up to 100 (the series maximum).
+	stableFloorPercentile = 10
+	// minFloorToMedianRatio gates volatile series: the floor must be at least
+	// this share of typical usage. 0.5 is exact in binary, so boundaries are exact.
+	minFloorToMedianRatio = 0.5
+	// trailingMinDays catches a sudden drop that the percentile terms miss.
+	trailingMinDays = 3
+)
+
+// finiteNonNeg reports whether v is finite and non-negative; NaN fails >=.
+func finiteNonNeg(v float64) bool { return v >= 0 && !math.IsInf(v, 0) }
+
+// unitFraction reports whether f is in [0, 1); NaN fails the comparisons.
+func unitFraction(f float64) bool { return f >= 0 && f < 1 }
+
+// validStableInputs guarantees a non-empty series of finite non-negative
+// values, a finite non-negative lowWater and f in [0, 1).
+func validStableInputs(series []float64, lowWater, f float64) bool {
+	if len(series) == 0 {
+		return false
+	}
+	for _, v := range series {
+		if !finiteNonNeg(v) {
+			return false
+		}
+	}
+	return finiteNonNeg(lowWater) && unitFraction(f)
+}
+
+// stableQuantiles returns the floor percentile and the nearest-rank median
+// (the lower median for even N).
+func stableQuantiles(series []float64) (p10, median float64, err error) {
+	p10, err = nearestRankPercentile(series, stableFloorPercentile)
+	if err != nil {
+		return 0, 0, err
+	}
+	median, err = nearestRankPercentile(series, 50)
+	if err != nil {
+		return 0, 0, err
+	}
+	return p10, median, nil
+}
+
+// stableGatePasses is true for a long enough, steady, non-zero series. The
+// ratio is compared by multiplication, and median > 0 stops an all-zero series
+// passing 0 >= 0.5*0.
+func stableGatePasses(n int, p10, median float64) bool {
+	return n >= minStableSeriesDays && median > 0 && p10 >= minFloorToMedianRatio*median
+}
+
+// stableBase is the lowest of the low-water mark, the floor percentile and the
+// trailing minimum, so every term can only lower the estimate. PRECONDITION:
+// len(series) >= trailingMinDays; the minStableSeriesDays gate must run first.
+func stableBase(series []float64, lowWater, p10 float64) float64 {
+	return min(lowWater, p10, slices.Min(series[len(series)-trailingMinDays:]))
+}
+
+// estimateStableUSDPerHour returns the post-buffer stable usage, or nil when
+// the series cannot support one (invalid input, short, volatile, or zero).
+// The error is reserved for a percentile failure, which validStableInputs makes
+// unreachable.
+func estimateStableUSDPerHour(series []float64, lowWater, bufferFraction float64) (*float64, error) {
+	if !validStableInputs(series, lowWater, bufferFraction) {
+		return nil, nil
+	}
+	p10, median, err := stableQuantiles(series)
+	if err != nil {
+		return nil, err
+	}
+	if !stableGatePasses(len(series), p10, median) {
+		return nil, nil
+	}
+	v := stableBase(series, lowWater, p10) * (1 - bufferFraction)
+	if v <= 0 || math.IsInf(v, 0) || math.IsNaN(v) {
+		return nil, nil
+	}
+	return ptr(v), nil
+}