Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 38 additions & 1 deletion internal/api/combined_plan_purchase_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,10 @@ func newCombinedPlanFixtureIn(t *testing.T, automatic bool, capAmount *float64,
global, err := f.store.GetGlobalConfig(ctx)
require.NoError(t, err)
recipient := "fixture@example.invalid"
global.NotificationEmail, global.AutoPurchaseEnabled, global.RequireDifferentApprover = &recipient, automatic, true
// 4-eyes is on for the manual (approval) flows only. With 4-eyes on, an
// AutoPurchase step waits for a human approval instead of auto-executing
// (#1771); internal/purchase/four_eyes_auto_plan_integration_test.go covers that.
global.NotificationEmail, global.AutoPurchaseEnabled, global.RequireDifferentApprover = &recipient, automatic, !automatic
global.PurchaseDelayHours = 0
require.NoError(t, f.store.SaveGlobalConfig(ctx, global))
response := combinedRoute(t, f.handler, "POST", "/api/plans", PlanRequest{
Expand Down Expand Up @@ -384,6 +387,40 @@ func TestCombinedPlanPurchase_NonUSDApprovalRefusedUnderCap(t *testing.T) {
f.assertCompleted(t)
}

// With 4-eyes on, an AutoPurchase plan's due step is not auto-executed: it
// waits for a human approval, and a different user's approval then buys it
// (#1771).
func TestCombinedPlanPurchase_AutomaticWithFourEyesWaitsForApproval(t *testing.T) {
capAmount := 2400.0
f := newCombinedPlanFixture(t, true, &capAmount)
f.assertAttached(t)
ctx := context.Background()
global, err := f.store.GetGlobalConfig(ctx)
require.NoError(t, err)
global.RequireDifferentApprover = true
require.NoError(t, f.store.SaveGlobalConfig(ctx, global))
_, err = f.pool.Exec(ctx, `UPDATE purchase_executions SET scheduled_date=now()-interval '1 minute' WHERE execution_id=$1`, f.root.ExecutionID)
require.NoError(t, err)

result, err := f.manager.ProcessScheduledPurchases(ctx)
require.NoError(t, err)
assert.Zero(t, result.Executed, "%+v", result)
assert.Zero(t, result.Failed, "%+v", result)
assert.Empty(t, f.service.recorded(), "4-eyes on: the due step must not be bought automatically")
row, err := f.store.GetExecutionByID(ctx, f.root.ExecutionID)
require.NoError(t, err)
assert.Equal(t, "notified", row.Status, "the step waits in the approval queue")

approver := uuid.NewString()
_, err = f.pool.Exec(ctx, `INSERT INTO users (id,email,password_hash,salt,active,group_ids,created_at,updated_at) SELECT $1,'approver@example.invalid','x','y',true,ARRAY[g.id],now(),now() FROM groups g ORDER BY g.name LIMIT 1`, approver)
require.NoError(t, err)
response := f.approveAs(t, approver, 0)
require.Equal(t, 200, response.StatusCode, response.Body)
f.assertCompleted(t)
assert.Len(t, f.service.recorded(), 1)
}

// With 4-eyes off (the fixture's automatic default), the due step auto-executes.
func TestCombinedPlanPurchase_AutomaticOnScheduledExecution(t *testing.T) {
capAmount := 2400.0
f := newCombinedPlanFixture(t, true, &capAmount)
Expand Down
8 changes: 7 additions & 1 deletion internal/config/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,13 @@ type GlobalConfig struct {
// mode first (the admin wildcard is NOT exempt from the restriction).
// While it is on, unattended RI exchange runs (RIExchangeMode "auto" and
// ladder reshapes) run as "manual" and only raise pending records
// (issue #679).
// (issue #679), and recommendation-bearing AutoPurchase steps are not
// auto-executed: they wait for a human approval in the approval queue
// (issue #1771); empty steps still route as before. A step root
// pre-created with the plan keeps its human creator, who then cannot
// approve it. A step root pre-created through the admin API key (no
// creator and not system-created) cannot be approved by anyone while this
// is on: it waits, then expires, and a human recreates the plan.
RequireDifferentApprover bool `json:"require_different_approver" dynamodbav:"require_different_approver"`
}

Expand Down
3 changes: 3 additions & 0 deletions internal/email/sender.go
Original file line number Diff line number Diff line change
Expand Up @@ -587,6 +587,9 @@ type NotificationData struct {
TotalRecurring float64
TotalUnits int
StepCap float64
// NeedsApproval is set when the step does not run on its date but waits
// for a human approval in the approval queue (manual plan, or 4-eyes on).
NeedsApproval bool
// RevocationToken is the one-time token embedded in the revocation link
// of a post-execution notification email. When non-empty, the template
// renders a "Revoke this purchase" CTA that hits
Expand Down
18 changes: 18 additions & 0 deletions internal/email/template_renderers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -797,3 +797,21 @@ func TestRenderRIExchangePendingApprovalEmail_PlainTextVerbatim(t *testing.T) {
assert.NotContains(t, body, "'", "plain-text must not HTML-encode apostrophe")
assert.NotContains(t, body, "&", "plain-text must not HTML-encode ampersand")
}

func TestRenderScheduledPurchaseEmail_NeedsApprovalWording(t *testing.T) {
data := NotificationData{
PlanName: "p", PurchaseDate: "November 3, 2026", DaysUntilPurchase: 3, DashboardURL: "https://d.example", ExecutionID: "e1",
TotalCommitment: 100, StepCap: 500, TotalUnits: 1, Recommendations: []RecommendationSummary{{Count: 1, ResourceType: "m5.large", Region: "r"}},
}
body, err := RenderScheduledPurchaseEmail(data)
require.NoError(t, err)
assert.Contains(t, body, ", cap $500.00, on November 3, 2026 unless canceled.")
assert.Contains(t, body, "You have 3 days to modify or cancel before automatic execution.")

data.NeedsApproval = true
body, err = RenderScheduledPurchaseEmail(data)
require.NoError(t, err)
assert.Contains(t, body, ", cap $500.00, due on November 3, 2026. It is bought once approved. Review & Edit below and approve it in the approval queue.")
assert.NotContains(t, body, "unless canceled")
assert.NotContains(t, body, "before automatic execution")
}
6 changes: 3 additions & 3 deletions internal/email/templates.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ This is an automated message from CUDly.
const scheduledPurchaseTemplate = `CUDly - Scheduled Purchase in {{.DaysUntilPurchase}} Days
=========================================

Based on your Purchase Plan "{{.PlanName}}", the following commitments will be purchased on {{.PurchaseDate}}:
Based on your Purchase Plan "{{.PlanName}}", the following commitments {{if .NeedsApproval}}are due on{{else}}will be purchased on{{end}} {{.PurchaseDate}}:
{{if gt .TotalCommitment 0.0}}
Will buy {{len .Recommendations}} recommendation(s) ({{.TotalUnits}} units) for ${{printf "%.2f" .TotalCommitment}} total commitment (upfront ${{printf "%.2f" .TotalUpfrontCost}} + ${{printf "%.2f" .TotalRecurring}} recurring over the terms){{if gt .StepCap 0.0}}, cap ${{printf "%.2f" .StepCap}}{{end}}, on {{.PurchaseDate}} unless canceled.
Will buy {{len .Recommendations}} recommendation(s) ({{.TotalUnits}} units) for ${{printf "%.2f" .TotalCommitment}} total commitment (upfront ${{printf "%.2f" .TotalUpfrontCost}} + ${{printf "%.2f" .TotalRecurring}} recurring over the terms){{if gt .StepCap 0.0}}, cap ${{printf "%.2f" .StepCap}}{{end}}, {{if .NeedsApproval}}due on {{.PurchaseDate}}. It is bought once approved. Review & Edit below and approve it in the approval queue.{{else}}on {{.PurchaseDate}} unless canceled.{{end}}
{{end}}
{{range .Recommendations}}
- {{.Count}}x {{.ResourceType}}{{if .Engine}} ({{.Engine}}){{end}} in {{.Region}}{{if .AvailabilityZone}} ({{.AvailabilityZone}}){{end}}{{if .Platform}} | {{.Platform}}{{end}}{{if .Tenancy}} | {{.Tenancy}}{{end}}
Expand All @@ -64,7 +64,7 @@ Actions:

[Cancel This Purchase] {{.DashboardURL}}/purchases/cancel/{{.ExecutionID}}?token={{urlquery .ApprovalToken}}

You have {{.DaysUntilPurchase}} days to modify or cancel before automatic execution.
{{if .NeedsApproval}}This purchase is not bought automatically. It needs a human approval in the approval queue; you have {{.DaysUntilPurchase}} days to review, modify or cancel it.{{else}}You have {{.DaysUntilPurchase}} days to modify or cancel before automatic execution.{{end}}

This is an automated message from CUDly.
`
Expand Down
2 changes: 1 addition & 1 deletion internal/purchase/claim_pin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ func TestNotification_EmailFailureLeavesRowPendingAndSchedulerRefusesIt(t *testi
sender.On("SendScheduledPurchaseNotification", ctx, mock.Anything).Return(errors.New("ses down"))
manager := &Manager{config: store, email: sender}

require.False(t, manager.sendPlanNotification(ctx, plan))
require.False(t, manager.sendPlanNotification(ctx, plan, false))
store.AssertNotCalled(t, "RotatePendingApprovalToken", mock.Anything, mock.Anything, mock.Anything, mock.Anything)

// The row as the DB still holds it: pending. The scheduler leaves it alone.
Expand Down
105 changes: 105 additions & 0 deletions internal/purchase/four_eyes_auto_plan_integration_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
//go:build integration

package purchase

import (
"context"
"testing"
"time"

"github.com/LeanerCloud/cloud-commitments-platform/internal/config"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// Issue #1771: with 4-eyes on, an armed auto plan's due step waits for a human
// approval instead of being bought by the sweep. Real PostgreSQL; the cloud is
// the labeled fake provider of the ramp-step fixture.

func fourEyesUser(ctx context.Context, t *testing.T, f *rampStepFixture, name string) string {
t.Helper()
id := uuid.New().String()
require.NoError(t, f.store.WithTx(ctx, func(tx pgx.Tx) error {
_, err := tx.Exec(ctx, `INSERT INTO users (id, email, password_hash, group_ids) VALUES ($1, $2, 'x', ARRAY[$1::uuid])`, id, name+"@example.invalid")
return err
}))
return id
}

func fourEyesAutoRow(ctx context.Context, t *testing.T, f *rampStepFixture, creator *string) *config.PurchaseExecution {
t.Helper()
row := &config.PurchaseExecution{
PlanID: f.plan.ID, ExecutionID: uuid.New().String(), Status: "notified", StepNumber: 3,
ScheduledDate: time.Now().Add(-time.Hour), CloudAccountID: &f.accounts[0].ID,
Recommendations: []config.RecommendationRecord{withAccount(rampStepRecommendation()[0], f.accounts[0].ID)},
CreatedByUserID: creator, CreatedBySystem: creator == nil,
}
require.NoError(t, f.store.SavePurchaseExecution(ctx, row))
return row
}

func setFourEyes(ctx context.Context, t *testing.T, f *rampStepFixture, on bool) {
t.Helper()
global, err := f.store.GetGlobalConfig(ctx)
require.NoError(t, err)
global.RequireDifferentApprover = on
require.NoError(t, f.store.SaveGlobalConfig(ctx, global))
}

func TestFourEyes_AutoPlanSweepLeavesRowForHumanApproval(t *testing.T) {
ctx := context.Background()
f := newRampStepFixture(ctx, t)
setFourEyes(ctx, t, f, true)
row := fourEyesAutoRow(ctx, t, f, nil)

result, err := f.manager.ProcessScheduledPurchases(ctx)
require.NoError(t, err)
assert.Equal(t, 0, result.Processed)
assert.Equal(t, 0, f.purchaseCount())
assert.Equal(t, "notified", rowStatus(ctx, t, f, row.ExecutionID).Status)

userB := fourEyesUser(ctx, t, f, "b")
_, _, err = f.manager.ApproveAndExecute(ctx, row.ExecutionID, "b@example.invalid", &userB, nil)
require.NoError(t, err)
assert.Equal(t, 1, f.purchaseCount())
assert.Equal(t, "completed", rowStatus(ctx, t, f, row.ExecutionID).Status)
}

func TestFourEyes_AutoPlanPreCreatedRootCreatorCannotApprove(t *testing.T) {
ctx := context.Background()
f := newRampStepFixture(ctx, t)
setFourEyes(ctx, t, f, true)
userA := fourEyesUser(ctx, t, f, "a")
userB := fourEyesUser(ctx, t, f, "b")
row := fourEyesAutoRow(ctx, t, f, &userA)

result, err := f.manager.ProcessScheduledPurchases(ctx)
require.NoError(t, err)
assert.Equal(t, 0, result.Processed)
assert.Equal(t, 0, f.purchaseCount())

_, _, err = f.manager.ApproveAndExecute(ctx, row.ExecutionID, "a@example.invalid", &userA, nil)
require.ErrorIs(t, err, ErrFourEyesDenied)
assert.Equal(t, 0, f.purchaseCount())
assert.Equal(t, "notified", rowStatus(ctx, t, f, row.ExecutionID).Status)

_, _, err = f.manager.ApproveAndExecute(ctx, row.ExecutionID, "b@example.invalid", &userB, nil)
require.NoError(t, err)
assert.Equal(t, 1, f.purchaseCount())
assert.Equal(t, "completed", rowStatus(ctx, t, f, row.ExecutionID).Status)
}

func TestFourEyes_AutoPlanSweepExecutesWhenFlagOff(t *testing.T) {
ctx := context.Background()
f := newRampStepFixture(ctx, t)
setFourEyes(ctx, t, f, false)
row := fourEyesAutoRow(ctx, t, f, nil)

result, err := f.manager.ProcessScheduledPurchases(ctx)
require.NoError(t, err)
assert.Equal(t, 1, result.Executed)
assert.Equal(t, 1, f.purchaseCount())
assert.Equal(t, "completed", rowStatus(ctx, t, f, row.ExecutionID).Status)
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,18 @@ func TestFourEyes_SweepMintedRowIsApprovableByIdentifiedApprover(t *testing.T) {
plan := attachPlanInDB(ctx, t, store, "four-eyes-sweep", time.Now().Add(24*time.Hour))
m := &Manager{config: store, email: newCountingSender(ctx),
resolver: &fakeResolver{recs: []config.RecommendationRecord{attachRec("rec-a", 2)}}}
require.True(t, m.sendPlanNotification(ctx, plan))
global, err := store.GetGlobalConfig(ctx)
require.NoError(t, err)
global.RequireDifferentApprover = true
require.NoError(t, store.SaveGlobalConfig(ctx, global))
require.True(t, m.sendPlanNotification(ctx, plan, planNeedsApproval(*plan, global)))

rows := stepRowsInDB(ctx, t, store, plan.ID)
require.Len(t, rows, 1)
require.Equal(t, "notified", rows[0].Status)
assert.True(t, rows[0].CreatedBySystem, "sweep-minted row must be marked system-created")
assert.Nil(t, rows[0].CreatedByUserID)

global, err := store.GetGlobalConfig(ctx)
require.NoError(t, err)
global.RequireDifferentApprover = true
require.NoError(t, store.SaveGlobalConfig(ctx, global))

require.NoError(t, m.enforceFourEyesPolicy(ctx, rows[0].ExecutionID, "approver@example.com", nil))
require.ErrorIs(t, m.enforceFourEyesPolicy(ctx, rows[0].ExecutionID, "", nil), ErrFourEyesDenied)
}
Expand Down
31 changes: 25 additions & 6 deletions internal/purchase/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -879,6 +879,9 @@ func (m *Manager) RecoverStrandedApprovals(ctx context.Context) (int, error) {
// - The global kill switch (GlobalConfig.AutoPurchaseEnabled) must be on. A
// read error is propagated. Only AutoPurchase plans reach this check, so
// manual purchases are unaffected.
// - With 4-eyes approval mode on (GlobalConfig.RequireDifferentApprover), a
// recommendation-bearing row is never auto-executed: it waits for a human
// approval in the approval queue (#1771).
// - A recommendation-bearing row must be "notified" (the notification email is
// the human veto window; a "pending" row may never have been emailed) and
// must fit the plan's max_step_commitment (CheckStepCommitmentCap, whose
Expand All @@ -902,16 +905,22 @@ func (m *Manager) executableByScheduler(ctx context.Context, exec *config.Purcha
if plan == nil {
return false, nil
}
enabled, err := m.autoPurchaseEnabled(ctx)
cfg, err := m.autoPurchaseConfig(ctx)
if err != nil {
return false, err
}
if !enabled {
if !cfg.AutoPurchaseEnabled {
return false, nil
}
if len(exec.Recommendations) == 0 {
return true, nil
}
// With 4-eyes on, no step buys without a second person: an auto plan's
// step waits in the approval queue for a human approval (session approve),
// like a manual plan's (#1771).
if cfg.RequireDifferentApprover {
return false, nil
}
if exec.Status != "notified" {
return false, nil
}
Expand Down Expand Up @@ -951,14 +960,24 @@ func (m *Manager) autoPurchasePlan(ctx context.Context, exec *config.PurchaseExe
// autoPurchaseEnabled reads the global auto-purchase kill switch. It fails
// closed: a read error is returned, never read as "enabled".
func (m *Manager) autoPurchaseEnabled(ctx context.Context) (bool, error) {
cfg, err := m.autoPurchaseConfig(ctx)
if err != nil {
return false, err
}
return cfg.AutoPurchaseEnabled, nil
}

// autoPurchaseConfig reads the global config the auto-purchase gates need. A
// read error or a missing config is returned, so no gate reads it as "on".
func (m *Manager) autoPurchaseConfig(ctx context.Context) (*config.GlobalConfig, error) {
cfg, err := m.config.GetGlobalConfig(ctx)
if err != nil {
return false, fmt.Errorf("failed to read global config for the auto-purchase kill switch: %w", err)
return nil, fmt.Errorf("failed to read global config for the auto-purchase kill switch: %w", err)
}
if cfg == nil {
return false, errors.New("global config unavailable for the auto-purchase kill switch")
return nil, errors.New("global config unavailable for the auto-purchase kill switch")
}
return cfg.AutoPurchaseEnabled, nil
return cfg, nil
}

// processOneExecution runs the full gate+claim+execute pipeline for a single
Expand All @@ -979,7 +998,7 @@ func (m *Manager) processOneExecution(ctx context.Context, exec config.PurchaseE
return
}
if !eligible {
logging.Infof("Skipping execution %s (AutoPurchase=false, kill switch off, source=web, or recommendation-bearing row not yet notified; requires explicit approval)", exec.ExecutionID)
logging.Infof("Skipping execution %s (AutoPurchase=false, kill switch off, source=web, 4-eyes on so it needs a human approval in the approval queue, or recommendation-bearing row not yet notified; requires explicit approval)", exec.ExecutionID)
return
}

Expand Down
43 changes: 43 additions & 0 deletions internal/purchase/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1275,3 +1275,46 @@ func TestManager_RecoverStrandedApprovals_AWSClaimLost_NoRedrive(t *testing.T) {
mockFactory.AssertNotCalled(t, "CreateAndValidateProvider", mock.Anything, mock.Anything, mock.Anything)
mockStore.AssertNotCalled(t, "SavePurchaseExecution", mock.Anything, mock.Anything)
}

// #1771: with 4-eyes on, an armed auto plan's rec-bearing notified row is not
// auto-executed; it waits for a human approval in the approval queue.
func TestManager_ProcessScheduledPurchases_FourEyesOnSkipsAutoRow(t *testing.T) {
cfg := autoPurchaseOnCfg()
cfg.RequireDifferentApprover = true
plan := &config.PurchasePlan{ID: "plan-gate", AutoPurchase: true, MaxStepCommitment: f64(1000)}
result := gateHarness(t, plan, dueRecExec("notified", upfrontRec(10)), cfg, nil)
assert.Equal(t, 0, result.Processed)
assert.Equal(t, 0, result.Executed)
assert.Equal(t, 0, result.Failed, "waiting for approval is a skip, not a failure")
}

// With 4-eyes off the gate is unchanged.
func TestExecutableByScheduler_FourEyesOffUnchanged(t *testing.T) {
ctx := context.Background()
store := new(MockConfigStore)
assertMocksOnCleanup(t, store)
plan := &config.PurchasePlan{ID: "plan-gate", AutoPurchase: true, MaxStepCommitment: f64(1000)}
store.On("GetPurchasePlan", ctx, "plan-gate").Return(plan, nil)
store.On("GetGlobalConfig", ctx).Return(autoPurchaseOnCfg(), nil)
manager := &Manager{config: store}
exec := dueRecExec("notified", upfrontRec(10))
ok, err := manager.executableByScheduler(ctx, &exec)
require.NoError(t, err)
assert.True(t, ok)
}

// An empty step buys nothing, so 4-eyes does not park it in limbo.
func TestExecutableByScheduler_FourEyesOnEmptyStepStillRoutes(t *testing.T) {
ctx := context.Background()
store := new(MockConfigStore)
assertMocksOnCleanup(t, store)
cfg := autoPurchaseOnCfg()
cfg.RequireDifferentApprover = true
store.On("GetPurchasePlan", ctx, "plan-gate").Return(&config.PurchasePlan{ID: "plan-gate", AutoPurchase: true}, nil)
store.On("GetGlobalConfig", ctx).Return(cfg, nil)
manager := &Manager{config: store}
exec := dueRecExec("notified")
ok, err := manager.executableByScheduler(ctx, &exec)
require.NoError(t, err)
assert.True(t, ok)
}
2 changes: 1 addition & 1 deletion internal/purchase/messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (m *Manager) checkAutoExecuteGate(ctx context.Context, execution *config.Pu
return fmt.Errorf("AutoPurchase gate check failed for execution %s: %w", execution.ExecutionID, err)
}
if !eligible {
return fmt.Errorf("execution %s (status=%s source=%q) is not eligible for auto-execution: AutoPurchase=false, kill switch off, source=web, or recommendation-bearing row not yet notified - approve via the approval link",
return fmt.Errorf("execution %s (status=%s source=%q) is not eligible for auto-execution: AutoPurchase=false, kill switch off, source=web, 4-eyes on (it needs a human approval in the approval queue), or recommendation-bearing row not yet notified - approve it in the approval queue",
execution.ExecutionID, execution.Status, execution.Source)
}
return nil
Expand Down
Loading
Loading