diff --git a/.nextchanges/cli/aitools-install-error-category.md b/.nextchanges/cli/aitools-install-error-category.md new file mode 100644 index 00000000000..ce3594049fa --- /dev/null +++ b/.nextchanges/cli/aitools-install-error-category.md @@ -0,0 +1 @@ +* `databricks aitools install --output json` now reports an `error_category` for a failed or skipped install (per agent, and at the top level for a failure with no per-agent entry), giving coding agents and CI a stable classification of why an install did not complete. ([#6482](https://github.com/databricks/cli/pull/6482)) diff --git a/acceptance/experimental/aitools/skills/install-output-json-error/out.test.toml b/acceptance/experimental/aitools/skills/install-output-json-error/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/experimental/aitools/skills/install-output-json-error/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/aitools/skills/install-output-json-error/output.txt b/acceptance/experimental/aitools/skills/install-output-json-error/output.txt new file mode 100644 index 00000000000..db5a78e81e4 --- /dev/null +++ b/acceptance/experimental/aitools/skills/install-output-json-error/output.txt @@ -0,0 +1,11 @@ + +=== install --output json reports a top-level error category and exits non-zero +>>> [CLI] aitools install --skills-only --scope=global --agents=claude-code --skills=nonexistent --output json +{ + "scope": "global", + "agents": [], + "error": "skill \"nonexistent\" not found", + "error_category": "SKILL_NOT_FOUND" +} + +Exit code: 1 diff --git a/acceptance/experimental/aitools/skills/install-output-json-error/script b/acceptance/experimental/aitools/skills/install-output-json-error/script new file mode 100644 index 00000000000..e4b34a80ff3 --- /dev/null +++ b/acceptance/experimental/aitools/skills/install-output-json-error/script @@ -0,0 +1,10 @@ +# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks. +sethome home + +title "install --output json reports a top-level error category and exits non-zero" +# A --skills entry absent from the manifest fails before any agent is touched, so the +# failure has no per-agent entry and surfaces in the top-level error/error_category +# fields (agents stays empty). In JSON mode progress is silenced and root prints no +# duplicate "Error:" line, so stdout carries only the JSON document; the command +# still exits non-zero. +trace $CLI aitools install --skills-only --scope=global --agents=claude-code --skills=nonexistent --output json diff --git a/acceptance/experimental/aitools/skills/install-output-json-error/test.toml b/acceptance/experimental/aitools/skills/install-output-json-error/test.toml new file mode 100644 index 00000000000..edf1106ca21 --- /dev/null +++ b/acceptance/experimental/aitools/skills/install-output-json-error/test.toml @@ -0,0 +1,28 @@ +# Mock server replaces raw.githubusercontent.com for manifest + skill files. +Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST" +Env.DATABRICKS_SKILLS_REF = "test-ref" + +Ignore = [ + "home", +] + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The manifest has one skill; the script asks for a different one so the resolve +# fails with a skill-not-found error. +[[Server]] +Pattern = "GET /test-ref/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-01-01T00:00:00Z", + "skills": { + "test-stable": { + "version": "1.0.0", + "description": "Stable test skill", + "files": ["SKILL.md"], + "repo_dir": "skills" + } + } +} +''' diff --git a/cmd/aitools/categorize.go b/cmd/aitools/categorize.go new file mode 100644 index 00000000000..33bbf297fb5 --- /dev/null +++ b/cmd/aitools/categorize.go @@ -0,0 +1,44 @@ +package aitools + +import ( + "errors" + + "github.com/databricks/cli/libs/aitools/installer" + "github.com/databricks/cli/libs/telemetry/protos" +) + +func classifyInstallError(err error) protos.AitoolsErrorCategory { + if err == nil { + return protos.AitoolsErrorCategoryUnspecified + } + + if blocked, ok := errors.AsType[*installer.BlockedError](err); ok { + return blockedErrorCategory(blocked) + } + if skill, ok := errors.AsType[*installer.SkillError](err); ok { + return skillErrorCategory(skill) + } + return protos.AitoolsErrorCategoryUncategorized +} + +func skillErrorCategory(e *installer.SkillError) protos.AitoolsErrorCategory { + switch e.Reason { + case installer.ReasonSkillNotFound: + return protos.AitoolsErrorCategorySkillNotFound + case installer.ReasonVersionIncompatible: + return protos.AitoolsErrorCategoryVersionIncompatible + default: + return protos.AitoolsErrorCategoryUncategorized + } +} + +func blockedErrorCategory(e *installer.BlockedError) protos.AitoolsErrorCategory { + switch e.Reason { + case installer.ReasonCLINotOnPath: + return protos.AitoolsErrorCategoryCLINotOnPath + case installer.ReasonInstallFailed: + return protos.AitoolsErrorCategoryPluginInstallFailed + default: + return protos.AitoolsErrorCategoryUncategorized + } +} diff --git a/cmd/aitools/categorize_test.go b/cmd/aitools/categorize_test.go new file mode 100644 index 00000000000..2b6e337fea6 --- /dev/null +++ b/cmd/aitools/categorize_test.go @@ -0,0 +1,70 @@ +package aitools + +import ( + "errors" + "fmt" + "testing" + + "github.com/databricks/cli/libs/aitools/installer" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/stretchr/testify/assert" +) + +func TestClassifyInstallError(t *testing.T) { + cases := []struct { + name string + err error + want protos.AitoolsErrorCategory + }{ + { + name: "nil is success", + err: nil, + want: protos.AitoolsErrorCategoryUnspecified, + }, + { + name: "blocked cli not on path", + err: &installer.BlockedError{Agent: "claude-code", Reason: installer.ReasonCLINotOnPath}, + want: protos.AitoolsErrorCategoryCLINotOnPath, + }, + { + name: "blocked install failed", + err: &installer.BlockedError{Agent: "codex", Reason: installer.ReasonInstallFailed}, + want: protos.AitoolsErrorCategoryPluginInstallFailed, + }, + { + name: "blocked no plugin is uncategorized", + err: &installer.BlockedError{Agent: "codex", Reason: installer.ReasonNoPlugin}, + want: protos.AitoolsErrorCategoryUncategorized, + }, + { + name: "wrapped skill not found", + err: fmt.Errorf("resolve failed: %w", &installer.SkillError{Skill: "databricks", Reason: installer.ReasonSkillNotFound, Detail: "not found"}), + want: protos.AitoolsErrorCategorySkillNotFound, + }, + { + name: "version incompatible", + err: &installer.SkillError{Skill: "databricks", Reason: installer.ReasonVersionIncompatible, Detail: "requires CLI version 0.5 (running 0.4)"}, + want: protos.AitoolsErrorCategoryVersionIncompatible, + }, + { + name: "skill error with unknown reason is uncategorized", + err: &installer.SkillError{Skill: "databricks", Reason: "some-future-reason"}, + want: protos.AitoolsErrorCategoryUncategorized, + }, + { + name: "blocked error joined with another error is still classified", + err: errors.Join(&installer.BlockedError{Agent: "codex", Reason: installer.ReasonInstallFailed}, errors.New("other")), + want: protos.AitoolsErrorCategoryPluginInstallFailed, + }, + { + name: "unrecognized error is uncategorized", + err: errors.New("boom"), + want: protos.AitoolsErrorCategoryUncategorized, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, classifyInstallError(tc.err)) + }) + } +} diff --git a/cmd/aitools/install.go b/cmd/aitools/install.go index 1e360bb08bb..a70be95c1cc 100644 --- a/cmd/aitools/install.go +++ b/cmd/aitools/install.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -57,11 +58,12 @@ func (d delivery) String() string { // agentPlanItem is the resolved plan for one agent: what we'll do and why. type agentPlanItem struct { - agent *agents.Agent - delivery delivery - scope string // agent-native plugin scope (deliveryPlugin only) - reason string // why the agent is skipped (deliverySkip only) - explicit bool // named via --agents (blocking it is an error) + agent *agents.Agent + delivery delivery + scope string // agent-native plugin scope (deliveryPlugin only) + reason string // why the agent is skipped (deliverySkip only) + skipError protos.AitoolsErrorCategory // error category for the skip (deliverySkip only) + explicit bool // named via --agents (blocking it is an error) } // agentChoice is one row in the interactive agent picker. @@ -206,12 +208,16 @@ Supported agents: ` + strings.Join(agents.SupportedNames(), ", "), } } - defer logInstallEvent(ctx, plan, installOpts{ - Scope: opts.Scope, - Experimental: opts.IncludeExperimental, - }) + var outcomes []agentOutcome + var runErr error + defer func() { + logInstallEvent(ctx, plan, installOpts{ + Scope: opts.Scope, + Experimental: opts.IncludeExperimental, + }, classifyInstallError(topLevelFailure(runErr)), outcomes) + }() - outcomes, runErr := executePlan(ctx, src, plan, opts, jsonMode) + outcomes, runErr = executePlan(ctx, src, plan, opts, jsonMode) if jsonMode { if jerr := renderJSON(cmd.OutOrStdout(), buildInstallOutput(opts.Scope, outcomes, runErr)); jerr != nil { // Rendering failed, so the JSON the caller parses is broken. @@ -408,6 +414,7 @@ func planItemFor(a *agents.Agent, scope string, skillsOnly, explicit bool) agent if scope == installer.ScopeProject && !a.SupportsProjectScope { item.delivery = deliverySkip item.reason = "does not support project-scoped skills" + item.skipError = protos.AitoolsErrorCategoryUnsupportedScope } else { item.delivery = deliverySkills } @@ -416,6 +423,7 @@ func planItemFor(a *agents.Agent, scope string, skillsOnly, explicit bool) agent if !ok { item.delivery = deliverySkip item.reason = reason + item.skipError = protos.AitoolsErrorCategoryUnsupportedScope } else { item.delivery = deliveryPlugin item.scope = nativeScope @@ -442,13 +450,15 @@ func printPlanSummary(ctx context.Context, plan []agentPlanItem, scope string) { } // agentOutcome is one agent's result after executePlan: how the databricks -// tools were delivered (or attempted), and, when the agent did not succeed, a -// human-readable message for --output json. +// tools were delivered (or attempted), and, when the agent did not succeed, the +// failure category and a human-readable message for --output json. The category +// is what telemetry records; the message is local-only and never sent. type agentOutcome struct { - agent *agents.Agent - delivery delivery - status outcomeStatus - message string // set when skipped or failed + agent *agents.Agent + delivery delivery + status outcomeStatus + errorCategory protos.AitoolsErrorCategory // Unspecified when status == outcomeInstalled + message string // set when skipped or failed } type outcomeStatus string @@ -460,9 +470,9 @@ const ( ) // agentErrors wraps the failures of explicitly named agents, which are already -// reported in their per-agent outcomes. Wrapping lets the JSON layer tell a -// per-agent failure apart from a top-level failure that has no per-agent entry, -// so each is surfaced exactly once. +// reported in their per-agent outcomes. Wrapping lets the JSON and telemetry +// layers tell a per-agent failure apart from a top-level failure that has no +// per-agent entry, so each is surfaced exactly once. type agentErrors struct{ err error } func (e *agentErrors) Error() string { return e.err.Error() } @@ -470,7 +480,7 @@ func (e *agentErrors) Unwrap() error { return e.err } // topLevelFailure returns the run error when it has no per-agent entry, or nil // when the failure is already reported per agent — so a per-agent failure is not -// duplicated in the top-level error field. +// duplicated in the top-level error/errorCategory fields. func topLevelFailure(runErr error) error { if _, ok := errors.AsType[*agentErrors](runErr); ok { return nil @@ -503,7 +513,8 @@ func executePlan(ctx context.Context, src installer.ManifestSource, plan []agent if !quiet { installer.PrintInstallingFor(ctx, skillsAgents) } - // A skills install runs as a group; on failure the whole command fails. + // A skills install runs as a group; on failure the whole command fails and + // the top-level error category classifies it. if err := installSkillsForAgentsFn(ctx, src, skillsAgents, opts); err != nil { return outcomes, err } @@ -529,10 +540,11 @@ func executePlan(ctx context.Context, src installer.ManifestSource, plan []agent cmdio.LogString(ctx, cmdio.Yellow(ctx, fmt.Sprintf("Skipped %s: %v", it.agent.DisplayName, err))) } outcomes = append(outcomes, agentOutcome{ - agent: it.agent, - delivery: deliveryPlugin, - status: outcomeFailed, - message: err.Error(), + agent: it.agent, + delivery: deliveryPlugin, + status: outcomeFailed, + errorCategory: classifyInstallError(err), + message: err.Error(), }) if it.explicit { explicitErrs = append(explicitErrs, err) @@ -563,10 +575,11 @@ func executePlan(ctx context.Context, src installer.ManifestSource, plan []agent cmdio.LogString(ctx, cmdio.Yellow(ctx, "Skipped "+it.agent.DisplayName+": "+it.reason)) } outcomes = append(outcomes, agentOutcome{ - agent: it.agent, - delivery: deliverySkip, - status: outcomeSkipped, - message: it.reason, + agent: it.agent, + delivery: deliverySkip, + status: outcomeSkipped, + errorCategory: it.skipError, + message: it.reason, }) if it.explicit { explicitErrs = append(explicitErrs, fmt.Errorf("%s: %s", it.agent.DisplayName, it.reason)) @@ -591,17 +604,20 @@ type installOutput struct { Scope string `json:"scope"` Agents []agentResultJSON `json:"agents"` - // Error is a top-level failure message with no per-agent entry (e.g. a - // skills-group install failure); empty on success. It is local-only and never - // sent to telemetry. - Error string `json:"error,omitempty"` + // Error and ErrorCategory describe a top-level failure with no per-agent + // entry (e.g. a skills-group install failure); both empty on success. Error + // is the local-only message; ErrorCategory is the classification telemetry + // also records. + Error string `json:"error,omitempty"` + ErrorCategory string `json:"error_category,omitempty"` } type agentResultJSON struct { - Name string `json:"name"` - Delivery string `json:"delivery"` - Status string `json:"status"` - Message string `json:"message,omitempty"` + Name string `json:"name"` + Delivery string `json:"delivery"` + Status string `json:"status"` + ErrorCategory string `json:"error_category,omitempty"` + Message string `json:"message,omitempty"` } func buildInstallOutput(scope string, outcomes []agentOutcome, runErr error) installOutput { @@ -613,6 +629,9 @@ func buildInstallOutput(scope string, outcomes []agentOutcome, runErr error) ins Status: string(o.status), Message: o.message, } + if o.errorCategory != "" { + entry.ErrorCategory = string(o.errorCategory) + } out.Agents = append(out.Agents, entry) } // A top-level failure (skills-group install, ref lookup, plugin recording) @@ -621,6 +640,7 @@ func buildInstallOutput(scope string, outcomes []agentOutcome, runErr error) ins // stay in the agents entries above and are not repeated here. if e := topLevelFailure(runErr); e != nil { out.Error = e.Error() + out.ErrorCategory = string(classifyInstallError(e)) } return out } diff --git a/cmd/aitools/install_test.go b/cmd/aitools/install_test.go index 1972262c066..d3f0b38fb25 100644 --- a/cmd/aitools/install_test.go +++ b/cmd/aitools/install_test.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -255,6 +256,7 @@ func TestExecutePlanSkipBlockedPluginExit0(t *testing.T) { require.NoError(t, err) require.Len(t, outcomes, 1) assert.Equal(t, outcomeFailed, outcomes[0].status) + assert.Equal(t, protos.AitoolsErrorCategoryCLINotOnPath, outcomes[0].errorCategory) // Explicit (--agents) blocked install is an error. planExplicit := buildPlan([]*agents.Agent{claude}, installer.ScopeGlobal, false, true) @@ -394,7 +396,7 @@ func TestInstallExplicitAgentWorksUndetected(t *testing.T) { assert.Equal(t, agents.NameCodex, (*plugins)[0].agent) } -func TestInstallOutputJSON(t *testing.T) { +func TestInstallOutputJSONReportsErrorCategories(t *testing.T) { tmp := t.TempDir() t.Setenv("HOME", tmp) t.Setenv("USERPROFILE", tmp) @@ -430,9 +432,11 @@ func TestInstallOutputJSON(t *testing.T) { assert.Equal(t, agents.NameCodex, got.Agents[0].Name) assert.Equal(t, deliveryPlugin.String(), got.Agents[0].Delivery) assert.Equal(t, string(outcomeFailed), got.Agents[0].Status) + assert.Equal(t, string(protos.AitoolsErrorCategoryPluginInstallFailed), got.Agents[0].ErrorCategory) // A per-agent failure stays in the agent entry; it is not repeated in the - // top-level error field. + // top-level error fields. assert.Empty(t, got.Error) + assert.Empty(t, got.ErrorCategory) } // TestInstallOutputJSONThroughRoot runs a failing `install --output json` through @@ -480,7 +484,7 @@ func TestInstallOutputJSONTopLevelFailure(t *testing.T) { orig := installSkillsForAgentsFn t.Cleanup(func() { installSkillsForAgentsFn = orig }) installSkillsForAgentsFn = func(context.Context, installer.ManifestSource, []*agents.Agent, installer.InstallOptions) error { - return errors.New(`skill "databricks" not found`) + return &installer.SkillError{Skill: "databricks", Reason: installer.ReasonSkillNotFound, Detail: "not found"} } var out bytes.Buffer @@ -501,6 +505,7 @@ func TestInstallOutputJSONTopLevelFailure(t *testing.T) { require.NoError(t, json.Unmarshal(out.Bytes(), &got)) assert.Empty(t, got.Agents) assert.Contains(t, got.Error, "databricks") + assert.Equal(t, string(protos.AitoolsErrorCategorySkillNotFound), got.ErrorCategory) } func TestInstallOutputJSONRequiresNonInteractiveFlags(t *testing.T) { diff --git a/cmd/aitools/telemetry.go b/cmd/aitools/telemetry.go index 1e85d5dfbf7..09eed34f3b2 100644 --- a/cmd/aitools/telemetry.go +++ b/cmd/aitools/telemetry.go @@ -1,6 +1,7 @@ package aitools import ( + "cmp" "context" "slices" @@ -30,16 +31,45 @@ type installOpts struct { } // logInstallEvent buffers an install event; cmd/root uploads it at exit. -func logInstallEvent(ctx context.Context, plan []agentPlanItem, opts installOpts) { +// errCategory is the top-level command outcome (Unspecified on success or when +// the failure is reported per agent), and outcomes carries the per-agent results +// so a skipped-with-warning failure is still recorded even when the command +// exits 0. +func logInstallEvent(ctx context.Context, plan []agentPlanItem, opts installOpts, errCategory protos.AitoolsErrorCategory, outcomes []agentOutcome) { telemetry.Log(ctx, protos.DatabricksCliLog{ AitoolsInstallEvent: &protos.AitoolsInstallEvent{ - Agents: agentsField(plan), - Scope: scopeType(opts.Scope), - Experimental: opts.Experimental, + Agents: agentsField(plan), + Scope: scopeType(opts.Scope), + Experimental: opts.Experimental, + ErrorCategory: errCategory, + AgentResults: agentResultsField(outcomes), }, }) } +// agentResultsField returns the per-agent failure/skip categories, one entry per +// non-successful agent, sorted by agent enum for stable output. Successful +// agents produce no entry. +func agentResultsField(outcomes []agentOutcome) []protos.AitoolsAgentResult { + var out []protos.AitoolsAgentResult + for _, o := range outcomes { + // Successful agents produce no entry; only failed/skipped outcomes carry a + // category. Keying on status (rather than errorCategory == "") avoids a trap + // where a future Unspecified category on a success would emit a bogus entry. + if o.status == outcomeInstalled { + continue + } + out = append(out, protos.AitoolsAgentResult{ + Agent: agentType(o.agent.Name), + ErrorCategory: o.errorCategory, + }) + } + slices.SortFunc(out, func(a, b protos.AitoolsAgentResult) int { + return cmp.Compare(a.Agent, b.Agent) + }) + return out +} + // agentsField returns the deduped agent enums from the plan, sorted so the // same set of agents always produces the same array on the analytics side. func agentsField(plan []agentPlanItem) []protos.AitoolsAgentType { diff --git a/cmd/aitools/telemetry_test.go b/cmd/aitools/telemetry_test.go index 85705161955..160bb4fe98e 100644 --- a/cmd/aitools/telemetry_test.go +++ b/cmd/aitools/telemetry_test.go @@ -84,3 +84,25 @@ func TestScopeType(t *testing.T) { assert.Equal(t, protos.AitoolsInstallScopeProject, scopeType(installer.ScopeProject)) assert.Equal(t, protos.AitoolsInstallScopeUnspecified, scopeType("")) } + +func TestAgentResultsField(t *testing.T) { + claude := &agents.Agent{Name: agents.NameClaudeCode} + codex := &agents.Agent{Name: agents.NameCodex} + cursor := &agents.Agent{Name: agents.NameCursor} + + outcomes := []agentOutcome{ + // Successful agents produce no entry; production leaves errorCategory unset. + {agent: cursor, status: outcomeInstalled}, + {agent: codex, status: outcomeFailed, errorCategory: protos.AitoolsErrorCategoryPluginInstallFailed}, + {agent: claude, status: outcomeSkipped, errorCategory: protos.AitoolsErrorCategoryUnsupportedScope}, + } + + // Sorted by agent enum, only non-successful agents included. + want := []protos.AitoolsAgentResult{ + {Agent: protos.AitoolsAgentTypeClaudeCode, ErrorCategory: protos.AitoolsErrorCategoryUnsupportedScope}, + {Agent: protos.AitoolsAgentTypeCodex, ErrorCategory: protos.AitoolsErrorCategoryPluginInstallFailed}, + } + assert.Equal(t, want, agentResultsField(outcomes)) + + assert.Nil(t, agentResultsField(nil)) +} diff --git a/libs/aitools/installer/errors.go b/libs/aitools/installer/errors.go new file mode 100644 index 00000000000..774b327df1b --- /dev/null +++ b/libs/aitools/installer/errors.go @@ -0,0 +1,31 @@ +package installer + +import "fmt" + +// SkillError reports that a skill named via --skills could not be resolved from +// the manifest. Reason drives telemetry categorization (the command layer maps +// it with errors.AsType); Detail is the human-readable remainder of the message. +// Building the message from fields keeps a classification tag out of the +// user-facing string, so the error is stated exactly once. +type SkillError struct { + Skill string + Reason string + Detail string +} + +// Reasons a --skills entry can fail to resolve. +const ( + // ReasonSkillNotFound: the named skill is absent from the resolved manifest. + ReasonSkillNotFound = "skill-not-found" + // ReasonVersionIncompatible: the skill requires a newer CLI than the one running. + ReasonVersionIncompatible = "version-incompatible" +) + +func (e *SkillError) Error() string { + // Detail is the human-readable remainder; fall back to Reason when it is empty + // so the message stays self-describing (mirrors BlockedError.Error()). + if e.Detail != "" { + return fmt.Sprintf("skill %q %s", e.Skill, e.Detail) + } + return fmt.Sprintf("skill %q %s", e.Skill, e.Reason) +} diff --git a/libs/aitools/installer/installer.go b/libs/aitools/installer/installer.go index 168c849f285..accaa086269 100644 --- a/libs/aitools/installer/installer.go +++ b/libs/aitools/installer/installer.go @@ -472,7 +472,7 @@ func resolveSkills(ctx context.Context, skills map[string]SkillMeta, opts Instal for _, name := range opts.SpecificSkills { meta, ok := skills[name] if !ok { - return nil, fmt.Errorf("skill %q not found", name) + return nil, &SkillError{Skill: name, Reason: ReasonSkillNotFound, Detail: "not found"} } candidates[name] = meta } @@ -492,7 +492,7 @@ func resolveSkills(ctx context.Context, skills map[string]SkillMeta, opts Instal if meta.MinCLIVer != "" && !isDev && semver.Compare("v"+cliVersion, "v"+meta.MinCLIVer) < 0 { if isSpecific { - return nil, fmt.Errorf("skill %q requires CLI version %s (running %s)", name, meta.MinCLIVer, cliVersion) + return nil, &SkillError{Skill: name, Reason: ReasonVersionIncompatible, Detail: fmt.Sprintf("requires CLI version %s (running %s)", meta.MinCLIVer, cliVersion)} } log.Warnf(ctx, "Skipping %s: requires CLI version %s (running %s)", name, meta.MinCLIVer, cliVersion) continue diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index 42d2b912490..7bd034b3665 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -28,6 +28,31 @@ const ( AitoolsInstallScopeProject AitoolsInstallScope = "PROJECT" ) +// AitoolsErrorCategory classifies why an `aitools install` run, or one agent +// within it, failed. It mirrors AitoolsErrorCategory.Type in the databricks_cli +// lumberjack proto and lets us aggregate install failures without sending any +// user-authored error text. AitoolsErrorCategoryUncategorized absorbs failures +// a newer CLI has not classified yet. +type AitoolsErrorCategory string + +const ( + AitoolsErrorCategoryUnspecified AitoolsErrorCategory = "TYPE_UNSPECIFIED" + AitoolsErrorCategoryVersionIncompatible AitoolsErrorCategory = "VERSION_INCOMPATIBLE" + AitoolsErrorCategorySkillNotFound AitoolsErrorCategory = "SKILL_NOT_FOUND" + AitoolsErrorCategoryCLINotOnPath AitoolsErrorCategory = "CLI_NOT_ON_PATH" + AitoolsErrorCategoryPluginInstallFailed AitoolsErrorCategory = "PLUGIN_INSTALL_FAILED" + AitoolsErrorCategoryUnsupportedScope AitoolsErrorCategory = "UNSUPPORTED_SCOPE" + AitoolsErrorCategoryUncategorized AitoolsErrorCategory = "UNCATEGORIZED" +) + +// AitoolsAgentResult records one agent's failed or skipped outcome within an +// install run. Successful agents produce no entry, and Category never carries +// user-authored text. +type AitoolsAgentResult struct { + Agent AitoolsAgentType `json:"agent"` + ErrorCategory AitoolsErrorCategory `json:"error_category"` +} + // AitoolsInstallEvent is emitted on every execution of the `databricks aitools // install` command. type AitoolsInstallEvent struct { @@ -39,4 +64,14 @@ type AitoolsInstallEvent struct { // Whether the user passed --experimental to include experimental skills. Experimental bool `json:"experimental,omitempty"` + + // ErrorCategory is the top-level command outcome: the category of the error + // that failed the run, or Unspecified when the command succeeded. It captures + // failures that have no per-agent entry (e.g. a skills-group install failure). + // Always populated (Unspecified on success), so no omitempty. + ErrorCategory AitoolsErrorCategory `json:"error_category"` + + // AgentResults records the per-agent failure/skip categories, one entry per + // non-successful agent. Empty when every targeted agent succeeded. + AgentResults []AitoolsAgentResult `json:"agent_results,omitempty"` }