aitools: categorize install errors

.nextchanges/cli/aitools-install-error-category.md

@@ -0,0 +1 @@
+`databricks aitools install --output json` now reports an `errorCategory` for a failed or skipped install (per agent, and at the top level for a failure with no per-agent entry), giving coding agents and CI a stable classification of why an install did not complete ([#6482](https://github.com/databricks/cli/pull/6482)).

acceptance/experimental/aitools/skills/install-output-json-error/output.txt

@@ -0,0 +1,14 @@
+
+=== install --output json reports a top-level error category and exits non-zero
+>>> [CLI] experimental aitools install --skills-only --scope=global --agents=claude-code --skills=nonexistent --output json
+Command "install" is deprecated, use "databricks aitools install" instead.
+Using skills version test-ref
+Fetching skills manifest...
+{
+  "scope": "global",
+  "agents": [],
+  "error": "skill \"nonexistent\" not found",
+  "errorCategory": "SKILL_NOT_FOUND"
+}
+
+Exit code: 1

acceptance/experimental/aitools/skills/install-output-json-error/script

@@ -0,0 +1,9 @@
+# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks.
+sethome home
+
+title "install --output json reports a top-level error category and exits non-zero"
+# A --skills entry absent from the manifest fails before any agent is touched, so the
+# failure has no per-agent entry and surfaces in the top-level error/errorCategory
+# fields (agents stays empty). The command exits non-zero, but root prints no
+# duplicate "Error:" line to stderr because the JSON already reported the failure.
+trace $CLI experimental aitools install --skills-only --scope=global --agents=claude-code --skills=nonexistent --output json

acceptance/experimental/aitools/skills/install-output-json-error/test.toml

@@ -0,0 +1,28 @@
+# Mock server replaces raw.githubusercontent.com for manifest + skill files.
+Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST"
+Env.DATABRICKS_SKILLS_REF = "test-ref"
+
+Ignore = [
+    "home",
+]
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+# The manifest has one skill; the script asks for a different one so the resolve
+# fails with a skill-not-found error.
+[[Server]]
+Pattern = "GET /test-ref/manifest.json"
+Response.Body = '''
+{
+  "version": "2",
+  "updated_at": "2026-01-01T00:00:00Z",
+  "skills": {
+    "test-stable": {
+      "version": "1.0.0",
+      "description": "Stable test skill",
+      "files": ["SKILL.md"],
+      "repo_dir": "skills"
+    }
+  }
+}
+'''

cmd/aitools/categorize.go

@@ -0,0 +1,44 @@
+package aitools
+
+import (
+	"errors"
+
+	"github.com/databricks/cli/libs/aitools/installer"
+	"github.com/databricks/cli/libs/telemetry/protos"
+)
+
+func classifyInstallError(err error) protos.AitoolsErrorCategory {
+	if err == nil {
+		return protos.AitoolsErrorCategoryUnspecified
+	}
+
+	if blocked, ok := errors.AsType[*installer.BlockedError](err); ok {
+		return blockedErrorCategory(blocked)
+	}
+	if skill, ok := errors.AsType[*installer.SkillError](err); ok {
+		return skillErrorCategory(skill)
+	}
+	return protos.AitoolsErrorCategoryUncategorized
+}
+
+func skillErrorCategory(e *installer.SkillError) protos.AitoolsErrorCategory {
+	switch e.Reason {
+	case installer.ReasonSkillNotFound:
+		return protos.AitoolsErrorCategorySkillNotFound
+	case installer.ReasonVersionIncompatible:
+		return protos.AitoolsErrorCategoryVersionIncompatible
+	default:
+		return protos.AitoolsErrorCategoryUncategorized
+	}
+}
+
+func blockedErrorCategory(e *installer.BlockedError) protos.AitoolsErrorCategory {
+	switch e.Reason {
+	case installer.ReasonCLINotOnPath:
+		return protos.AitoolsErrorCategoryCLINotOnPath
+	case installer.ReasonInstallFailed:
+		return protos.AitoolsErrorCategoryPluginInstallFailed
+	default:
+		return protos.AitoolsErrorCategoryUncategorized
+	}
+}

cmd/aitools/categorize_test.go

@@ -0,0 +1,70 @@
+package aitools
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/databricks/cli/libs/aitools/installer"
+	"github.com/databricks/cli/libs/telemetry/protos"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestClassifyInstallError(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+		want protos.AitoolsErrorCategory
+	}{
+		{
+			name: "nil is success",
+			err:  nil,
+			want: protos.AitoolsErrorCategoryUnspecified,
+		},
+		{
+			name: "blocked cli not on path",
+			err:  &installer.BlockedError{Agent: "claude-code", Reason: installer.ReasonCLINotOnPath},
+			want: protos.AitoolsErrorCategoryCLINotOnPath,
+		},
+		{
+			name: "blocked install failed",
+			err:  &installer.BlockedError{Agent: "codex", Reason: installer.ReasonInstallFailed},
+			want: protos.AitoolsErrorCategoryPluginInstallFailed,
+		},
+		{
+			name: "blocked no plugin is uncategorized",
+			err:  &installer.BlockedError{Agent: "codex", Reason: installer.ReasonNoPlugin},
+			want: protos.AitoolsErrorCategoryUncategorized,
+		},
+		{
+			name: "wrapped skill not found",
+			err:  fmt.Errorf("resolve failed: %w", &installer.SkillError{Skill: "databricks", Reason: installer.ReasonSkillNotFound, Detail: "not found"}),
+			want: protos.AitoolsErrorCategorySkillNotFound,
+		},
+		{
+			name: "version incompatible",
+			err:  &installer.SkillError{Skill: "databricks", Reason: installer.ReasonVersionIncompatible, Detail: "requires CLI version 0.5 (running 0.4)"},
+			want: protos.AitoolsErrorCategoryVersionIncompatible,
+		},
+		{
+			name: "skill error with unknown reason is uncategorized",
+			err:  &installer.SkillError{Skill: "databricks", Reason: "some-future-reason"},
+			want: protos.AitoolsErrorCategoryUncategorized,
+		},
+		{
+			name: "blocked error joined with another error is still classified",
+			err:  errors.Join(&installer.BlockedError{Agent: "codex", Reason: installer.ReasonInstallFailed}, errors.New("other")),
+			want: protos.AitoolsErrorCategoryPluginInstallFailed,
+		},
+		{
+			name: "unrecognized error is uncategorized",
+			err:  errors.New("boom"),
+			want: protos.AitoolsErrorCategoryUncategorized,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, classifyInstallError(tc.err))
+		})
+	}
+}

cmd/aitools/install.go

@@ -13,6 +13,7 @@ import (
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/flags"
 	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/telemetry/protos"
 	"github.com/spf13/cobra"
 )
 
@@ -57,11 +58,12 @@ func (d delivery) String() string {
 
 // agentPlanItem is the resolved plan for one agent: what we'll do and why.
 type agentPlanItem struct {
-	agent    *agents.Agent
-	delivery delivery
-	scope    string // agent-native plugin scope (deliveryPlugin only)
-	reason   string // why the agent is skipped (deliverySkip only)
-	explicit bool   // named via --agents (blocking it is an error)
+	agent     *agents.Agent
+	delivery  delivery
+	scope     string                      // agent-native plugin scope (deliveryPlugin only)
+	reason    string                      // why the agent is skipped (deliverySkip only)
+	skipError protos.AitoolsErrorCategory // error category for the skip (deliverySkip only)
+	explicit  bool                        // named via --agents (blocking it is an error)
 }
 
 // agentChoice is one row in the interactive agent picker.
@@ -200,12 +202,16 @@ Supported agents: ` + strings.Join(agents.SupportedNames(), ", "),
 				}
 			}
 
-			defer logInstallEvent(ctx, plan, installOpts{
-				Scope:        opts.Scope,
-				Experimental: opts.IncludeExperimental,
-			})
+			var outcomes []agentOutcome
+			var runErr error
+			defer func() {
+				logInstallEvent(ctx, plan, installOpts{
+					Scope:        opts.Scope,
+					Experimental: opts.IncludeExperimental,
+				}, classifyInstallError(topLevelFailure(runErr)), outcomes)
+			}()
 
-			outcomes, runErr := executePlan(ctx, src, plan, opts, jsonMode)
+			outcomes, runErr = executePlan(ctx, src, plan, opts, jsonMode)
 			if jsonMode {
 				if jerr := renderJSON(cmd.OutOrStdout(), buildInstallOutput(opts.Scope, outcomes, runErr)); jerr != nil {
 					// Rendering failed, so the JSON the caller parses is broken.
@@ -402,6 +408,7 @@ func planItemFor(a *agents.Agent, scope string, skillsOnly, explicit bool) agent
 		if scope == installer.ScopeProject && !a.SupportsProjectScope {
 			item.delivery = deliverySkip
 			item.reason = "does not support project-scoped skills"
+			item.skipError = protos.AitoolsErrorCategoryUnsupportedScope
 		} else {
 			item.delivery = deliverySkills
 		}
@@ -410,6 +417,7 @@ func planItemFor(a *agents.Agent, scope string, skillsOnly, explicit bool) agent
 		if !ok {
 			item.delivery = deliverySkip
 			item.reason = reason
+			item.skipError = protos.AitoolsErrorCategoryUnsupportedScope
 		} else {
 			item.delivery = deliveryPlugin
 			item.scope = nativeScope
@@ -436,13 +444,15 @@ func printPlanSummary(ctx context.Context, plan []agentPlanItem, scope string) {
 }
 
 // agentOutcome is one agent's result after executePlan: how the databricks
-// tools were delivered (or attempted), and, when the agent did not succeed, a
-// human-readable message for --output json.
+// tools were delivered (or attempted), and, when the agent did not succeed, the
+// failure category and a human-readable message for --output json. The category
+// is what telemetry records; the message is local-only and never sent.
 type agentOutcome struct {
-	agent    *agents.Agent
-	delivery delivery
-	status   outcomeStatus
-	message  string // set when skipped or failed
+	agent         *agents.Agent
+	delivery      delivery
+	status        outcomeStatus
+	errorCategory protos.AitoolsErrorCategory // Unspecified when status == outcomeInstalled
+	message       string                      // set when skipped or failed
 }
 
 type outcomeStatus string
@@ -454,17 +464,17 @@ const (
 )
 
 // agentErrors wraps the failures of explicitly named agents, which are already
-// reported in their per-agent outcomes. Wrapping lets the JSON layer tell a
-// per-agent failure apart from a top-level failure that has no per-agent entry,
-// so each is surfaced exactly once.
+// reported in their per-agent outcomes. Wrapping lets the JSON and telemetry
+// layers tell a per-agent failure apart from a top-level failure that has no
+// per-agent entry, so each is surfaced exactly once.
 type agentErrors struct{ err error }
 
 func (e *agentErrors) Error() string { return e.err.Error() }
 func (e *agentErrors) Unwrap() error { return e.err }
 
 // topLevelFailure returns the run error when it has no per-agent entry, or nil
 // when the failure is already reported per agent — so a per-agent failure is not
-// duplicated in the top-level error field.
+// duplicated in the top-level error/errorCategory fields.
 func topLevelFailure(runErr error) error {
 	if _, ok := errors.AsType[*agentErrors](runErr); ok {
 		return nil
@@ -497,7 +507,8 @@ func executePlan(ctx context.Context, src installer.ManifestSource, plan []agent
 		if !quiet {
 			installer.PrintInstallingFor(ctx, skillsAgents)
 		}
-		// A skills install runs as a group; on failure the whole command fails.
+		// A skills install runs as a group; on failure the whole command fails and
+		// the top-level error category classifies it.
 		if err := installSkillsForAgentsFn(ctx, src, skillsAgents, opts); err != nil {
 			return outcomes, err
 		}
@@ -523,10 +534,11 @@ func executePlan(ctx context.Context, src installer.ManifestSource, plan []agent
 					cmdio.LogString(ctx, cmdio.Yellow(ctx, fmt.Sprintf("Skipped %s: %v", it.agent.DisplayName, err)))
 				}
 				outcomes = append(outcomes, agentOutcome{
-					agent:    it.agent,
-					delivery: deliveryPlugin,
-					status:   outcomeFailed,
-					message:  err.Error(),
+					agent:         it.agent,
+					delivery:      deliveryPlugin,
+					status:        outcomeFailed,
+					errorCategory: classifyInstallError(err),
+					message:       err.Error(),
 				})
 				if it.explicit {
 					explicitErrs = append(explicitErrs, err)
@@ -557,10 +569,11 @@ func executePlan(ctx context.Context, src installer.ManifestSource, plan []agent
 			cmdio.LogString(ctx, cmdio.Yellow(ctx, "Skipped "+it.agent.DisplayName+": "+it.reason))
 		}
 		outcomes = append(outcomes, agentOutcome{
-			agent:    it.agent,
-			delivery: deliverySkip,
-			status:   outcomeSkipped,
-			message:  it.reason,
+			agent:         it.agent,
+			delivery:      deliverySkip,
+			status:        outcomeSkipped,
+			errorCategory: it.skipError,
+			message:       it.reason,
 		})
 		if it.explicit {
 			explicitErrs = append(explicitErrs, fmt.Errorf("%s: %s", it.agent.DisplayName, it.reason))
@@ -585,17 +598,20 @@ type installOutput struct {
 	Scope  string            `json:"scope"`
 	Agents []agentResultJSON `json:"agents"`
 
-	// Error is a top-level failure message with no per-agent entry (e.g. a
-	// skills-group install failure); empty on success. It is local-only and never
-	// sent to telemetry.
-	Error string `json:"error,omitempty"`
+	// Error and ErrorCategory describe a top-level failure with no per-agent
+	// entry (e.g. a skills-group install failure); both empty on success. Error
+	// is the local-only message; ErrorCategory is the classification telemetry
+	// also records.
+	Error         string `json:"error,omitempty"`
+	ErrorCategory string `json:"errorCategory,omitempty"`
 }
 
 type agentResultJSON struct {
-	Name     string `json:"name"`
-	Delivery string `json:"delivery"`
-	Status   string `json:"status"`
-	Message  string `json:"message,omitempty"`
+	Name          string `json:"name"`
+	Delivery      string `json:"delivery"`
+	Status        string `json:"status"`
+	ErrorCategory string `json:"errorCategory,omitempty"`
+	Message       string `json:"message,omitempty"`
 }
 
 func buildInstallOutput(scope string, outcomes []agentOutcome, runErr error) installOutput {
@@ -607,6 +623,9 @@ func buildInstallOutput(scope string, outcomes []agentOutcome, runErr error) ins
 			Status:   string(o.status),
 			Message:  o.message,
 		}
+		if o.errorCategory != "" {
+			entry.ErrorCategory = string(o.errorCategory)
+		}
 		out.Agents = append(out.Agents, entry)
 	}
 	// A top-level failure (skills-group install, ref lookup, plugin recording)
@@ -615,6 +634,7 @@ func buildInstallOutput(scope string, outcomes []agentOutcome, runErr error) ins
 	// stay in the agents entries above and are not repeated here.
 	if e := topLevelFailure(runErr); e != nil {
 		out.Error = e.Error()
+		out.ErrorCategory = string(classifyInstallError(e))
 	}
 	return out
 }