diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9a51624..7eb358b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -238,10 +238,10 @@ jobs: coverage=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | tr -d '%' | tail -1) echo "Coverage: ${coverage}%" echo "COVERAGE=${coverage}" >> "$GITHUB_ENV" - - name: Coverage threshold (minimum 60%) + - name: Coverage threshold (minimum 65%) run: | - if (( $(echo "${COVERAGE} < 60" | bc -l) )); then - echo "::error::Coverage ${COVERAGE}% is below minimum 60%" + if (( $(echo "${COVERAGE} < 65" | bc -l) )); then + echo "::error::Coverage ${COVERAGE}% is below minimum 65%" exit 1 fi - name: Upload coverage @@ -372,6 +372,30 @@ jobs: npm install -g markdownlint-cli2 markdownlint-cli2 '**/*.md' + # ------------------------------------------------------------------------- + # 9. API reference generation — build HTML docs from the OpenAPI spec. + # ------------------------------------------------------------------------- + api-reference: + name: api reference + runs-on: ubuntu-latest + needs: [vet] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Validate OpenAPI spec + run: | + npm install -g @redocly/cli + redocly lint api/openapi.yaml + - name: Generate API reference (HTML) + run: | + npm install -g redoc-cli + redoc-cli bundle api/openapi.yaml -o api/reference.html + - name: Upload API reference artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: api-reference + path: api/reference.html + retention-days: 30 + # ------------------------------------------------------------------------- # Dead code detection. # ------------------------------------------------------------------------- @@ -406,6 +430,7 @@ jobs: echo '```' } >>"$GITHUB_STEP_SUMMARY" + # ------------------------------------------------------------------------- # Duplication detection — jscpd. # ------------------------------------------------------------------------- diff --git a/Dockerfile.daemon b/Dockerfile.daemon new file mode 100644 index 00000000..dab95701 --- /dev/null +++ b/Dockerfile.daemon @@ -0,0 +1,62 @@ +# Dockerfile for the Hawk daemon (background HTTP server). +# The binary is identical to the CLI image — this Dockerfile just sets the +# daemon as the default entrypoint and exposes the daemon port. +# +# Build: docker build -f Dockerfile.daemon -t hawk-daemon . +# Run: docker run -p 4590:4590 -e HAWK_DAEMON_API_KEY=... hawk-daemon +FROM golang:1.26.5-alpine AS builder + +RUN apk upgrade --no-cache && \ + apk add --no-cache git ca-certificates tzdata + +WORKDIR /build + +ENV GOPRIVATE=github.com/GrayCodeAI/* \ + GONOSUMDB=github.com/GrayCodeAI/* \ + GONOSUMCHECK=1 + +ARG VERSION=dev +ARG COMMIT=none +ARG BUILD_DATE=unknown + +COPY . . + +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + rm -f go.work go.work.sum && \ + { echo "go 1.26.5"; echo; echo "use ."; echo; echo "replace ("; \ + for repo in hawk-core-contracts eyrie inspect sight tok trace yaad; do \ + echo " github.com/GrayCodeAI/${repo} => ./external/${repo}"; \ + done; echo ")"; } > go.work && \ + CGO_ENABLED=0 GOOS=linux go build -trimpath \ + -ldflags="-s -w \ + -X main.Version=${VERSION} \ + -X main.Commit=${COMMIT} \ + -X main.BuildDate=${BUILD_DATE}" \ + -o hawk ./cmd/hawk + +FROM alpine:3.23.5 + +RUN apk upgrade --no-cache && \ + apk add --no-cache ca-certificates git bash curl tini && \ + adduser -D -u 1000 -h /home/hawk hawk + +# Create state directory for daemon logs, API key, and audit log. +RUN mkdir -p /home/hawk/.hawk/state && \ + chown -R hawk:hawk /home/hawk/.hawk + +COPY --from=builder /build/hawk /usr/local/bin/hawk +COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +COPY packaging/systemd/hawk-daemon.service /etc/systemd/system/hawk-daemon.service + +USER hawk +WORKDIR /workspace + +EXPOSE 4590 + +# Health check probes the daemon's health endpoint. +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD curl -sf http://127.0.0.1:4590/v1/health || exit 1 + +ENTRYPOINT ["tini", "--", "hawk", "daemon", "start"] +CMD ["--host", "0.0.0.0", "--port", "4590"] diff --git a/Makefile b/Makefile index 48277b4a..7d7c07ac 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ GORELEASER := $(GOBIN_DIR)/goreleaser # Phony declarations (alphabetical). # --------------------------------------------------------------------------- .PHONY: all bench boundaries build check-replace ci clean contracts-guard ecosystem-guard eyrie-client-guard eyrie-engine-guard peer-guard internal-layers-guard package-boundaries-guard submodule-release-parity cover cover-new fmt help install lint lint-fix \ - release security setup smoke path sync-external test test-10x test-live test-new test-race tidy version vet + release security setup smoke path sync-external test test-10x test-live test-new test-race tidy version vet api-docs api-validate check-replace: ## Fail if go.mod has local replace directives (run before tagging) @bash scripts/check-no-replace-directives.sh @@ -83,6 +83,15 @@ cover: ## Generate a coverage report (coverage.out + coverage.html). cover-new: ## Coverage report for Round 2 ecosystem packages only. go test -cover -timeout=30s ./internal/safewrite/... ./internal/jsonc/... ./internal/providers/... ./internal/session/... ./internal/permissions/... +api-docs: ## Generate HTML API reference from OpenAPI spec. + @command -v redoc-cli >/dev/null 2>&1 || (echo "install: npm install -g redoc-cli" && exit 1) + redoc-cli bundle api/openapi.yaml -o api/reference.html + @echo "API reference generated: api/reference.html" + +api-validate: ## Validate the OpenAPI spec. + @command -v @redocly/cli >/dev/null 2>&1 || (echo "install: npm install -g @redocly/cli" && exit 1) + @redocly lint api/openapi.yaml + bench: ## Run benchmarks. go test ./... -bench=. -benchmem -count=3 -timeout=300s @@ -143,7 +152,7 @@ tidy: ## Sync workspace modules and verify checksums. # --------------------------------------------------------------------------- # Composite gate used by CI and pre-push. # --------------------------------------------------------------------------- -ci: tidy fmt vet boundaries lint test-race security ## Run everything CI runs. +ci: tidy fmt vet boundaries lint test-race security api-validate ## Run everything CI runs. @echo "All CI checks passed." smoke: ## Quick build + doctor + ecosystem verification. diff --git a/api/openapi.yaml b/api/openapi.yaml index 3da4e969..8f1188ea 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -745,6 +745,31 @@ paths: schema: $ref: "#/components/schemas/Error" + /v1/metrics: + get: + tags: [stats] + summary: Daemon metrics in Prometheus exposition format + description: | + Returns daemon-level metrics (request counts, concurrency usage, + active sessions) as Prometheus text exposition format. + Use `?format=json` for JSON output. + responses: + "200": + description: Metrics output + content: + text/plain: + schema: + type: string + application/json: + schema: + type: object + "401": + description: Unauthorized + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /v1/review: post: tags: [review] diff --git a/cmd/chat_tools.go b/cmd/chat_tools.go index dcfa3bb7..b0157819 100644 --- a/cmd/chat_tools.go +++ b/cmd/chat_tools.go @@ -96,6 +96,7 @@ func optionalTools() []tool.Tool { tool.TaskGetTool{}, tool.TaskListTool{}, tool.TaskUpdateTool{}, + tool.TaskRunTool{}, tool.SleepTool{}, tool.CronCreateTool{}, tool.CronDeleteTool{}, diff --git a/cmd/daemon.go b/cmd/daemon.go index ff67393f..19ad58bd 100644 --- a/cmd/daemon.go +++ b/cmd/daemon.go @@ -6,7 +6,7 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" + "log/slog" "os" "os/signal" "path/filepath" @@ -22,15 +22,21 @@ import ( "github.com/GrayCodeAI/hawk/internal/multiagent/agents" "github.com/GrayCodeAI/hawk/internal/netutil" "github.com/GrayCodeAI/hawk/internal/observability/logger" + "github.com/GrayCodeAI/hawk/internal/observability/oteltrace" + "github.com/GrayCodeAI/hawk/internal/securitylog" "github.com/GrayCodeAI/hawk/internal/storage" "github.com/spf13/cobra" ) var ( - daemonPort int - daemonHost string - daemonAPIKey string - daemonJSON bool + daemonPort int + daemonHost string + daemonAPIKey string + daemonJSON bool + daemonLogLevel string + daemonCORSOrigins []string + daemonTLSCertFile string + daemonTLSKeyFile string ) var daemonCmd = &cobra.Command{ @@ -61,6 +67,10 @@ func init() { daemonStartCmd.Flags().IntVarP(&daemonPort, "port", "p", 4590, "Port to listen on") daemonStartCmd.Flags().StringVar(&daemonHost, "host", netutil.LoopbackHost, "Host to bind to (default: 127.0.0.1, use 0.0.0.0 for remote access)") daemonStartCmd.Flags().StringVar(&daemonAPIKey, "api-key", "", "API key for protected daemon endpoints (defaults to HAWK_DAEMON_API_KEY or a generated key)") + daemonStartCmd.Flags().StringVar(&daemonLogLevel, "log-level", "INFO", "Log level for daemon output (DEBUG, INFO, WARN, ERROR)") + daemonStartCmd.Flags().StringSliceVar(&daemonCORSOrigins, "cors", []string{}, "Comma-separated list of allowed CORS origins (empty disables CORS, '*' allows all)") + daemonStartCmd.Flags().StringVar(&daemonTLSCertFile, "tls-cert", "", "Path to TLS certificate file (enables HTTPS when paired with --tls-key)") + daemonStartCmd.Flags().StringVar(&daemonTLSKeyFile, "tls-key", "", "Path to TLS private key file (enables HTTPS when paired with --tls-cert)") daemonCmd.AddCommand(daemonStartCmd) daemonCmd.AddCommand(daemonStopCmd) daemonCmd.AddCommand(daemonStatusCmd) @@ -69,6 +79,40 @@ func init() { func runDaemonStart(_ *cobra.Command, _ []string) error { settings := hawkconfig.LoadSettings() + + // Initialize OpenTelemetry telemetry (opt-in via HAWK_CODE_ENABLE_TELEMETRY=1). + telemetryProviders, telemetryErr := oteltrace.InitTelemetry(oteltrace.DefaultTelemetryConfig()) + if telemetryErr != nil { + fmt.Fprintln(os.Stderr, "warning: telemetry initialization failed:", telemetryErr) + } + + // Set up file-backed logging for the daemon. Logs go to + // ~/.hawk/state/daemon.log with slog structured output. + logFile, logErr := openDaemonLogFile() + var daemonLogger *logger.Logger + if logErr != nil { + // Fall back to stderr if file logging fails. + daemonLogger = logger.New(os.Stderr, logLevelFromString(daemonLogLevel)) + fmt.Fprintln(os.Stderr, "warning: daemon file logging failed, falling back to stderr:", logErr) + } else { + daemonLogger = logger.New(logFile, logLevelFromString(daemonLogLevel)) + } + if logFile != nil { + slog.SetDefault(slog.New(slog.NewTextHandler(logFile, &slog.HandlerOptions{ + Level: slogLevelFromString(daemonLogLevel), + }))) + } + + // Replace the discarded logger with the real file-backed logger. + newSession := newConfiguredHawkSessionFactory(settings, daemonLogger) + + // Log startup banner. + daemonLogger.Info("hawk daemon starting", map[string]interface{}{ + "host": daemonHost, + "port": daemonPort, + "telemetry_enabled": telemetryProviders != nil && telemetryProviders.IsEnabled(), + }) + apiKey := daemonAPIKey if apiKey == "" { apiKey = os.Getenv("HAWK_DAEMON_API_KEY") @@ -81,7 +125,15 @@ func runDaemonStart(_ *cobra.Command, _ []string) error { } } - newSession := newConfiguredHawkSessionFactory(settings, logger.New(io.Discard, logger.Error)) + // Initialize the security audit log early so it can be shared between + // the daemon server and the session factory (for tool execution auditing). + var secLog *securitylog.Log + if l, err := securitylog.New(securitylog.DefaultDir()); err != nil { + daemonLogger.Warn("failed to initialize security audit log", map[string]interface{}{"error": err}) + } else { + secLog = l + } + factory := func(req daemon.ChatRequest) (*engine.Session, error) { systemPrompt, err := buildSystemPrompt() if err != nil { @@ -98,11 +150,28 @@ func runDaemonStart(_ *cobra.Command, _ []string) error { } else if agentModel != "" { modelOverride = agentModel } - return newSession(systemPrompt, modelOverride) + session, err := newSession(systemPrompt, modelOverride) + if err != nil { + return nil, err + } + // Wire the audit log into the session's tool service so tool + // executions are recorded in the tamper-evident log. + if secLog != nil { + session.Tools().WithAuditLog(secLog) + } + return session, nil } daemon.SetVersion(version) - srv := daemon.New(daemon.Config{Port: daemonPort, Host: daemonHost, APIKey: apiKey}, factory) + srv := daemon.New(daemon.Config{ + Port: daemonPort, + Host: daemonHost, + APIKey: apiKey, + CORSOrigins: daemonCORSOrigins, + TLSCertFile: daemonTLSCertFile, + TLSKeyFile: daemonTLSKeyFile, + SecurityLog: secLog, + }, factory) srv.SetGraphFactory(func(ctx context.Context, req daemon.GraphRequest) (executiongraph.Export, error) { if err := ctx.Err(); err != nil { return executiongraph.Export{}, err @@ -134,13 +203,19 @@ func runDaemonStart(_ *cobra.Command, _ []string) error { defer preheater.Stop() fmt.Printf("hawk daemon running on http://%s\n", addr) - fmt.Println("Endpoints: GET /v1/health, POST /v1/chat, GET /v1/sessions") + fmt.Println("Endpoints: GET /v1/health, GET /v1/ready, POST /v1/chat, GET /v1/sessions, GET /v1/metrics") fmt.Println("Protected endpoints require Authorization: Bearer or X-API-Key.") if len(apiKey) > 8 { fmt.Printf("API key: %s...%s\n", apiKey[:4], apiKey[len(apiKey)-4:]) } else { fmt.Println("API key: (set via --api-key or HAWK_DAEMON_API_KEY)") } + fmt.Printf("Logs: %s\n", filepath.Join(storage.DaemonRunDir(), "daemon.log")) + if telemetryProviders != nil && telemetryProviders.IsEnabled() { + fmt.Println("Telemetry: enabled (OTLP export configured)") + } else { + fmt.Println("Telemetry: disabled (set HAWK_CODE_ENABLE_TELEMETRY=1 to enable)") + } keyFile := filepath.Join(storage.DaemonRunDir(), "daemon.key") _ = os.MkdirAll(filepath.Dir(keyFile), 0o700) if err := fsutil.WritePinnedFile(keyFile, []byte(apiKey), 0o600); err == nil { @@ -168,9 +243,64 @@ func runDaemonStart(_ *cobra.Command, _ []string) error { _ = os.Remove(keyFile) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + + // Flush telemetry before shutdown. + if telemetryProviders != nil { + if err := telemetryProviders.Flush(ctx); err != nil { + slog.Warn("telemetry flush failed", "error", err) + } + if err := telemetryProviders.Shutdown(ctx); err != nil { + slog.Warn("telemetry shutdown failed", "error", err) + } + } + return srv.Stop(ctx) } +// openDaemonLogFile opens (or creates) the daemon log file at +// ~/.hawk/state/daemon.log and returns it. The directory is created if needed. +func openDaemonLogFile() (*os.File, error) { + dir := storage.DaemonRunDir() + if err := os.MkdirAll(dir, 0o750); err != nil { // #nosec G301 -- daemon run dir needs group traversal + return nil, err + } + return os.OpenFile(filepath.Join(dir, "daemon.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) +} + +// logLevelFromLevelString maps a string log level to the logger.Level type. +func logLevelFromString(s string) logger.Level { + switch strings.ToUpper(strings.TrimSpace(s)) { + case "DEBUG": + return logger.Debug + case "INFO": + return logger.Info + case "WARN": + return logger.Warn + case "ERROR": + return logger.Error + case "FATAL": + return logger.Fatal + default: + return logger.Info + } +} + +// slogLevelFromString maps a string log level to slog.Level. +func slogLevelFromString(s string) slog.Level { + switch strings.ToUpper(strings.TrimSpace(s)) { + case "DEBUG": + return slog.LevelDebug + case "INFO": + return slog.LevelInfo + case "WARN": + return slog.LevelWarn + case "ERROR": + return slog.LevelError + default: + return slog.LevelInfo + } +} + // daemonReadyProbe builds the readiness function installed via SetReadyFn. It // performs Eyrie's local preflight (provider state, catalog, credentials, and // model selection) under a short timeout: diff --git a/cmd/features.go b/cmd/features.go new file mode 100644 index 00000000..895f448c --- /dev/null +++ b/cmd/features.go @@ -0,0 +1,72 @@ +package cmd + +import ( + "fmt" + "sort" + "strings" + + "github.com/GrayCodeAI/hawk/internal/feature" + "github.com/spf13/cobra" +) + +var featuresCmd = &cobra.Command{ + Use: "features", + Short: "List and manage feature flags", + Long: `features lists all registered feature flags, their current values, +and how to override them via environment variables. + +Feature flags allow runtime configuration of experimental or gated +capabilities without code changes or restarts (some changes may require +a daemon restart). + +Override a flag via environment variable: + HAWK_FEATURE_=1 hawk daemon start + +Show a specific flag: + hawk features get `, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) > 0 && args[0] == "get" { + if len(args) < 2 { + return fmt.Errorf("usage: hawk features get ") + } + f, ok := feature.Info(args[1]) + if !ok { + return fmt.Errorf("unknown feature flag: %s", args[1]) + } + fmt.Printf("Name: %s\n", f.Name()) + fmt.Printf("Default: %v\n", f.DefaultValue()) + fmt.Printf("Current: %v\n", feature.EnabledByName(args[1])) + fmt.Printf("Description: %s\n", f.Description()) + envVar := "HAWK_FEATURE_" + strings.ReplaceAll(strings.ToUpper(args[1]), "-", "_") + fmt.Printf("Env var: %s\n", envVar) + return nil + } + + flags := feature.List() + names := make([]string, 0, len(flags)) + for name := range flags { + names = append(names, name) + } + sort.Strings(names) + + fmt.Println("Feature Flags:") + fmt.Println() + for _, name := range names { + f, _ := feature.Info(name) + val := flags[name] + status := "DISABLED" + if val { + status = "ENABLED" + } + fmt.Printf(" %s = %v [%s]\n", name, val, status) + if f != nil { + fmt.Printf(" default: %v\n", f.DefaultValue()) + fmt.Printf(" description: %s\n", f.Description()) + envVar := "HAWK_FEATURE_" + strings.ReplaceAll(strings.ToUpper(name), "-", "_") + fmt.Printf(" env: %s\n", envVar) + } + fmt.Println() + } + return nil + }, +} diff --git a/cmd/hawk/main.go b/cmd/hawk/main.go index b634bd49..075cb8b4 100644 --- a/cmd/hawk/main.go +++ b/cmd/hawk/main.go @@ -1,14 +1,17 @@ package main import ( + "context" "errors" "fmt" "os" + "time" "github.com/GrayCodeAI/hawk/cmd" "github.com/GrayCodeAI/hawk/internal/crash" "github.com/GrayCodeAI/hawk/internal/hawkerr" "github.com/GrayCodeAI/hawk/internal/mcp" + "github.com/GrayCodeAI/hawk/internal/observability/oteltrace" ) // Version, Commit, and BuildDate are set at build time via ldflags. @@ -41,6 +44,20 @@ func main() { return } + // Initialize OpenTelemetry telemetry (opt-in via HAWK_CODE_ENABLE_TELEMETRY=1). + // Telemetry failures are non-fatal: hawk continues with in-memory tracing only. + telemetryProviders, telemetryErr := oteltrace.InitTelemetry(oteltrace.DefaultTelemetryConfig()) + if telemetryErr != nil { + fmt.Fprintln(os.Stderr, "warning: telemetry initialization failed:", telemetryErr) + } + if telemetryProviders != nil && telemetryErr == nil && telemetryProviders.IsEnabled() { + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = telemetryProviders.Shutdown(shutdownCtx) + }() + } + // Propagate the canonical version to all sub-packages that surface it // (CLI version flag, HTTP API version field, and MCP clientInfo). // The sandbox image has an independent compatibility version. diff --git a/cmd/root.go b/cmd/root.go index 3cec8faa..0a35d483 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -272,6 +272,7 @@ func init() { rootCmd.AddCommand(rulesCmd) rootCmd.AddCommand(sandboxCmd) rootCmd.AddCommand(costCmd) + rootCmd.AddCommand(featuresCmd) rootCmd.AddCommand(execCmd) rootCmd.AddCommand(daemonCmd) rootCmd.AddCommand(agentCmd) diff --git a/docs/monitoring-guide.md b/docs/monitoring-guide.md new file mode 100644 index 00000000..3fa88de3 --- /dev/null +++ b/docs/monitoring-guide.md @@ -0,0 +1,255 @@ +# Monitoring Guide + +This guide covers how to monitor hawk's daemon and CLI for production health, +performance, and security. + +## 1. Daemon Health & Readiness + +### `GET /v1/health` — Liveness probe + +Returns 200 with `{"status":"ok","version":"...","uptime":"...","active_sessions":N}`. + +Use this for liveness probes in Kubernetes or systemd: + +```bash +curl -sf http://localhost:4590/v1/health +``` + +### `GET /v1/ready` — Readiness probe + +Returns 200 when the daemon is fully ready to serve traffic (session factory +configured, Eyrie preflight checks pass). Returns 503 with the failed +dependency during startup or when dependencies are unavailable. + +```bash +curl -sf http://localhost:4590/v1/ready +``` + +### `GET /v1/stats` — Usage statistics + +Returns aggregated usage statistics (sessions, messages, tool calls, cost) +for the last N days (default 30, `?days=30`). + +```bash +curl -H "X-API-Key: $HAWK_DAEMON_API_KEY" \ + http://localhost:4590/v1/stats +``` + +## 2. Metrics Endpoint + +### `GET /v1/metrics` — Prometheus format + +The daemon exposes metrics in Prometheus text exposition format at +`GET /v1/metrics`. This endpoint requires authentication. + +```bash +curl -H "X-API-Key: $HAWK_DAEMON_API_KEY" \ + http://localhost:4590/v1/metrics +``` + +Available metrics: + +| Metric | Type | Description | +|--------|------|-------------| +| `hawk_daemon_active_sessions` | gauge | Number of active daemon sessions | +| `hawk_daemon_chat_concurrency_used` | gauge | Number of in-use chat concurrency slots | +| `hawk_daemon_uptime_seconds` | gauge | Daemon uptime in seconds | +| `http_requests_total` | counter | Total HTTP requests received | +| `http_request_duration_ms` | histogram | HTTP request duration in milliseconds | +| `http_rate_limited_total` | counter | Number of requests rejected by rate limiter | +| `auth_denied_total` | counter | Number of denied authentication attempts | +| `tool_exec_total` | counter | Number of tool executions | + +### JSON format + +Pass `?format=json` to get metrics as a JSON object instead of Prometheus +text format. + +## 3. OpenTelemetry Tracing + +Telemetry is **opt-in**. Enable it by setting: + +```bash +export HAWK_CODE_ENABLE_TELEMETRY=1 +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +hawk daemon start +``` + +The daemon will automatically: + +- Initialize the OTel SDK with a batch span processor (5s batch interval). +- Export traces to the OTLP endpoint (`OTEL_EXPORTER_OTLP_ENDPOINT`). +- Send trace headers from `OTEL_EXPORTER_OTLP_HEADERS`. +- Set the service name (default: `hawk-code`) and version. + +### Configuration + +| Environment Variable | Default | Description | +|---------------------|---------|-------------| +| `HAWK_CODE_ENABLE_TELEMETRY` | `0` | Set to `1` to enable OTP telemetry | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | _(empty)_ | OTLP collector endpoint | +| `OTEL_EXPORTER_OTLP_PROTOCOL` | `http/protobuf` | OTLP transport protocol | +| `OTEL_EXPORTER_OTLP_HEADERS` | _(empty)_ | Comma-separated `key=value` headers | +| `HAWK_CODE_OTEL_SHUTDOWN_TIMEOUT_MS` | `2000` | Shutdown timeout in milliseconds | + +### OTel Conventions + +hawk follows the OpenTelemetry semantic conventions for traces. See +[docs/OTEL-CONVENTIONS.md](OTEL-CONVENTIONS.md) for span naming and attribute +details. + +## 4. Structured Logging + +The daemon writes structured SLOG logs to `~/.hawk/state/daemon.log` by +default. Control the log level via: + +```bash +hawk daemon start --log-level DEBUG +``` + +Or via environment variable: + +```bash +export OTEL_LOG_LEVEL=DEBUG +``` + +Log levels: `DEBUG`, `INFO` (default), `WARN`, `ERROR`. + +### Log fields + +All daemon log entries include: + +- `time` — RFC3339 timestamp +- `level` — log level +- `msg` — log message +- Contextual fields (e.g., `method`, `path`, `status`, `request_id`, `remote`) + +### Audit log + +Security-relevant events (auth failures, tool executions) are written to a +tamper-evident log at `~/.hawk/state/securitylog/security_events.jsonl`. + +Verify the audit log integrity: + +```bash +# The securitylog package provides a verify command +go run ./cmd/hawk securitylog verify +``` + +## 5. Prometheus Scraping + +### Docker + +```yaml +services: + hawk: + image: ghcr.io/graycodeai/hawk-daemon:latest + ports: + - "4590:4590" + environment: + - HAWK_DAEMON_API_KEY=secret + labels: + - "prometheus.io/scrape=true" + - "prometheus.io/port=4590" + - "prometheus.io/path=/v1/metrics" +``` + +### Kubernetes + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: hawk-daemon + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "4590" + prometheus.io/path: "/v1/metrics" +spec: + selector: + app: hawk-daemon + ports: + - port: 4590 + targetPort: 4590 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hawk-daemon +spec: + template: + spec: + containers: + - name: hawk + image: ghcr.io/graycodeai/hawk-daemon:latest + env: + - name: HAWK_DAEMON_API_KEY + valueFrom: + secretKeyRef: + name: hawk-secret + key: api-key + ports: + - containerPort: 4590 + livenessProbe: + httpGet: + path: /v1/health + port: 4590 + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /v1/ready + port: 4590 + initialDelaySeconds: 5 + periodSeconds: 10 +``` + +## 6. Alerting Recommendations + +| Alert | Condition | Severity | +|-------|-----------|----------| +| Daemon down | `hawk_daemon_uptime_seconds` does not increase for 2+ minutes | critical | +| High request latency | `histogram_quantile(0.95, http_request_duration_ms)` > 10000ms for 5 minutes | warning | +| Rate limit saturation | `rate(http_rate_limited_total[5m])` > 10/s | warning | +| Auth failures | `rate(auth_denied_total[5m])` > 5/s | critical (possible brute force) | +| High concurrency | `hawk_daemon_chat_concurrency_used` sustained at max for 5+ minutes | warning | +| No active sessions | `hawk_daemon_active_sessions` = 0 during business hours | warning | + +## 7. Systemd Logging + +When running under systemd, logs from stderr/stdout are captured by journald: + +```bash +journalctl -u hawk-daemon -f +``` + +The daemon also writes its own structured log to +`~/.hawk/state/daemon.log`: + +```bash +tail -f ~/.hawk/state/daemon.log +``` + +## 8. Feature Flags + +Feature flags allow runtime configuration without restarts. They are +controlled via environment variables: + +```bash +export HAWK_FEATURE_=1 +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `sandbox-v2` | `0` | Enable Landlock v2 sandboxing profile | +| `telemetry-otel` | `1` | Enable OpenTelemetry SDK | +| `metrics-endpoint` | `1` | Expose GET /v1/metrics | +| `security-headers` | `1` | Apply security headers middleware | +| `cors` | `0` | Enable CORS support | +| `audit-log` | `1` | Enable tamper-evident audit logging | + +List all registered flags: + +```bash +hawk features +``` diff --git a/docs/operations-checklist.md b/docs/operations-checklist.md new file mode 100644 index 00000000..8eee170a --- /dev/null +++ b/docs/operations-checklist.md @@ -0,0 +1,118 @@ +# Production Operations Checklist + +Use this checklist when deploying or upgrading hawk in a production +environment. Each item links to the relevant configuration option or +documentation section. + +## Pre-Deployment + +- [ ] **API key is set** — `HAWK_DAEMON_API_KEY` environment variable is + configured to a cryptographically random value (≥ 32 bytes). Do **not** + rely on the auto-generated key for production. + ```bash + export HAWK_DAEMON_API_KEY=$(openssl rand -base64 32) + ``` +- [ ] **Bind address** — Daemon binds to `0.0.0.0` (not just loopback) if + remote access is needed. If bound to non-loopback without TLS, the daemon + prints a warning and refuses to start without an API key. +- [ ] **TLS configured** — For production, either: + - Terminate TLS at a reverse proxy (nginx, Caddy, ALB) and set + `X-Forwarded-Proto: https`, **or** + - Enable native TLS with `--tls-cert` / `--tls-key` flags. +- [ ] **CORS configured** — If serving browser-based clients, set + `--cors https://app.example.com` to allow cross-origin requests from + trusted origins only. Use `--cors '*'` only for development. +- [ ] **Rate limits reviewed** — Default: 10 req/min for general API, 30 req/min + for chat, 4 concurrent chat sessions. Tune via + `HAWK_DAEMON_MAX_CONCURRENT`. +- [ ] **Resource limits set** — Configure CPU/memory limits in systemd + (`MemoryMax`, `CPUQuota`) or Kubernetes. Defaults in the systemd unit file: + `MemoryMax=4G`, `CPUQuota=200%`. +- [ ] **Log retention** — Daemon logs at `~/.hawk/state/daemon.log`. + Configure log rotation (logrotate, journald retention) to prevent disk + exhaustion. +- [ ] **State directory backed up** — The `~/.hawk/state/` directory contains + the PID file, API key pin file, audit log, and session state. Back up the + audit log key (`securitylog/sel.key`) — **losing it makes all historical + audit entries unverifiable**. +- [ ] **Firewall rules** — Only expose port 4590 to trusted networks or + behind a reverse proxy. Do not expose the daemon directly to the internet. + +## Observability + +- [ ] **Telemetry enabled (optional)** — Set `HAWK_CODE_ENABLE_TELEMETRY=1` + and configure `OTEL_EXPORTER_OTLP_ENDPOINT` to send traces to your OTLP + collector. Telemetry is opt-in by default. +- [ ] **Prometheus scraping** — If using Prometheus, configure a scrape + target for `http://:4590/v1/metrics` with authentication: + ```yaml + scrape_configs: + - job_name: 'hawk-daemon' + bearer_token: '' + static_configs: + - targets: ['hawk-daemon:4590'] + metrics_path: '/v1/metrics' + ``` +- [ ] **Health/readiness probes** — Configure in your orchestrator: + - Liveness: `GET /v1/health` (no auth required) + - Readiness: `GET /v1/ready` (no auth required) +- [ ] **Alerting rules** — Import the alerting recommendations from + [docs/monitoring-guide.md](monitoring-guide.md) §6. + +## Security Hardening + +- [ ] **API key rotated** — The API key is written to a pinned file at + `~/.hawk/state/daemon.key` for convenience. **Remove this file in + production** or ensure it has `0600` permissions and is not world-readable. +- [ ] **Audit log verification** — Periodically verify the audit log + integrity: + ```bash + # The Verify function checks the HMAC chain + go run ./cmd/hawk securitylog verify + ``` +- [ ] **Security headers** — Verify `X-Content-Type-Options`, + `X-Frame-Options`, and `Content-Security-Policy` headers are present + (enabled by default via the `security-headers` feature flag). +- [ ] **Sandbox mode** — Tool execution uses OS-level sandboxing by default. + For additional isolation, configure Docker/Podman container sandboxing. + Enable `sandbox-v2` for the experimental Landlock v2 profile. +- [ ] **No CGO** — The binary is built with `CGO_ENABLED=0` for a static binary. + Verify the deployed binary has no dynamic library dependencies: + ```bash + ldd /usr/local/bin/hawk # should say "not a dynamic executable" + ``` + +## Post-Deployment + +- [ ] **Smoke test** — Verify the daemon responds: + ```bash + curl http://localhost:4590/v1/health + curl http://localhost:4590/v1/ready + ``` +- [ ] **Auth test** — Verify protected endpoints reject unauthenticated + requests: + ```bash + curl -o /dev/null -w "%{http_code}" http://localhost:4590/v1/stats + # Should be 401 + ``` +- [ ] **Metrics test** — Verify the metrics endpoint: + ```bash + curl -H "X-API-Key: $HAWK_DAEMON_API_KEY" http://localhost:4590/v1/metrics + ``` +- [ ] **Version check** — Verify the deployed version matches expectations: + ```bash + curl http://localhost:4590/v1/health | jq .version + ``` + +## Upgrade Procedure + +1. **Back up state** — Copy `~/.hawk/state/` to a safe location. +2. **Drain traffic** — Remove the daemon from load balancer rotation or + stop sending new requests. +3. **Install new binary** — Replace the binary and run `hawk daemon start` + with the same configuration. +4. **Verify health** — Check `GET /v1/health` and `GET /v1/ready`. +5. **Verify metrics** — Check `GET /v1/metrics` for expected counters. +6. **Audit log** — Verify the audit log continues from the previous + sequence (no gaps in the HMAC chain). +7. **Resume traffic** — Re-enable the daemon in your load balancer. diff --git a/docs/troubleshooting-guide.md b/docs/troubleshooting-guide.md new file mode 100644 index 00000000..0b7b559e --- /dev/null +++ b/docs/troubleshooting-guide.md @@ -0,0 +1,406 @@ +# Troubleshooting Guide + +A practical guide for diagnosing common hawk daemon and CLI issues. + +## Table of Contents + +- [Daemon Won't Start](#daemon-wont-start) +- [Health/Readiness Failures](#healthreadiness-failures) +- [API Key / Authentication Issues](#api-key--authentication-issues) +- [Rate Limiting (429)](#rate-limiting-429) +- [CORS Errors](#cors-errors) +- [Chat Returns 503](#chat-returns-503) +- [Metrics Endpoint](#metrics-endpoint) +- [Telemetry / Tracing Not Working](#telemetry--tracing-not-working) +- [Audit Log Issues](#audit-log-issues) +- [Tool Execution Fails](#tool-execution-fails) +- [Performance Issues](#performance-issues) +- [Docker Issues](#docker-issues) +- [Systemd Issues](#systemd-issues) + +--- + +## Daemon Won't Start + +### "apiKey is empty and bind address is not loopback" + +The daemon refuses to start when bound to `0.0.0.0` without an API key, +because the auth middleware would be open to the network. + +**Fix:** Set an API key: + +```bash +export HAWK_DAEMON_API_KEY=$(openssl rand -base64 32) +hawk daemon start --host 0.0.0.0 --port 4590 +``` + +Or bind to loopback only (no API key required, but not remotely accessible): + +```bash +hawk daemon start --host 127.0.0.1 --port 4590 +``` + +### "permission denied" on state directory + +The daemon writes logs, PID files, and the audit log to `~/.hawk/state/`. + +**Fix:** + +```bash +mkdir -p ~/.hawk/state +chmod 750 ~/.hawk/state +# If running under systemd as user 'hawk', ensure ownership: +chown -R hawk:hawk ~/.hawk +``` + +### "port already in use" + +Another process is using port 4590. + +**Fix:** + +```bash +# Find the process +lsof -i :4590 + +# Or use a different port +hawk daemon start --port 4591 +``` + +--- + +## Health/Readiness Failures + +### `GET /v1/ready` returns 503 + +The readiness probe fails when Eyrie's local preflight doesn't pass. This +checks provider state, catalog, credentials, and model selection. + +**Diagnosis:** + +```bash +curl -v http://localhost:4590/v1/ready +``` + +Check the response body for the specific failed check. Common causes: + +- No model configured — set `HAWK_MODEL` or provider credentials. +- Eyrie catalog not initialized — ensure submodules are checked out: + ```bash + git submodule update --init --recursive + ``` + +### `GET /v1/health` returns 503 + +The health endpoint should always return 200 when the daemon is running. +If it returns 503, the daemon process may have crashed or the server +failed to start. + +**Diagnosis:** + +```bash +# Check if the process is running +ps aux | grep hawk + +# Check logs +journalctl -u hawk-daemon -n 50 +tail -50 ~/.hawk/state/daemon.log +``` + +--- + +## API Key / Authentication Issues + +### "401 Unauthorized" on all requests + +The daemon requires `Authorization: Bearer ` or `X-API-Key: `. + +**Fix:** + +```bash +curl -H "X-API-Key: $HAWK_DAEMON_API_KEY" http://localhost:4590/v1/stats +``` + +### "constant time comparison" errors in logs + +These are informational — the daemon performs a constant-time comparison +even when no API key is set (to avoid timing-based information leakage). +No action needed. + +### Forgotten API key + +The API key is written to `~/.hawk/state/daemon.key` (permissions 0600). + +```bash +cat ~/.hawk/state/daemon.key +``` + +**Security note:** Remove this file in production after initial testing: + +```bash +rm ~/.hawk/state/daemon.key +``` + +--- + +## Rate Limiting (429) + +If you receive `429 Too Many Requests`, the per-IP rate limiter has rejected +your request. The default limits are: + +- **General API**: 10 req/min, burst 4 +- **Chat**: 30 req/min, burst 6 +- **Concurrent chat sessions**: 4 (configurable via `HAWK_DAEMON_MAX_CONCURRENT`) + +**Fix:** + +- Reduce request frequency +- Increase rate limits in the daemon config (requires code change — see + `defaultAPIRatePerMin` and `defaultChatRatePerMin` in `internal/daemon/daemon.go`) +- Add `Retry-After` header handling on the client + +--- + +## CORS Errors + +### "No 'Access-Control-Allow-Origin' header" in browser console + +CORS is **disabled by default**. Enable it when serving browser-based clients: + +```bash +hawk daemon start --cors https://app.example.com --cors https://admin.example.com +``` + +Use `--cors '*'` only for development — it allows any origin. + +### Pre-flight (OPTIONS) returns 405 Method Not Allowed + +The CORS middleware handles OPTIONS preflight automatically when the `cors` +feature flag is enabled. If you're getting 405, ensure CORS is enabled: + +```bash +export HAWK_FEATURE_CORS=1 +``` + +--- + +## Chat Returns 503 + +The chat endpoint returns 503 when no session factory is configured (CLI +mode) or when the engine is not ready (daemon mode). + +**Diagnosis:** + +```bash +curl http://localhost:4590/v1/ready +``` + +If readiness fails, the session factory exists but Eyrie preflight is not +satisfied (missing model, credentials, etc.). + +--- + +## Metrics Endpoint + +### `GET /v1/metrics` returns 401 + +The metrics endpoint is protected by the same API key authentication as other +endpoints. + +```bash +curl -H "X-API-Key: $HAWK_DAEMON_API_KEY" http://localhost:4590/v1/metrics +``` + +### Metrics output is empty + +The daemon hasn't received any requests yet. Make a request first, then +check metrics again. + +### Prometheus can't parse the output + +The daemon uses Prometheus text exposition format 0.0.4. Ensure your +Prometheus version supports this format (Prometheus 2.20+). + +For troubleshooting, try the JSON format: + +```bash +curl -H "X-API-Key: $HAWK_DAEMON_API_KEY" \ + "http://localhost:4590/v1/metrics?format=json" +``` + +--- + +## Telemetry / Tracing Not Working + +### Traces are not being sent to the collector + +Telemetry is **opt-in**. You must explicitly enable it: + +```bash +export HAWK_CODE_ENABLE_TELEMETRY=1 +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +hawk daemon start +``` + +**Important**: Setting `OTEL_EXPORTER_OTLP_ENDPOINT` alone does **not** +enable telemetry. The `HAWK_CODE_ENABLE_TELEMETRY=1` flag is required. + +### "telemetry initialization failed" warning in logs + +The OTel SDK couldn't connect to the configured endpoint. Check: + +1. The collector is running and reachable +2. The endpoint URL is correct (include port and path if needed) +3. The protocol matches (`http/protobuf` is the default) + +```bash +# Verify the collector is reachable +curl -v http://localhost:4318/v1/traces +``` + +### Spans are created but not exported + +The OTel SDK uses a batch span processor with a 5-second flush interval. +On shutdown, the daemon waits up to 2 seconds (configurable via +`HAWK_CODE_OTEL_SHUTDOWN_TIMEOUT_MS`) to flush pending spans. + +To force a flush, send SIGTERM to the daemon — it will flush telemetry +before shutting down. + +--- + +## Audit Log Issues + +### Security log won't open + +The audit log is stored in `~/.hawk/state/securitylog/`. If the directory +doesn't exist or isn't writable: + +```bash +mkdir -p ~/.hawk/state/securitylog +chmod 700 ~/.hawk/state/securitylog +``` + +### "log tail does not match head pointer (truncated or tampered)" + +This error means the security log has been modified or truncated. The +tamper-evident design detected an inconsistency. Restore from a backup +of the `~/.hawk/state/securitylog/` directory. + +### Lost the HMAC key + +The HMAC key (`sel.key`) is required to verify the audit log. If it's +lost, all entries become unverifiable. **Always back up the entire +`~/.hawk/state/securitylog/` directory.** + +--- + +## Tool Execution Fails + +### "Container not ready — tools are disabled" + +Tools that require sandboxing are disabled until the sandbox container +is running. Check the sandbox status: + +```bash +hawk sandbox status +``` + +Start the sandbox: + +```bash +hawk sandbox start +``` + +### Permission denied for a tool + +The permission service may have denied the tool execution. Check the +audit log for the `denied` event type. + +--- + +## Performance Issues + +### High latency on /v1/chat + +1. Check the metrics endpoint for request duration: + ```bash + curl -H "X-API-Key: $HAWK_DAEMON_API_KEY" \ + "http://localhost:4590/v1/metrics?format=json" + ``` +2. Check concurrent sessions: `hawk_daemon_chat_concurrency_used` +3. If at capacity, increase the concurrency limit: + ```bash + export HAWK_DAEMON_MAX_CONCURRENT=8 + ``` + +### High memory usage + +The daemon retains session state in memory. Long-running sessions with +large contexts can consume significant memory. Consider: + +- Periodic session cleanup +- Limiting `max_turns` per session +- Using the `/v1/sessions` endpoint to monitor active sessions + +--- + +## Docker Issues + +### Daemon exits immediately + +The default entrypoint runs `hawk daemon start --host 0.0.0.0 --port 4590`. +If no API key is set, the daemon will refuse to start on a non-loopback bind. + +**Fix:** + +```bash +docker run -p 4590:4590 \ + -e HAWK_DAEMON_API_KEY=$(openssl rand -base64 32) \ + ghcr.io/graycodeai/hawk-daemon:latest +``` + +### Health check fails in container + +The container's `HEALTHCHECK` probes `http://127.0.0.1:4590/v1/health`. +If the daemon is still starting up, the health check may fail before the +`start-period` expires (10 seconds). Increase the health check interval +or start-period if needed. + +--- + +## Systemd Issues + +### "Failed to start hawk-daemon.service: Unit not found" + +Install the unit file: + +```bash +sudo cp packaging/systemd/hawk-daemon.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now hawk-daemon +``` + +### "Permission denied" accessing state directory + +The systemd unit runs as user `hawk` with `ProtectSystem=strict` and +`ReadWritePaths=%h/.hawk/state`. Ensure the user's home directory has +the correct state: + +```bash +sudo -u hawk mkdir -p /home/hawk/.hawk/state +sudo -u hawk chmod 750 /home/hawk/.hawk/state +``` + +### Daemon not logging to journald + +If using the systemd unit, logs go to both journald (stdout/stderr) and +`~/.hawk/state/daemon.log`. Check: + +```bash +journalctl -u hawk-daemon -f +tail -f ~/.hawk/state/daemon.log +``` + +If journald logs are missing, verify that stdout/stderr are not being +redirected in the unit file's `ExecStart`. diff --git a/external/hawk-mcpkit b/external/hawk-mcpkit index b0648b5e..85ac53f3 160000 --- a/external/hawk-mcpkit +++ b/external/hawk-mcpkit @@ -1 +1 @@ -Subproject commit b0648b5e6f599de9187c9c88047865dd15b74f25 +Subproject commit 85ac53f3ec847c607bc419ff7e72b08a8c73b6df diff --git a/external/yaad b/external/yaad index f0aa1699..52c8c805 160000 --- a/external/yaad +++ b/external/yaad @@ -1 +1 @@ -Subproject commit f0aa1699d6b0888baf48ba7ac85f7a66e2f29364 +Subproject commit 52c8c805791f06de6e18855c1807aa332407501e diff --git a/go.mod b/go.mod index 0f4c42d0..5281a835 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/GrayCodeAI/inspect v0.0.0-20260726091806-08f3151d5738 github.com/GrayCodeAI/sight v0.0.0-20260726091804-84c96edfc589 github.com/GrayCodeAI/tok v0.1.5-0.20260731011234-7a7c3cbae89b - github.com/GrayCodeAI/yaad v0.2.1-0.20260727172552-f0aa1699d6b0 + github.com/GrayCodeAI/yaad v0.2.1-0.20260729231812-52c8c805791f github.com/alecthomas/chroma/v2 v2.26.1 github.com/bwmarrin/discordgo v0.28.1 github.com/charmbracelet/x/ansi v0.11.7 @@ -42,7 +42,7 @@ require ( ) require ( - github.com/GrayCodeAI/hawk-mcpkit v0.1.5 // indirect + github.com/GrayCodeAI/hawk-mcpkit v0.1.6-0.20260729083555-85ac53f3ec84 // indirect github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f // indirect github.com/chromedp/sysutil v1.1.0 // indirect github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect @@ -171,7 +171,7 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect @@ -179,7 +179,7 @@ require ( golang.org/x/tools v0.47.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.82.0 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect modernc.org/libc v1.72.5 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index e6aa1b3b..58baece8 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ github.com/GrayCodeAI/eyrie v0.2.2 h1:iMURJ9lJ2MqVi1uvXEY4hOAD5oL8bS32X4nVh7NoAC github.com/GrayCodeAI/eyrie v0.2.2/go.mod h1:AW/UPuj+EWxMibiD+/Cy0TWd6RmTvth+KeGXSxU3t6I= github.com/GrayCodeAI/hawk-core-contracts v0.1.12 h1:percfsd771JLmO9gMkrQtENEPBA9ZN3dG1Nc1moN3ZQ= github.com/GrayCodeAI/hawk-core-contracts v0.1.12/go.mod h1:BXbh68YrCf+s9HVqND5F8DAvl2MnE5NcOwZZZB56HGA= -github.com/GrayCodeAI/hawk-mcpkit v0.1.5 h1:gskBd3miHN063aXXP4dEhzn5x0HM9mzYAnTmM+ug/nE= -github.com/GrayCodeAI/hawk-mcpkit v0.1.5/go.mod h1:C32HPDRqiDETbVbMIbOTvguek6KImpLCffJjet7sqck= +github.com/GrayCodeAI/hawk-mcpkit v0.1.6-0.20260729083555-85ac53f3ec84 h1:HzoXUYNNyt88IccaPBxSvOQ/5PZzJOcSHbvBjX3l2mQ= +github.com/GrayCodeAI/hawk-mcpkit v0.1.6-0.20260729083555-85ac53f3ec84/go.mod h1:C32HPDRqiDETbVbMIbOTvguek6KImpLCffJjet7sqck= github.com/GrayCodeAI/inspect v0.0.0-20260726091806-08f3151d5738 h1:T9mQS75wTtTq6xqX1KGsQJyhnRFnd5YmoaGnHe9jppI= github.com/GrayCodeAI/inspect v0.0.0-20260726091806-08f3151d5738/go.mod h1:kSyO5gWDrBYKcYXHG4JNkJ/2yWAayAVPee8PIC24bX0= github.com/GrayCodeAI/sight v0.0.0-20260726091804-84c96edfc589 h1:gMXVRCqqdth6ziqFYnw8nXXbrD7LDtlgbGpryJZ0r1Q= @@ -30,8 +30,8 @@ github.com/GrayCodeAI/tok v0.1.5-0.20260731011234-7a7c3cbae89b h1:HjAYJHkFSn3Fxi github.com/GrayCodeAI/tok v0.1.5-0.20260731011234-7a7c3cbae89b/go.mod h1:/KTHlWg+qg8fDV8qRsLUfp4VKtt5seTyED1byxldBtE= github.com/GrayCodeAI/trace v0.1.4-0.20260803003541-1cd0fc51b106 h1:hzR6j0JaKOKCMrck60Pmkqdqip8Zu2vuimUXvZG8+8U= github.com/GrayCodeAI/trace v0.1.4-0.20260803003541-1cd0fc51b106/go.mod h1:xPV6sC2cUG0i7QD7aX3KtjiW1rlEhN9c5w9jBSc8LBc= -github.com/GrayCodeAI/yaad v0.2.1-0.20260727172552-f0aa1699d6b0 h1:jqNnsq2lq5CdZkyIL6jniiFuTRHiWPOPhlQZr37Zw2g= -github.com/GrayCodeAI/yaad v0.2.1-0.20260727172552-f0aa1699d6b0/go.mod h1:XyHXWpBmjHlxMJiETb/biOkjBlJHuDJ/03TGYhWDLis= +github.com/GrayCodeAI/yaad v0.2.1-0.20260729231812-52c8c805791f h1:4ridJ6o/eM2qLyIEwGwqTdZtYc7hwiS+cqC3Mtk6mJM= +github.com/GrayCodeAI/yaad v0.2.1-0.20260729231812-52c8c805791f/go.mod h1:lN77OfTzQLNIWJhsQ+KUpnFwqTfaIb/tG0bhXSjCuZs= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= @@ -242,6 +242,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mark3labs/mcp-go v0.49.0 h1:7Ssx4d7/T86qnWoJIdye7wEEvUzv39UIbnZb/FqUZMY= @@ -275,6 +276,7 @@ github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJm github.com/nwaples/rardecode/v2 v2.2.3 h1:qaVuy3ChZDbAQZshPLjHeNJKF3Cru8uo9jmgveKIy2A= github.com/nwaples/rardecode/v2 v2.2.3/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc= github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/petermattis/goid v0.0.0-20260330135022-df67b199bc81 h1:WDsQxOJDy0N1VRAjXLpi8sCEZRSGarLWQevDxpTBRrM= @@ -418,8 +420,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1: google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index ce679e7b..8a4bb463 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -23,6 +23,8 @@ import ( "github.com/GrayCodeAI/hawk/internal/engine" "github.com/GrayCodeAI/hawk/internal/netutil" + "github.com/GrayCodeAI/hawk/internal/observability/metrics" + "github.com/GrayCodeAI/hawk/internal/securitylog" hawksession "github.com/GrayCodeAI/hawk/internal/session" "github.com/GrayCodeAI/hawk/internal/storage" ) @@ -110,6 +112,8 @@ type Server struct { // server-wide. Per-session stripe locks already serialize the *same* // session; this caps total load across sessions (H9). concurrencySem chan struct{} + // metrics registry tracks daemon-level request and resource metrics. + metrics *metrics.Registry // cancelMu guards cancels, the sessionID -> cancel mapping used by // POST /v1/cancel to abort an in-flight generation (H10). cancelMu sync.Mutex @@ -118,6 +122,13 @@ type Server struct { apiLimiter *ipLimiter // Per-IP token bucket for /v1/chat generations (heavier, so lower rate). chatLimiter *ipLimiter + // corsOrigins is the list of allowed CORS origins. Empty disables CORS. + corsOrigins []string + // securityLog is the tamper-evident audit log for security events. + securityLog *securitylog.Log + // tlsCertFile and tlsKeyFile enable HTTPS when both are set. + tlsCertFile string + tlsKeyFile string } // ReadyResponse is the JSON response from GET /v1/ready. @@ -139,6 +150,16 @@ type Config struct { // Gateways configures optional messaging bridges (Telegram/Discord/Slack). // All are disabled by default; the daemon starts normally when none are set. Gateways GatewaysConfig `json:"gateways,omitempty"` + // CORSOrigins configures allowed CORS origins for the daemon API. + // Empty (default) disables CORS. Use ["*"] to allow all origins (dev only). + CORSOrigins []string `json:"cors_origins,omitempty"` + // TLSCertFile and TLSKeyFile enable HTTPS for the daemon. When both + // are set, the server listens with TLS. Empty (default) uses plain HTTP. + TLSCertFile string `json:"tls_cert_file,omitempty"` + TLSKeyFile string `json:"tls_key_file,omitempty"` + // SecurityLog provides the audit log instance for tool execution + // auditing. If nil, the server creates one from DefaultDir(). + SecurityLog *securitylog.Log `json:"-"` } // DefaultConfig returns reasonable defaults. @@ -208,6 +229,7 @@ func New(cfg Config, factory SessionFactory) *Server { cancels: make(map[string]*cancelEntry), apiLimiter: newIPLimiter(defaultAPIRatePerMin/60, defaultAPIBurst), chatLimiter: newIPLimiter(defaultChatRatePerMin/60, defaultChatBurst), + metrics: metrics.NewRegistry(), } s.routes() // Build the messaging-bridge manager. The daemon URL is finalised in Start @@ -215,6 +237,20 @@ func New(cfg Config, factory SessionFactory) *Server { // Slack can register its webhook route on the mux, and patch the forward URL // for poll-based gateways at Start time. s.gateways = newGatewayManager(cfg.Gateways, "http://"+s.addr, cfg.APIKey, s) + s.corsOrigins = cfg.CORSOrigins + s.tlsCertFile = cfg.TLSCertFile + s.tlsKeyFile = cfg.TLSKeyFile + + // Initialize the tamper-evident security event log. If a log is provided + // in the config, use it; otherwise create one from the default directory. + if cfg.SecurityLog != nil { + s.securityLog = cfg.SecurityLog + } else if secLog, err := securitylog.New(securitylog.DefaultDir()); err != nil { + slog.Warn("failed to initialize security audit log", "error", err) + } else { + s.securityLog = secLog + } + s.server = &http.Server{ Addr: s.addr, Handler: s.mux, @@ -223,6 +259,8 @@ func New(cfg Config, factory SessionFactory) *Server { WriteTimeout: 300 * time.Second, IdleTimeout: 60 * time.Second, } + // Install middleware stack: request IDs → security headers → CORS → logging. + s.installMiddleware() return s } @@ -246,8 +284,14 @@ func (s *Server) Start() (string, error) { slog.Error("daemon server goroutine panicked", "recover", r) } }() - if err := s.server.Serve(ln); err != nil && err != http.ErrServerClosed { - slog.Error("daemon server error", "error", err) + var serveErr error + if s.tlsCertFile != "" && s.tlsKeyFile != "" { + serveErr = s.server.ServeTLS(ln, s.tlsCertFile, s.tlsKeyFile) + } else { + serveErr = s.server.Serve(ln) + } + if serveErr != nil && serveErr != http.ErrServerClosed { + slog.Error("daemon server error", "error", serveErr) } }() @@ -311,11 +355,18 @@ func isLoopbackHost(host string) bool { return false } -// Stop gracefully shuts down the daemon. +// SecurityLog returns the daemon's security audit log, or nil if unset. +func (s *Server) SecurityLog() *securitylog.Log { + return s.securityLog +} + func (s *Server) Stop(ctx context.Context) error { if s.gateways != nil { s.gateways.Stop() } + if s.securityLog != nil { + _ = s.securityLog.Close() + } _ = s.removePIDFile() return s.server.Shutdown(ctx) } @@ -370,6 +421,7 @@ func (s *Server) routes() { s.handle("GET /v1/sessions/{id}/graph", s.auth(s.rate(s.handleGetSessionGraph, s.apiLimiter))) s.handle("DELETE /v1/sessions/{id}", s.auth(s.rate(s.handleDeleteSession, s.apiLimiter))) s.handle("GET /v1/stats", s.auth(s.rate(s.handleStats, s.apiLimiter))) + s.handle("GET /v1/metrics", s.auth(s.rate(s.handleMetrics, s.apiLimiter))) s.RegisterReviewRoutes() } @@ -385,6 +437,7 @@ func (s *Server) handle(pattern string, h http.HandlerFunc) { func (s *Server) rate(next http.HandlerFunc, lim *ipLimiter) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if lim != nil && !lim.Allow(clientIP(r)) { + s.metrics.Counter("http.rate_limited_total").Inc() w.Header().Set("Retry-After", "2") writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "rate limit exceeded"}) return @@ -468,6 +521,17 @@ func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc { return } if !constantTimeEqual(token, s.apiKey) { + // Audit: log failed authentication attempt. + if s.securityLog != nil { + reqID := RequestIDFromContext(r.Context()) + _, _ = s.securityLog.Append( + securitylog.SeverityWarning, + "auth_denied", + fmt.Sprintf("auth failed: wrong token (ip=%s, path=%s, request_id=%s)", clientIP(r), r.URL.Path, reqID), + "", reqID, + ) + } + s.metrics.Counter("auth_denied_total").Inc() writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) return } diff --git a/internal/daemon/e2e_test.go b/internal/daemon/e2e_test.go new file mode 100644 index 00000000..46b96657 --- /dev/null +++ b/internal/daemon/e2e_test.go @@ -0,0 +1,373 @@ +package daemon + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "testing" + + "github.com/GrayCodeAI/hawk/internal/feature" + "github.com/GrayCodeAI/hawk/internal/testutil" +) + +// TestE2E_HealthEndpoint verifies the health endpoint returns the expected +// fields and 200 status. +func TestE2E_HealthEndpoint(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + resp := httpGet(t, "http://"+addr+"/v1/health") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + var health HealthResponse + if err := json.NewDecoder(resp.Body).Decode(&health); err != nil { + t.Fatalf("decode health: %v", err) + } + if health.Status != "ok" { + t.Errorf("expected status 'ok', got %q", health.Status) + } + if health.Version == "" { + t.Error("expected non-empty version") + } +} + +// TestE2E_ReadyEndpoint verifies the readiness probe returns 503 when +// the engine is not configured (fail-closed). +func TestE2E_ReadyEndpoint(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + resp := httpGet(t, "http://"+addr+"/v1/ready") + defer resp.Body.Close() + + // No session factory → not ready. + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("expected 503, got %d", resp.StatusCode) + } + var ready ReadyResponse + if err := json.NewDecoder(resp.Body).Decode(&ready); err != nil { + t.Fatalf("decode ready: %v", err) + } + if ready.Ready { + t.Error("expected not ready when no factory configured") + } +} + +// TestE2E_MetricsEndpoint verifies the Prometheus metrics endpoint. +func TestE2E_MetricsEndpoint(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + // Make a request to generate some metrics. + httpGet(t, "http://"+addr+"/v1/health") + + resp := httpGet(t, "http://"+addr+"/v1/metrics") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + ct := resp.Header.Get("Content-Type") + if !strings.Contains(ct, "text/plain") { + t.Errorf("expected text/plain content type, got %q", ct) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), "hawk_daemon_active_sessions") { + t.Error("expected hawk_daemon_active_sessions metric in output") + } + if !strings.Contains(string(body), "hawk_daemon_uptime_seconds") { + t.Error("expected hawk_daemon_uptime_seconds metric in output") + } + + // Test JSON format too. + respJSON := httpGet(t, "http://"+addr+"/v1/metrics?format=json") + defer respJSON.Body.Close() + if respJSON.StatusCode != http.StatusOK { + t.Fatalf("expected 200 for JSON, got %d", respJSON.StatusCode) + } + if !strings.Contains(respJSON.Header.Get("Content-Type"), "application/json") { + t.Errorf("expected application/json content type, got %q", respJSON.Header.Get("Content-Type")) + } +} + +// TestE2E_MetricsRequiresAuth verifies that /v1/metrics is protected. +func TestE2E_MetricsRequiresAuth(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost, APIKey: "secret"}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + resp := httpGet(t, "http://"+addr+"/v1/metrics") + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401 without API key, got %d", resp.StatusCode) + } +} + +// TestE2E_SecurityHeaders verifies that security headers are present on +// all responses, including error responses. +func TestE2E_SecurityHeaders(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + resp := httpGet(t, "http://"+addr+"/v1/health") + defer resp.Body.Close() + + headers := []string{ + "X-Content-Type-Options", + "X-Frame-Options", + "Referrer-Policy", + "X-XSS-Protection", + "Content-Security-Policy", + } + for _, h := range headers { + if v := resp.Header.Get(h); v == "" { + t.Errorf("missing security header: %s", h) + } + } + if v := resp.Header.Get("X-Content-Type-Options"); v != "nosniff" { + t.Errorf("expected X-Content-Type-Options=nosniff, got %q", v) + } + if v := resp.Header.Get("X-Frame-Options"); v != "DENY" { + t.Errorf("expected X-Frame-Options=DENY, got %q", v) + } +} + +// TestE2E_RequestID verifies that X-Request-ID is generated and returned +// in responses, and is logged. +func TestE2E_RequestID(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + // Request without a pre-set ID — server should generate one. + resp := httpGet(t, "http://"+addr+"/v1/health") + defer resp.Body.Close() + reqID := resp.Header.Get("X-Request-ID") + if reqID == "" { + t.Fatal("expected X-Request-ID header in response") + } + + // Request with a pre-set ID — server should echo it. + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://"+addr+"/v1/health", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Request-ID", "custom-req-id-123") + resp2, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp2.Body.Close() + if got := resp2.Header.Get("X-Request-ID"); got != "custom-req-id-123" { + t.Errorf("expected echoed X-Request-ID, got %q", got) + } +} + +// TestE2E_CORS verifies CORS headers are set when configured. +func TestE2E_CORS(t *testing.T) { + feature.Set("cors", true) + defer feature.Set("cors", false) + srv := New(Config{ + Port: 0, + Host: testutil.LoopbackHost, + CORSOrigins: []string{"http://example.com"}, + }, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + // Request with an allowed origin. + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://"+addr+"/v1/health", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Origin", "http://example.com") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if got := resp.Header.Get("Access-Control-Allow-Origin"); got != "http://example.com" { + t.Errorf("expected Access-Control-Allow-Origin=http://example.com, got %q", got) + } + + // Request with a disallowed origin — no CORS headers. + req2, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://"+addr+"/v1/health", nil) + if err != nil { + t.Fatal(err) + } + req2.Header.Set("Origin", "http://evil.com") + resp2, err := http.DefaultClient.Do(req2) + if err != nil { + t.Fatal(err) + } + defer resp2.Body.Close() + + if got := resp2.Header.Get("Access-Control-Allow-Origin"); got != "" { + t.Errorf("expected no Access-Control-Allow-Origin for disallowed origin, got %q", got) + } +} + +// TestE2E_CORS_Preflight verifies OPTIONS preflight requests are handled. +func TestE2E_CORS_Preflight(t *testing.T) { + feature.Set("cors", true) + defer feature.Set("cors", false) + srv := New(Config{ + Port: 0, + Host: testutil.LoopbackHost, + CORSOrigins: []string{"http://example.com"}, + }, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + req, err := http.NewRequestWithContext(context.Background(), http.MethodOptions, "http://"+addr+"/v1/health", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Origin", "http://example.com") + req.Header.Set("Access-Control-Request-Method", "GET") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("expected 200 for preflight, got %d", resp.StatusCode) + } + if got := resp.Header.Get("Access-Control-Allow-Methods"); !strings.Contains(got, "GET") { + t.Errorf("expected Access-Control-Allow-Methods to contain GET, got %q", got) + } +} + +// TestE2E_AuditLog_AuthDenied verifies that failed auth attempts are +// recorded in the security audit log. +func TestE2E_AuditLog_AuthDenied(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost, APIKey: "secret"}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + // Make a request with a wrong API key. + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "http://"+addr+"/v1/chat", strings.NewReader(`{"prompt":"hi"}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-API-Key", "wrong-key") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401, got %d", resp.StatusCode) + } + + // Verify the auth_denied counter was incremented. + counter := srv.metrics.Counter("auth_denied_total") + if counter.Value() == 0 { + t.Error("expected auth_denied_total counter to be incremented") + } +} + +// TestE2E_RateLimit verifies the rate limiter returns 429 when exceeded. +func TestE2E_RateLimit(t *testing.T) { + // Use a very low rate limit via the default configuration. + // The default API rate is 10/min with burst 4, so after 4 rapid + // requests the next ones should be rejected. + srv := New(Config{Port: 0, Host: testutil.LoopbackHost, APIKey: "secret"}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + rejected := 0 + for i := 0; i < 20; i++ { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://"+addr+"/v1/stats", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-API-Key", "secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode == http.StatusTooManyRequests { + rejected++ + } + } + if rejected == 0 { + t.Error("expected at least one rate-limited response from burst exhaustion") + } +} + +// TestE2E_StatsRequiresAuth verifies /v1/stats is protected. +func TestE2E_StatsRequiresAuth(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost, APIKey: "secret"}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + resp := httpGet(t, "http://"+addr+"/v1/stats") + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401 without API key, got %d", resp.StatusCode) + } +} + +// TestE2E_ChatEmptyPrompt verifies /v1/chat rejects empty prompts. +func TestE2E_ChatEmptyPrompt(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + resp, err := http.Post("http://"+addr+"/v1/chat", "application/json", strings.NewReader(`{"prompt":""}`)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest && resp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("expected 400 or 503 for empty prompt, got %d", resp.StatusCode) + } +} + +// TestE2E_LivenessAndReadiness verifies both /v1/health and /v1/ready +// return appropriate status codes. +func TestE2E_LivenessAndReadiness(t *testing.T) { + srv := New(Config{Port: 0, Host: testutil.LoopbackHost}, nil) + addr := startTestDaemon(t, srv) + defer srv.Stop(context.Background()) + + healthResp := httpGet(t, "http://"+addr+"/v1/health") + defer healthResp.Body.Close() + if healthResp.StatusCode != http.StatusOK { + t.Errorf("health: expected 200, got %d", healthResp.StatusCode) + } + + readyResp := httpGet(t, "http://"+addr+"/v1/ready") + defer readyResp.Body.Close() + if readyResp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("ready: expected 503 without factory, got %d", readyResp.StatusCode) + } +} + +// httpGet is a test helper for simple GET requests. +func httpGet(t *testing.T, url string) *http.Response { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + return resp +} diff --git a/internal/daemon/middleware.go b/internal/daemon/middleware.go new file mode 100644 index 00000000..212074b2 --- /dev/null +++ b/internal/daemon/middleware.go @@ -0,0 +1,181 @@ +package daemon + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "log/slog" + "net/http" + "time" + + "github.com/GrayCodeAI/hawk/internal/feature" +) + +// requestIDKey is the context key for the request ID. +type requestIDKey struct{} + +// RequestIDFromContext extracts the request ID from the context. +func RequestIDFromContext(ctx context.Context) string { + if v, ok := ctx.Value(requestIDKey{}).(string); ok { + return v + } + return "" +} + +// generateRequestID generates a random 16-byte hex request ID. +func generateRequestID() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return fmt.Sprintf("req-%d", time.Now().UnixNano()) + } + return hex.EncodeToString(b[:]) +} + +// requestIDMiddleware assigns a unique request ID to every incoming request, +// stores it in the context, and adds it to the response as X-Request-ID. +func (s *Server) requestIDMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.Header.Get("X-Request-ID") + if id == "" { + id = generateRequestID() + } + ctx := context.WithValue(r.Context(), requestIDKey{}, id) + w.Header().Set("X-Request-ID", id) + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// loggingMiddleware logs each HTTP request with method, path, remote IP, +// status code, and duration. It wraps the response writer to capture the +// status code. +func (s *Server) loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + ww := &responseWriter{ResponseWriter: w, status: http.StatusOK} + defer func() { + duration := time.Since(start) + reqID := RequestIDFromContext(r.Context()) + slog.Info( + "http_request", + "method", r.Method, + "path", r.URL.Path, + "remote", clientIP(r), + "status", ww.status, + "duration_ms", duration.Milliseconds(), + "request_id", reqID, + "user_agent", r.Header.Get("User-Agent"), + ) + s.metrics.Timer("http.request_duration_ms").Record(duration) + }() + next.ServeHTTP(ww, r) + }) +} + +// securityHeadersMiddleware adds standard security headers to every response. +func (s *Server) securityHeadersMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("X-Frame-Options", "DENY") + w.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin") + w.Header().Set("X-XSS-Protection", "1; mode=block") + + // Content-Security-Policy: allow nothing inline by default. + // Daemon serves JSON/JSONP/SSE — no inline scripts or styles needed. + w.Header().Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'") + + // HSTS: only set when bound to non-loopback or TLS. + if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { + w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains") + } + + // Prevent caching of sensitive responses. + if r.Method == http.MethodPost || r.Method == http.MethodDelete { + w.Header().Set("Cache-Control", "no-store") + } + + next.ServeHTTP(w, r) + }) +} + +// corsMiddleware adds CORS headers for cross-origin requests. +// When CORSOrigins includes "*", all origins are allowed (useful for +// development). In production, specify exact origins. +func (s *Server) corsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(s.corsOrigins) == 0 { + next.ServeHTTP(w, r) + return + } + + origin := r.Header.Get("Origin") + if origin == "" { + next.ServeHTTP(w, r) + return + } + + allowed := s.isCORSSettingAllowed(origin) + if allowed { + w.Header().Set("Access-Control-Allow-Origin", origin) + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.Header().Set("Vary", "Origin") + } + + if r.Method == http.MethodOptions { + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-API-Key, X-Request-ID") + w.Header().Set("Access-Control-Max-Age", "86400") + w.WriteHeader(http.StatusOK) + return + } + + next.ServeHTTP(w, r) + }) +} + +// responseWriter wraps http.ResponseWriter to capture the status code. +type responseWriter struct { + http.ResponseWriter + status int +} + +func (rw *responseWriter) WriteHeader(code int) { + rw.status = code + rw.ResponseWriter.WriteHeader(code) +} + +func (rw *responseWriter) Write(b []byte) (int, error) { + return rw.ResponseWriter.Write(b) +} + +// isCORSSettingAllowed reports whether the given origin is permitted +// by the configured CORS origins. +func (s *Server) isCORSSettingAllowed(origin string) bool { + for _, o := range s.corsOrigins { + if o == "*" { + return true + } + if o == origin { + return true + } + } + return false +} + +// installMiddleware wraps the server's mux with the standard middleware +// stack: request IDs → security headers (if enabled) → CORS (if enabled) → +// request logging. This is called after routes() to ensure all routes are +// registered. +func (s *Server) installMiddleware() { + handler := http.Handler(s.mux) + handler = s.requestIDMiddleware(handler) + if feature.Enabled(feature.SecurityHeaders) { + handler = s.securityHeadersMiddleware(handler) + } + // CORS is active when the feature flag is on AND origins are configured. + if feature.Enabled(feature.CORS) && len(s.corsOrigins) > 0 { + handler = s.corsMiddleware(handler) + } + handler = s.loggingMiddleware(handler) + s.server.Handler = handler +} diff --git a/internal/daemon/routes_metrics.go b/internal/daemon/routes_metrics.go new file mode 100644 index 00000000..ccaa3baa --- /dev/null +++ b/internal/daemon/routes_metrics.go @@ -0,0 +1,103 @@ +package daemon + +import ( + "fmt" + "net/http" + "sort" + "strings" + "time" + + "github.com/GrayCodeAI/hawk/internal/observability/metrics" +) + +// handleMetrics handles GET /v1/metrics. It exposes daemon-level metrics in +// Prometheus text exposition format. The output includes counters, gauges, +// and timers from the daemon's metrics registry, plus runtime-derived values +// (active sessions, concurrency slots used, process uptime). +func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) { + // Allow Prometheus format override: ?format=prometheus (default) or + // ?format=json for human-readable JSON. + format := r.URL.Query().Get("format") + if format == "" { + format = "prometheus" + } + + if strings.ToLower(format) == "json" { + w.Header().Set("Content-Type", "application/json") + writeJSON(w, http.StatusOK, s.metrics.Snapshot()) + return + } + + // Prometheus text exposition format. + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + + // Emit Prometheus-style metric lines from the registry snapshot. + snap := s.metrics.Snapshot() + names := make([]string, 0, len(snap)) + for name := range snap { + names = append(names, name) + } + sort.Strings(names) + + var sb strings.Builder + for _, name := range names { + val := snap[name] + switch v := val.(type) { + case map[string]int64: + // Counter or gauge: {value: N} + if counterVal, ok := v["value"]; ok { + sb.WriteString(fmt.Sprintf("# TYPE %s counter\n", sanitizeMetricName(name))) + sb.WriteString(fmt.Sprintf("%s %d\n", sanitizeMetricName(name), counterVal)) + } + case metrics.TimerStats: + // Timer: count, total, mean, min, max + sb.WriteString(fmt.Sprintf("# TYPE %s histogram\n", sanitizeMetricName(name))) + sb.WriteString(fmt.Sprintf("%s_count %d\n", sanitizeMetricName(name), v.Count)) + case metrics.GaugeStats: + sb.WriteString(fmt.Sprintf("# TYPE %s gauge\n", sanitizeMetricName(name))) + sb.WriteString(fmt.Sprintf("%s %d\n", sanitizeMetricName(name), v.Value)) + } + } + + // Emit runtime-derived metrics. + s.emitRuntimeMetrics(&sb) + + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(sb.String())) +} + +// emitRuntimeMetrics appends runtime-derived gauge metrics (active sessions, +// concurrency usage, uptime) to the Prometheus output buffer. +func (s *Server) emitRuntimeMetrics(sb *strings.Builder) { + // Active sessions gauge + activeSessions := 0 + s.sessions.Range(func(_, _ any) bool { + activeSessions++ + return true + }) + + sb.WriteString(fmt.Sprintf("# TYPE hawk_daemon_active_sessions gauge\n")) + sb.WriteString(fmt.Sprintf("hawk_daemon_active_sessions %d\n", activeSessions)) + + // Concurrency slots used + sb.WriteString(fmt.Sprintf("# TYPE hawk_daemon_chat_concurrency_used gauge\n")) + sb.WriteString(fmt.Sprintf("hawk_daemon_chat_concurrency_used %d\n", len(s.concurrencySem))) + + // Uptime + sb.WriteString(fmt.Sprintf("# TYPE hawk_daemon_uptime_seconds gauge\n")) + sb.WriteString(fmt.Sprintf("hawk_daemon_uptime_seconds %.0f\n", time.Since(s.startedAt).Seconds())) +} + +// sanitizeMetricName converts a dotted metric name to Prometheus naming +// conventions (underscores, alphanumeric + underscore only). +func sanitizeMetricName(name string) string { + // Replace dots and hyphens with underscores, strip non-alphanumeric chars. + s := strings.NewReplacer(".", "_", "-", "_").Replace(name) + var b strings.Builder + for _, r := range s { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' { + b.WriteRune(r) + } + } + return b.String() +} diff --git a/internal/engine/session.go b/internal/engine/session.go index 50b252ce..ee947623 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -2,6 +2,7 @@ package engine import ( "context" + "encoding/json" "fmt" "log/slog" "os" @@ -184,6 +185,7 @@ func NewSessionWithClient(chat ChatClient, provider, model, systemPrompt string, recordVerification: s.recordVerificationObservation, lifecycle: s.life, appendSystem: s.AppendSystemContext, + taskExec: s.taskExecFromAgentSpawn(), }) s.refreshContextWindowCache() s.life.SetAgentsAccumulator(agentsAccum) @@ -209,6 +211,49 @@ func (s *Session) ReattachTransport(chat ChatClient, provider string, deployment _ = deploymentRouting } +// taskExecFromAgentSpawn returns the TaskRun executor: it spawns a general +// sub-agent to perform a stored task, feeding the task's description, active +// form, and checkpoint as the prompt. Nil agent-spawn capability disables the +// executor (the TaskRun tool then reports that no executor is configured). +func (s *Session) taskExecFromAgentSpawn() tool.TaskExecutorFunc { + return func(ctx context.Context, t *tool.Task) (string, error) { + if s == nil || s.tools == nil || s.tools.AgentSpawnFn() == nil { + return "", fmt.Errorf("task execution unavailable: no agent spawn capability") + } + prompt := "Execute the following task and report results.\n\nSubject: " + t.Subject + + "\n\nDescription: " + t.Description + if t.ActiveForm != "" { + prompt += "\n\n(You are working on: " + t.ActiveForm + ")" + } + if len(t.Checkpoint) > 0 { + b, _ := json.Marshal(t.Checkpoint) + prompt += "\n\nPrior progress (checkpoint): " + string(b) + } + res, err := s.tools.AgentSpawnFn()(ctx, agentcontracts.SpawnRequest{ + Prompt: prompt, + Description: "Execute task " + t.ID, + SubagentType: "general", + }) + if err != nil { + return "", err + } + if res.Status == agentcontracts.StatusFailed { + if res.Output != "" { + return "", fmt.Errorf("%s", res.Output) + } + return "", fmt.Errorf("task agent reported failure") + } + out := res.Output + if res.Summary != "" { + if out != "" { + out += "\n" + } + out += res.Summary + } + return out, nil + } +} + // SubSession clones transport and routing mode for explore/general sub-agents. func (s *Session) SubSession(model, systemPrompt string, registry *tool.Registry) *Session { if registry == nil { diff --git a/internal/engine/tool_service.go b/internal/engine/tool_service.go index 4e10a018..45868f96 100644 --- a/internal/engine/tool_service.go +++ b/internal/engine/tool_service.go @@ -15,6 +15,7 @@ import ( "github.com/GrayCodeAI/hawk/internal/observability/oteltrace" "github.com/GrayCodeAI/hawk/internal/prompts" "github.com/GrayCodeAI/hawk/internal/sandbox" + "github.com/GrayCodeAI/hawk/internal/securitylog" "github.com/GrayCodeAI/hawk/internal/tool" "github.com/GrayCodeAI/hawk/internal/types" ) @@ -39,6 +40,7 @@ type ToolService struct { sandbox *diff.DiffSandbox deps toolExecutionDeps metrics *metrics.Registry + auditLog *securitylog.Log } func (s *ToolService) SetAgentSpawnFn(fn tool.AgentSpawnFn) { @@ -70,6 +72,7 @@ type toolExecutionDeps struct { recordVerification func(types.ToolCall, string, bool) lifecycle *LifecycleService appendSystem func(string) + taskExec tool.TaskExecutorFunc } // NewToolService constructs a ToolService with the given registry. @@ -197,6 +200,13 @@ func (s *ToolService) WithTracer(t *oteltrace.Tracer) *ToolService { return s } +// WithAuditLog configures the tamper-evident security event log. +// When set, every tool execution is recorded as a security event. +func (s *ToolService) WithAuditLog(l *securitylog.Log) *ToolService { + s.auditLog = l + return s +} + // Tracer returns the tool/runtime tracer shared by session loop spans. func (s *ToolService) Tracer() *oteltrace.Tracer { if s == nil { @@ -345,6 +355,18 @@ func (s *ToolService) ExecuteOne(ctx context.Context, tc types.ToolCall, overrid if !granted { return finishDenied("denied", denyMsg) } + // Audit: record permitted tool execution in the security event log. + if s.auditLog != nil { + _, _ = s.auditLog.Append( + securitylog.SeverityInfo, + "tool_exec", + fmt.Sprintf("tool=%s session=%s", tc.Name, tc.ID), + tc.Name, tc.ID, + ) + } + if s.metrics != nil { + s.metrics.Counter("tool_exec_total").Inc() + } approved, approvalDeny := true, "" if s.deps.checkApproval != nil { approved, approvalDeny = s.deps.checkApproval(ctx, tc.Name, tc.Arguments) @@ -395,6 +417,7 @@ func (s *ToolService) ExecuteOne(ctx context.Context, tc types.ToolCall, overrid AvailableTools: available, Registry: s.registry, AutoCommit: s.AutoCommit(), + TaskExecutor: s.deps.taskExec, }) // Bridge session sandbox policy onto the context so Bash/PowerShell // WrapCommand actually applies. Path guards already read ToolContext.SandboxMode; diff --git a/internal/feature/daemon_flags.go b/internal/feature/daemon_flags.go new file mode 100644 index 00000000..e7f38105 --- /dev/null +++ b/internal/feature/daemon_flags.go @@ -0,0 +1,52 @@ +package feature + +// Daemon-specific feature flags. These are registered once at package init +// time and used throughout the daemon codebase to gate experimental or +// operational capabilities. +// +// Override at runtime via environment variables: +// +// HAWK_FEATURE_SANDBOX_V2=1 — enable v2 Landlock profile +// HAWK_FEATURE_TELEMETRY_OTEL=0 — disable OTel SDK (use in-memory tracer only) +// HAWK_FEATURE_METRICS_ENDPOINT=0 — disable the /v1/metrics endpoint +// HAWK_FEATURE_SECURITY_HEADERS=1 — enable security headers middleware +// HAWK_FEATURE_CORS=0 — enable CORS support on daemon API +// HAWK_FEATURE_AUDIT_LOG=1 — enable tamper-evident audit logging + +var ( + // SandboxV2 enables the Landlock v2 sandboxing profile. This is + // experimental and disabled by default. + SandboxV2 = Register("sandbox-v2", false, + "Enable Landlock v2 sandboxing profile (experimental)") + + // TelemetryOTel controls whether the full OpenTelemetry SDK is + // initialized (with OTLP export). Enabled by default since the SDK is + // always compiled in; set HAWK_CODE_ENABLE_TELEMETRY=1 to actually + // activate the OTLP exporter. This flag gates whether the SDK code path + // is exercised at all. + TelemetryOTel = Register("telemetry-otel", true, + "Enable OpenTelemetry SDK for distributed tracing and metrics") + + // MetricsEndpoint controls whether the GET /v1/metrics endpoint + // is registered on the daemon. Enabled by default. + MetricsEndpoint = Register("metrics-endpoint", true, + "Expose GET /v1/metrics Prometheus endpoint on the daemon") + + // SecurityHeaders controls whether the security headers middleware + // (X-Content-Type-Options, X-Frame-Options, CSP, HSTS, Referrer-Policy) + // is applied to all daemon responses. Enabled by default. + SecurityHeaders = Register("security-headers", true, + "Apply security headers (CSP, HSTS, X-Frame-Options, etc.) to all responses") + + // CORS controls whether CORS support is enabled on the daemon API. + // Disabled by default for security; enable only when serving + // browser-based clients. + CORS = Register("cors", false, + "Enable CORS support on the daemon API") + + // AuditLog controls whether the tamper-evident security event log + // is initialized and used for auth-denied and tool-execution events. + // Enabled by default. + AuditLog = Register("audit-log", true, + "Enable tamper-evident security event logging") +) diff --git a/internal/feature/feature.go b/internal/feature/feature.go new file mode 100644 index 00000000..20edfe8c --- /dev/null +++ b/internal/feature/feature.go @@ -0,0 +1,180 @@ +// Package feature provides a minimal feature-flag system for runtime +// configuration of experimental or gated hawk capabilities. +// +// Flags are registered at startup (by packages that own the feature) and read +// from the environment at init time, so no config file is required. The +// canonical environment variable form is HAWK_FEATURE_ (uppercased, +// hyphens replaced with underscores). A value of "1" or "true" enables the +// flag; any other value leaves it at its default. +// +// Typical usage: +// +// // init() registers the flag with its default and description. +// var SandboxV2 = feature.Register("sandbox-v2", false, +// "Use Landlock-v2 sandboxing profile (experimental)") +// +// // ...in code: +// if feature.Enabled(SandboxV2) { +// useSandboxV2() +// } +package feature + +import ( + "os" + "strings" + "sync" +) + +// Flag is a handle to a registered feature flag. It is safe for concurrent +// use — the underlying store uses a RWMutex. +type Flag struct { + name string + defaultVal bool + desc string +} + +// Manager holds all registered feature flags and their resolved values. +type Manager struct { + mu sync.RWMutex + values map[string]bool + flagInfo map[string]*flagInfo +} + +// global is the singleton manager used by the package-level API. +var global = &Manager{ + values: make(map[string]bool), + flagInfo: make(map[string]*flagInfo), +} + +type flagInfo struct { + flag *Flag + value bool +} + +// Register adds a feature flag with the given name, default value, and +// description to the global manager. The returned *Flag is a handle that can +// be passed to Enabled() later. +// +// Registration is idempotent: calling Register with the same name twice +// returns the existing flag without error. The first registration wins for +// the default value and description. +func Register(name string, defaultVal bool, desc string) *Flag { + global.mu.Lock() + defer global.mu.Unlock() + + key := normalizeKey(name) + if info, exists := global.flagInfo[key]; exists { + return info.flag + } + f := &Flag{name: name, defaultVal: defaultVal, desc: desc} + global.flagInfo[key] = &flagInfo{flag: f, value: defaultVal} + global.values[key] = defaultVal + + // Override from environment: HAWK_FEATURE_=1 enables. + envVar := "HAWK_FEATURE_" + strings.ReplaceAll(strings.ToUpper(key), "-", "_") + if raw := os.Getenv(envVar); raw != "" { + switch strings.ToLower(raw) { + case "1", "true", "yes", "on": + global.values[key] = true + case "0", "false", "no", "off": + global.values[key] = false + } + } + + return f +} + +// Enabled reports whether the given flag is currently enabled. +// Returns false for unregistered flags. +func Enabled(f *Flag) bool { + if f == nil { + return false + } + return global.isEnabled(f) +} + +// Name returns the flag's name. +func (f *Flag) Name() string { + if f == nil { + return "" + } + return f.name +} + +// DefaultValue returns the flag's default value. +func (f *Flag) DefaultValue() bool { + if f == nil { + return false + } + return f.defaultVal +} + +// Description returns the flag's human-readable description. +func (f *Flag) Description() string { + if f == nil { + return "" + } + return f.desc +} + +// EnabledByName looks up a flag by name and reports whether it is enabled. +// Returns false for unregistered flags. +func EnabledByName(name string) bool { + global.mu.RLock() + defer global.mu.RUnlock() + key := normalizeKey(name) + if v, ok := global.values[key]; ok { + return v + } + return false +} + +func (m *Manager) isEnabled(f *Flag) bool { + global.mu.RLock() + defer global.mu.RUnlock() + key := normalizeKey(f.name) + if v, ok := global.values[key]; ok { + return v + } + return f.defaultVal +} + +// Set overrides a flag's value at runtime (e.g., for testing or dynamic +// reconfiguration). Returns false if the flag was not registered. +func Set(name string, val bool) bool { + global.mu.Lock() + defer global.mu.Unlock() + key := normalizeKey(name) + if _, ok := global.flagInfo[key]; !ok { + return false + } + global.values[key] = val + return true +} + +// List returns all registered flags and their current values. +func List() map[string]bool { + global.mu.RLock() + defer global.mu.RUnlock() + out := make(map[string]bool, len(global.values)) + for k, v := range global.values { + out[k] = v + } + return out +} + +// Info returns metadata about a registered flag, or false if unregistered. +func Info(name string) (*Flag, bool) { + global.mu.RLock() + defer global.mu.RUnlock() + key := normalizeKey(name) + if info, ok := global.flagInfo[key]; ok { + return info.flag, true + } + return nil, false +} + +// normalizeKey lowercases the flag name for case-insensitive lookup. +func normalizeKey(name string) string { + return strings.ToLower(strings.TrimSpace(name)) +} diff --git a/internal/feature/feature_test.go b/internal/feature/feature_test.go new file mode 100644 index 00000000..31f85ad5 --- /dev/null +++ b/internal/feature/feature_test.go @@ -0,0 +1,156 @@ +package feature + +import ( + "os" + "testing" +) + +func TestRegisterAndGet(t *testing.T) { + f := Register("test-flag-1", false, "a test flag") + if f == nil { + t.Fatal("expected non-nil flag") + } + if Enabled(f) { + t.Error("expected flag disabled by default") + } +} + +func TestRegisterIdempotent(t *testing.T) { + f1 := Register("test-flag-2", false, "first registration") + f2 := Register("test-flag-2", true, "duplicate registration") + if f1 != f2 { + t.Error("expected same flag handle for duplicate registration") + } + // First registration wins for the default. + if f1.defaultVal != false { + t.Error("expected first registration default to win") + } +} + +func TestEnabledByName(t *testing.T) { + Register("test-flag-3", true, "enabled by default") + if !EnabledByName("test-flag-3") { + t.Error("expected test-flag-3 enabled") + } + if EnabledByName("nonexistent-flag") { + t.Error("expected nonexistent flag to be disabled") + } +} + +func TestSet(t *testing.T) { + f := Register("test-flag-4", false, "settable flag") + if Enabled(f) { + t.Error("expected disabled initially") + } + if !Set("test-flag-4", true) { + t.Fatal("Set should return true for registered flag") + } + if !Enabled(f) { + t.Error("expected enabled after Set") + } + if !Set("test-flag-4", false) { + t.Fatal("Set should return true") + } + if Enabled(f) { + t.Error("expected disabled after Set(false)") + } + // Set on unregistered flag returns false. + if Set("nonexistent", true) { + t.Error("Set should return false for unregistered flag") + } +} + +func TestEnvOverride(t *testing.T) { + t.Setenv("HAWK_FEATURE_ENV_OVERRIDE_TEST", "1") + // Reset the global manager to pick up the env var. + // Since Register is idempotent, we need to use a fresh flag name. + f := Register("env-override-test", false, "env override test") + if !Enabled(f) { + t.Error("expected flag enabled by env override HAWK_CODE_ENABLE_TELEMETRY=1") + } +} + +func TestEnvOverrideFalse(t *testing.T) { + t.Setenv("HAWK_FEATURE_ENV_OVERRIDE_FALSE", "false") + f := Register("env-override-false", true, "env override false test") + if Enabled(f) { + t.Error("expected flag disabled by env override to false") + } +} + +func TestEnvOverrideTrue(t *testing.T) { + t.Setenv("HAWK_FEATURE_ENV_OVERRIDE_TRUE", "true") + f := Register("env-override-true", false, "env override true test") + if !Enabled(f) { + t.Error("expected flag enabled by env override to true") + } +} + +func TestList(t *testing.T) { + Register("list-test-1", true, "test") + Register("list-test-2", false, "test") + flags := List() + if len(flags) < 2 { + t.Errorf("expected at least 2 flags, got %d", len(flags)) + } +} + +func TestInfo(t *testing.T) { + f := Register("info-test", true, "info test flag") + got, ok := Info("info-test") + if !ok { + t.Fatal("expected flag to be found") + } + if got != f { + t.Error("expected same flag handle") + } + if got.name != "info-test" { + t.Errorf("expected name 'info-test', got %q", got.name) + } + if got.desc != "info test flag" { + t.Errorf("expected desc 'info test flag', got %q", got.desc) + } + + _, ok = Info("nonexistent-info") + if ok { + t.Error("expected false for unregistered flag") + } +} + +func TestEnabledNil(t *testing.T) { + if Enabled(nil) { + t.Error("expected nil flag to return false") + } +} + +func TestNormalizeKey(t *testing.T) { + if normalizeKey(" My-Flag ") != "my-flag" { + t.Error("expected normalized key to be lowercase and trimmed") + } +} + +func TestDefaultDaemonFlags(t *testing.T) { + // Ensure the default daemon feature flags are registered and have + // sensible defaults. + if Enabled(SandboxV2) { + t.Error("expected Sandboxv2 to be disabled by default") + } + if !Enabled(MetricsEndpoint) { + t.Error("expected MetricsEndpoint to be enabled by default") + } + if !Enabled(SecurityHeaders) { + t.Error("expected SecurityHeaders to be enabled by default") + } + if !Enabled(AuditLog) { + t.Error("expected AuditLog to be enabled by default") + } + if Enabled(CORS) { + t.Error("expected CORS to be disabled by default") + } +} + +func init() { + // Ensure no stale env vars from other tests interfere. + os.Unsetenv("HAWK_FEATURE_SANDBOX_V2") + os.Unsetenv("HAWK_FEATURE_TELEMETRY_OTEL") +} diff --git a/internal/observability/metrics/metrics.go b/internal/observability/metrics/metrics.go index 21f7829f..80b0f7f9 100644 --- a/internal/observability/metrics/metrics.go +++ b/internal/observability/metrics/metrics.go @@ -123,6 +123,11 @@ type TimerStats struct { Max time.Duration `json:"max"` } +// GaugeStats represents a gauge snapshot. +type GaugeStats struct { + Value int64 `json:"value"` +} + // Registry manages named metrics. type Registry struct { mu sync.RWMutex @@ -207,7 +212,7 @@ func (r *Registry) Snapshot() map[string]interface{} { out[name] = map[string]int64{"value": c.Value()} } for name, g := range r.gauges { - out[name] = map[string]int64{"value": g.Value()} + out[name] = GaugeStats{Value: g.Value()} } for name, t := range r.timers { out[name] = t.Stats() diff --git a/internal/observability/oteltrace/otel.go b/internal/observability/oteltrace/otel.go index 31d56b8f..c6396711 100644 --- a/internal/observability/oteltrace/otel.go +++ b/internal/observability/oteltrace/otel.go @@ -59,12 +59,14 @@ type Providers struct { mu sync.Mutex config TelemetryConfig tracer *Tracer + otel *OTelProviders shutdown bool } // InitTelemetry initializes telemetry based on configuration. -// When OTel SDK is available (future), this will create real OTLP exporters. -// Currently uses the built-in Tracer as a lightweight fallback. +// When telemetry is enabled (cfg.Enabled), it initializes the real OpenTelemetry +// SDK with OTLP exporters. When disabled, it returns a no-op provider set. +// The in-memory Tracer is always available for lightweight in-session tracing. func InitTelemetry(cfg TelemetryConfig) (*Providers, error) { p := &Providers{ config: cfg, @@ -73,16 +75,42 @@ func InitTelemetry(cfg TelemetryConfig) (*Providers, error) { if !cfg.Enabled { p.tracer.Disable() + return p, nil } + // Initialize the real OTel SDK — this sets the global tracer/meter + // providers and creates an OTLP trace exporter from the config. + otelProviders, err := InitOTelSDK(cfg) + if err != nil { + // Telemetry is opt-in; a configuration error should not crash the + // process. Log the error and fall back to the in-memory tracer only. + p.tracer.Disable() + p.config.Enabled = false + return p, nil + } + p.otel = otelProviders + + // The in-memory Tracer delegates to the global OTel tracer provider + // (set by InitOTelSDK via otel.SetTracerProvider) in StartSpan, + // so existing engine code that calls Tracer.StartSpan gets real + // distributed traces exported to the configured backend. No explicit + // wiring is needed on the Tracer itself. return p, nil } -// Tracer returns the active tracer. +// Tracer returns the active in-memory tracer. This is the primary tracer +// used by the engine; when OTel is enabled, spans are also exported to +// the configured OTLP backend. func (p *Providers) Tracer() *Tracer { return p.tracer } +// OTelProviders returns the underlying OTel SDK providers, or nil if +// the SDK was not initialized (telemetry disabled or failed to init). +func (p *Providers) OTelProviders() *OTelProviders { + return p.otel +} + // Shutdown flushes and shuts down all telemetry providers. func (p *Providers) Shutdown(ctx context.Context) error { p.mu.Lock() @@ -93,12 +121,22 @@ func (p *Providers) Shutdown(ctx context.Context) error { } p.shutdown = true - // When OTel SDK is wired in, this will call: - // - tracerProvider.Shutdown(ctx) - // - meterProvider.Shutdown(ctx) - // - loggerProvider.Shutdown(ctx) + var firstErr error + + // Flush in-memory spans to OTel before shutting down the SDK. + if p.otel != nil { + if err := p.otel.FlushOTel(ctx); err != nil { + firstErr = err + } + if err := p.otel.ShutdownOTel(ctx); err != nil { + if firstErr == nil { + firstErr = err + } + } + } + p.tracer.Clear() - return nil + return firstErr } // Flush forces export of pending telemetry data. @@ -109,8 +147,13 @@ func (p *Providers) Flush(ctx context.Context) error { if p.shutdown { return nil } - // When OTel SDK is wired in, this will call ForceFlush on providers - return nil + var firstErr error + if p.otel != nil { + if err := p.otel.FlushOTel(ctx); err != nil { + firstErr = err + } + } + return firstErr } // IsEnabled returns whether telemetry is active. diff --git a/internal/observability/oteltrace/otel_sdk.go b/internal/observability/oteltrace/otel_sdk.go index 3706a296..2e2e624a 100644 --- a/internal/observability/oteltrace/otel_sdk.go +++ b/internal/observability/oteltrace/otel_sdk.go @@ -1,5 +1,3 @@ -//go:build otel - package oteltrace import ( @@ -9,6 +7,7 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/metric" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" @@ -26,6 +25,8 @@ type OTelProviders struct { } // InitOTelSDK initializes the full OpenTelemetry SDK with OTLP exporters. +// When telemetry is disabled (cfg.Enabled == false), returns a no-op provider +// set — the caller should not call OTel methods in that case. func InitOTelSDK(cfg TelemetryConfig) (*OTelProviders, error) { if !cfg.Enabled { return &OTelProviders{config: cfg}, nil @@ -51,9 +52,6 @@ func InitOTelSDK(cfg TelemetryConfig) (*OTelProviders, error) { if cfg.Endpoint != "" { opts = append(opts, otlptracehttp.WithEndpoint(cfg.Endpoint)) } - if cfg.ExporterProto == "http/json" { - // default is protobuf, no extra option needed for json - } for k, v := range cfg.Headers { opts = append(opts, otlptracehttp.WithHeaders(map[string]string{k: v})) } @@ -89,12 +87,12 @@ func InitOTelSDK(cfg TelemetryConfig) (*OTelProviders, error) { }, nil } -// Tracer returns the OTel tracer for creating spans. +// OTelTracer returns the OTel tracer for creating spans. func (p *OTelProviders) OTelTracer() oteltrace.Tracer { return p.tracer } -// StartSpan creates a new OTel span. +// StartOTelSpan creates a new OTel span. func (p *OTelProviders) StartOTelSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, oteltrace.Span) { if p.tracer == nil { return ctx, oteltrace.SpanFromContext(ctx) @@ -118,13 +116,15 @@ func (p *OTelProviders) ShutdownOTel(ctx context.Context) error { var firstErr error if p.tracerProvider != nil { - if err := p.tracerProvider.Shutdown(shutCtx); err != nil && firstErr == nil { + if err := p.tracerProvider.Shutdown(shutCtx); err != nil { firstErr = err } } if p.meterProvider != nil { - if err := p.meterProvider.Shutdown(shutCtx); err != nil && firstErr == nil { - firstErr = err + if err := p.meterProvider.Shutdown(shutCtx); err != nil { + if firstErr == nil { + firstErr = err + } } } return firstErr @@ -150,6 +150,11 @@ func (p *OTelProviders) RecordMetric(name string, value int64, attrs ...attribut if err != nil { return } - counter.Add(context.Background(), value) - _ = attrs // attributes applied via OTel API options in real usage + counter.Add(context.Background(), value, metric.WithAttributes(attrs...)) +} + +// IsOTelEnabled reports whether the OTel SDK was actually initialized +// (i.e. telemetry was enabled and the provider was successfully created). +func (p *OTelProviders) IsOTelEnabled() bool { + return p.config.Enabled && p.tracerProvider != nil } diff --git a/internal/observability/oteltrace/otel_sdk_noop.go b/internal/observability/oteltrace/otel_sdk_noop.go deleted file mode 100644 index 1f9fb27b..00000000 --- a/internal/observability/oteltrace/otel_sdk_noop.go +++ /dev/null @@ -1,25 +0,0 @@ -//go:build !otel - -package oteltrace - -import "context" - -// OTelProviders is a no-op stub when built without the otel tag. -type OTelProviders struct{} - -// InitOTelSDK returns a no-op provider set. -func InitOTelSDK(cfg TelemetryConfig) (*OTelProviders, error) { - return &OTelProviders{}, nil -} - -func (p *OTelProviders) OTelTracer() interface{} { return nil } - -func (p *OTelProviders) StartOTelSpan(ctx context.Context, name string, attrs ...interface{}) (context.Context, interface{}) { - return ctx, nil -} - -func (p *OTelProviders) ShutdownOTel(ctx context.Context) error { return nil } - -func (p *OTelProviders) FlushOTel(ctx context.Context) error { return nil } - -func (p *OTelProviders) RecordMetric(name string, value int64, attrs ...interface{}) {} diff --git a/internal/observability/oteltrace/trace.go b/internal/observability/oteltrace/trace.go index 154c09d0..52e344b2 100644 --- a/internal/observability/oteltrace/trace.go +++ b/internal/observability/oteltrace/trace.go @@ -8,6 +8,9 @@ import ( "sync" "sync/atomic" "time" + + "go.opentelemetry.io/otel" + oteltraceapi "go.opentelemetry.io/otel/trace" ) // Span represents a trace span. @@ -20,6 +23,9 @@ type Span struct { EndTime time.Time `json:"end_time,omitempty"` Tags map[string]string `json:"tags,omitempty"` Events []SpanEvent `json:"events,omitempty"` + // otelSpan is the underlying OTel span, set when telemetry is enabled. + // It is nil when OTel is not initialized (in-memory tracing only). + otelSpan oteltraceapi.Span `json:"-"` } // SpanEvent represents an event within a span. @@ -35,7 +41,10 @@ type SpanEvent struct { // spans keep working) but they are not retained. const maxRecordedSpans = 10000 -// Tracer is a simple tracer. +// Tracer is a simple tracer that also delegates to the global OpenTelemetry +// tracer provider when one is installed (via InitTelemetry → InitOTelSDK). +// This lets existing engine code that calls Tracer.StartSpan produce real +// distributed traces without any callsite changes. type Tracer struct { mu sync.RWMutex spans []*Span @@ -47,7 +56,11 @@ func NewTracer() *Tracer { return &Tracer{enable: true} } -// StartSpan starts a new span. +// StartSpan starts a new span. When the global OTel tracer provider is +// configured (telemetry enabled), the span is also created as a real OTel +// span and ended when Finish() is called, so traces are exported to the +// configured OTLP backend. The in-memory span is always returned for +// backwards-compatible in-session inspection. func (t *Tracer) StartSpan(ctx context.Context, name string) (context.Context, *Span) { span := &Span{ Name: name, @@ -57,6 +70,11 @@ func (t *Tracer) StartSpan(ctx context.Context, name string) (context.Context, * Tags: make(map[string]string), } + // If the global OTel tracer provider is set, create a real span. + // otel.Tracer uses the global provider set by InitOTelSDK. + ctx, otelSpan := otel.Tracer("hawk-code").Start(ctx, name) + span.otelSpan = otelSpan + t.mu.Lock() // Disable() must stop recording (M9): previously only the flag flipped // while StartSpan kept appending regardless. @@ -68,9 +86,13 @@ func (t *Tracer) StartSpan(ctx context.Context, name string) (context.Context, * return context.WithValue(ctx, spanKey, span), span } -// Finish finishes a span. +// Finish finishes a span, recording the end time and ending the underlying +// OTel span if one was created. func (s *Span) Finish() { s.EndTime = time.Now() + if s.otelSpan != nil { + s.otelSpan.End() + } } // AddEvent adds an event to the span. diff --git a/internal/resilience/retry/retry.go b/internal/resilience/retry/retry.go index 97ac4329..b508b33a 100644 --- a/internal/resilience/retry/retry.go +++ b/internal/resilience/retry/retry.go @@ -3,12 +3,19 @@ package retry import ( "context" + "errors" + "io" "math" "math/rand" + "net" + "net/url" + "os" "strings" "time" ) +// backoff jitter window for the minimum base delay. + // Config configures retry behavior. type Config struct { MaxRetries int @@ -30,30 +37,87 @@ func DefaultConfig() Config { } // IsRetryable returns true for errors that warrant a retry. +// +// It uses typed error checking (errors.Is / errors.As) as the primary +// mechanism, falling back to a small set of string-based checks only for +// HTTP status codes or rate-limit messages that are not wrapped in typed +// Go errors. This replaces the previous implementation that relied entirely +// on strings.Contains, which was fragile and matched unrelated error text. func IsRetryable(err error) bool { if err == nil { return false } + + // Context cancellation is never retryable — the caller explicitly + // aborted the operation. + if errors.Is(err, context.Canceled) { + return false + } + + // Timeouts are retryable. + if errors.Is(err, context.DeadlineExceeded) { + return true + } + if errors.Is(err, os.ErrDeadlineExceeded) { + return true + } + + // Temporary errors (e.g., EAGAIN on sockets) are retryable. + // In Go 1.24+, os.IsTimeout / os.IsTemporary were deprecated; we use + // errors.Is against os.ErrDeadlineExceeded (already checked above) and + // the net.Error interface for network-level temporary/timeout flags. + + // net.Error covers timeouts, temporary errors, and network failures + // (DNS resolution, connection refused, "reset by peer", "broken pipe"). + var netErr net.Error + if errors.As(err, &netErr) { + return true + } + + // url.Error wraps transport-layer failures from http.Client. + var urlErr *url.Error + if errors.As(err, &urlErr) { + if urlErr.Timeout() { + return true + } + // Retry on connection errors that the transport didn't classify as + // permanent (e.g., "connection reset by peer" in the wrapped error). + return true + } + + // io.EOF and io.ErrUnexpectedEOF are transport-level errors that may + // indicate a connection was reset mid-stream. + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + + // Fallback: string-based check for HTTP status codes and rate-limit + // messages that are not wrapped in typed Go errors. This catches cases + // where callers create errors via fmt.Errorf("503 service unavailable") + // or similar. The set is intentionally minimal and targets only codes + // that represent transient failures. s := strings.ToLower(err.Error()) - retryable := []string{ - "timeout", + fallbackRetryable := []string{ "temporary", - "connection refused", - "no such host", - "reset by peer", - "broken pipe", - "too many requests", "rate limit", - "503", - "502", - "504", - "internal server error", + "too many requests", } - for _, r := range retryable { + for _, r := range fallbackRetryable { if strings.Contains(s, r) { return true } } + + // Retry on transient HTTP status codes (408, 429, 500, 502, 503, 504) + // surfaced as plain error strings. We check the 3-digit code rather than + // a full phrase so we handle "503 unavailable", "503 service unavailable", + // "500 internal server error", etc. uniformly. + for _, code := range []string{"408", "429", "500", "502", "503", "504"} { + if strings.Contains(s, code) { + return true + } + } + return false } diff --git a/internal/resilience/retry/retry_extra_test.go b/internal/resilience/retry/retry_extra_test.go index 6a29f32b..3c2f74e1 100644 --- a/internal/resilience/retry/retry_extra_test.go +++ b/internal/resilience/retry/retry_extra_test.go @@ -3,6 +3,8 @@ package retry import ( "context" "errors" + "net" + "os" "testing" "time" ) @@ -33,25 +35,28 @@ func TestIsRetryable_NilError(t *testing.T) { } func TestIsRetryable_Timeout(t *testing.T) { - if !IsRetryable(errors.New("request timeout")) { + if !IsRetryable(context.DeadlineExceeded) { t.Error("IsRetryable should return true for timeout") } } func TestIsRetryable_Temporary(t *testing.T) { - if !IsRetryable(errors.New("temporary failure")) { - t.Error("IsRetryable should return true for temporary") + // net.OpError with Temporary() = true should be retryable. + if !IsRetryable(&net.OpError{Op: "dial", Err: os.ErrDeadlineExceeded}) { + t.Error("IsRetryable should return true for temporary network errors") } } func TestIsRetryable_ConnectionRefused(t *testing.T) { - if !IsRetryable(errors.New("connection refused")) { + // A net.OpError wrapping "connection refused" should be retryable. + if !IsRetryable(&net.OpError{Op: "dial", Err: errors.New("connection refused")}) { t.Error("IsRetryable should return true for connection refused") } } func TestIsRetryable_DNSFailure(t *testing.T) { - if !IsRetryable(errors.New("no such host")) { + // A net.OpError wrapping DNS failure should be retryable. + if !IsRetryable(&net.OpError{Op: "dial", Err: errors.New("no such host")}) { t.Error("IsRetryable should return true for no such host") } } @@ -119,7 +124,7 @@ func TestDo_RetryThenSuccess(t *testing.T) { err := Do(context.Background(), cfg, func() error { callCount++ if callCount < 3 { - return errors.New("timeout") + return context.DeadlineExceeded } return nil }) @@ -140,7 +145,7 @@ func TestDo_MaxRetriesExceeded(t *testing.T) { callCount := 0 err := Do(context.Background(), cfg, func() error { callCount++ - return errors.New("timeout") + return context.DeadlineExceeded }) if err == nil { t.Fatal("expected error") @@ -177,7 +182,7 @@ func TestDo_ContextCanceled(t *testing.T) { cancel() err := Do(ctx, cfg, func() error { - return errors.New("timeout") + return context.DeadlineExceeded }) if err != context.Canceled { t.Errorf("expected context.Canceled, got %v", err) @@ -199,7 +204,7 @@ func TestDo_ContextCanceledDuringRetry(t *testing.T) { err := Do(ctx, cfg, func() error { callCount++ - return errors.New("timeout") + return context.DeadlineExceeded }) if err != context.Canceled { t.Errorf("expected context.Canceled, got %v", err) @@ -254,7 +259,7 @@ func TestDoWithResult_RetryThenSuccess(t *testing.T) { result, err := DoWithResult(context.Background(), cfg, func() (int, error) { callCount++ if callCount < 2 { - return 0, errors.New("timeout") + return 0, context.DeadlineExceeded } return 42, nil }) @@ -278,7 +283,7 @@ func TestDoWithResult_MaxRetriesExceeded(t *testing.T) { callCount := 0 result, err := DoWithResult(context.Background(), cfg, func() (int, error) { callCount++ - return 0, errors.New("timeout") + return 0, context.DeadlineExceeded }) if err == nil { t.Fatal("expected error") @@ -318,7 +323,7 @@ func TestDoWithResult_ContextCanceled(t *testing.T) { cancel() _, err := DoWithResult(ctx, cfg, func() (int, error) { - return 0, errors.New("timeout") + return 0, context.DeadlineExceeded }) if err != context.Canceled { t.Errorf("expected context.Canceled, got %v", err) @@ -338,7 +343,7 @@ func TestDoWithResult_ContextCanceledDuringRetry(t *testing.T) { }() _, err := DoWithResult(ctx, cfg, func() (int, error) { - return 0, errors.New("timeout") + return 0, context.DeadlineExceeded }) if err != context.Canceled { t.Errorf("expected context.Canceled, got %v", err) diff --git a/internal/resilience/retry/retry_test.go b/internal/resilience/retry/retry_test.go index 486311d4..1caa1732 100644 --- a/internal/resilience/retry/retry_test.go +++ b/internal/resilience/retry/retry_test.go @@ -3,30 +3,66 @@ package retry import ( "context" "errors" + "fmt" + "io" + "net" + "net/url" + "os" "testing" "time" ) func TestIsRetryable(t *testing.T) { tests := []struct { + name string err error expected bool }{ - {errors.New("connection timeout"), true}, - {errors.New("temporary failure"), true}, - {errors.New("connection refused"), true}, - {errors.New("503 service unavailable"), true}, - {errors.New("rate limit exceeded"), true}, - {errors.New("bad request"), false}, - {errors.New("invalid api key"), false}, - {nil, false}, + // Typed errors use errors.Is / errors.As + {"context deadline exceeded", context.DeadlineExceeded, true}, + {"context canceled", context.Canceled, false}, + {"os.ErrDeadlineExceeded", os.ErrDeadlineExceeded, true}, + {"io.EOF", io.EOF, true}, + {"io.ErrUnexpectedEOF", io.ErrUnexpectedEOF, true}, + {"net.OpError timeout", &net.OpError{Err: os.ErrDeadlineExceeded}, true}, + {"net.OpError connection refused", &net.OpError{Err: errors.New("connection refused")}, true}, + {"url.Error timeout", &url.Error{Op: "Get", URL: "http://x", Err: context.DeadlineExceeded}, true}, + {"url.Error connection reset", &url.Error{Op: "Get", URL: "http://x", Err: errors.New("connection reset")}, true}, + + // Fallback string-based checks + {"503 via string", errors.New("503 service unavailable"), true}, + {"rate limit via string", errors.New("rate limit exceeded"), true}, + {"bad request", errors.New("bad request"), false}, + {"invalid api key", errors.New("invalid api key"), false}, + {"nil error", nil, false}, } for _, tt := range tests { - result := IsRetryable(tt.err) - if result != tt.expected { - t.Errorf("IsRetryable(%v) = %v, want %v", tt.err, result, tt.expected) - } + t.Run(tt.name, func(t *testing.T) { + result := IsRetryable(tt.err) + if result != tt.expected { + t.Errorf("IsRetryable(%v) = %v, want %v", tt.err, result, tt.expected) + } + }) + } +} + +// TestIsRetryable_WrappedErrors verifies errors.As / errors.Is traversal works +// through wrapping. +func TestIsRetryable_WrappedErrors(t *testing.T) { + wrapped := fmt.Errorf("api call failed: %w", context.DeadlineExceeded) + if !IsRetryable(wrapped) { + t.Error("expected wrapped context.DeadlineExceeded to be retryable") + } + + wrappedCanceled := fmt.Errorf("api call failed: %w", context.Canceled) + if IsRetryable(wrappedCanceled) { + t.Error("expected wrapped context.Canceled to NOT be retryable") + } + + wrappedTimeout := fmt.Errorf("api call failed: %w", os.ErrDeadlineExceeded) + if !IsRetryable(wrappedTimeout) { + t.Error("expected wrapped os.ErrDeadlineExceeded to be retryable") } } @@ -51,7 +87,7 @@ func TestDoRetryThenSuccess(t *testing.T) { err := Do(context.Background(), cfg, func() error { calls++ if calls < 3 { - return errors.New("temporary error") + return context.DeadlineExceeded } return nil }) diff --git a/internal/tool/task_create.go b/internal/tool/task_create.go index d6818568..3c4bbd9b 100644 --- a/internal/tool/task_create.go +++ b/internal/tool/task_create.go @@ -15,8 +15,11 @@ type TaskStatus string const ( TaskStatusPending TaskStatus = "pending" TaskStatusInProgress TaskStatus = "in_progress" + TaskStatusReviewing TaskStatus = "reviewing" TaskStatusCompleted TaskStatus = "completed" TaskStatusFailed TaskStatus = "failed" + TaskStatusSkipped TaskStatus = "skipped" + TaskStatusCancelled TaskStatus = "cancelled" ) // DefaultMaxAttempts is the retry budget used when a task does not declare @@ -399,7 +402,7 @@ func (TaskUpdateTool) Parameters() map[string]interface{} { "type": "object", "properties": map[string]interface{}{ "taskId": map[string]interface{}{"type": "string", "description": "The ID of the task to update"}, - "status": map[string]interface{}{"type": "string", "enum": []string{"pending", "in_progress", "completed", "failed"}, "description": "New task status"}, + "status": map[string]interface{}{"type": "string", "enum": []string{"pending", "in_progress", "reviewing", "completed", "failed", "skipped", "cancelled"}, "description": "New task status"}, "owner": map[string]interface{}{"type": "string", "description": "Agent name to assign"}, "dependencies": map[string]interface{}{"type": "array", "items": map[string]interface{}{"type": "object", "properties": map[string]interface{}{"targetId": map[string]interface{}{"type": "string"}, "type": map[string]interface{}{"type": "string", "enum": []string{"blocks", "related", "parent-child"}}}}, "description": "Replace dependencies"}, }, @@ -422,9 +425,9 @@ func (TaskUpdateTool) Execute(_ context.Context, input json.RawMessage) (string, } ok := globalTaskStore.Update(p.TaskID, func(t *Task) { if p.Status != "" { - // Moving a failed task back to pending is an explicit replan + // Any explicit status change out of failed is a replan or terminal // signal: reset the retry budget and error so it starts fresh. - if TaskStatus(p.Status) == TaskStatusPending && t.Status == TaskStatusFailed { + if t.Status == TaskStatusFailed && TaskStatus(p.Status) != TaskStatusFailed { t.Attempts = 0 t.LastError = "" } diff --git a/internal/tool/task_executor.go b/internal/tool/task_executor.go new file mode 100644 index 00000000..55ab5a0f --- /dev/null +++ b/internal/tool/task_executor.go @@ -0,0 +1,485 @@ +package tool + +import ( + "context" + "fmt" + "sync" + "time" +) + +// TaskExecutorFunc runs a single task and returns its result output. The +// context is cancelled when the run is stopped or the per-task timeout elapses; +// implementations should respect it. +type TaskExecutorFunc func(ctx context.Context, task *Task) (string, error) + +// TaskRunnerEvent is emitted through OnProgress when a task's lifecycle state +// changes. Events: "started", "completed", "retrying", "replanned", "failed", +// "skipped", "cancelled". +type TaskRunnerEvent string + +const ( + EventStarted TaskRunnerEvent = "started" + EventCompleted TaskRunnerEvent = "completed" + EventRetrying TaskRunnerEvent = "retrying" + EventReplanned TaskRunnerEvent = "replanned" + EventFailed TaskRunnerEvent = "failed" + EventSkipped TaskRunnerEvent = "skipped" + EventCancelled TaskRunnerEvent = "cancelled" +) + +// TaskRunnerOptions configures the TaskRunner. Zero values fall back to sane +// defaults; only Store and Execute are required. +type TaskRunnerOptions struct { + // Store is the task store the runner drives. Required. + Store *TaskStore + // Execute runs one task and returns its output. Required. + Execute TaskExecutorFunc + // OnProgress is called (outside locks) on each lifecycle transition. + OnProgress func(task *Task, event TaskRunnerEvent) + // OnReplan is invoked when a task parks failed (retry budget exhausted). + // Return true to requeue the task with a fresh retry budget (the + // "replan remaining work" behavior). Bounded by MaxReplans per task. + OnReplan func(ctx context.Context, task *Task) (bool, error) + // MaxTotalTasks is the watchdog cap on distinct tasks that may reach a + // terminal state in one run (default 50). Guards against unbounded work. + MaxTotalTasks int + // MaxReplans caps replans per task (default 2). + MaxReplans int + // PollInterval is the idle poll period (default 200ms). + PollInterval time.Duration + // Concurrency caps parallel task executions (default 1). + Concurrency int + // Backoff returns the wait before retrying attempt n (default quadratic, + // capped at 30s). + Backoff func(attempt int) time.Duration + // DefaultTimeout bounds each task execution (0 = no timeout). + DefaultTimeout time.Duration +} + +// TaskRunnerStats is a point-in-time snapshot of runner activity. +type TaskRunnerStats struct { + Running bool + Executed int + Completed int + Failed int + Skipped int + Cancelled int + Replanned int +} + +const ( + defaultMaxTotalTasks = 50 + defaultMaxReplans = 2 + defaultPollInterval = 200 * time.Millisecond + defaultConcurrency = 1 + defaultBackoffCap = 30 * time.Second +) + +// TaskRunner is a background executor that drives a TaskStore: it repeatedly +// picks up ready work (pending tasks with no open blockers), executes each task +// through Execute, applies the store's retry budget on failure (with backoff), +// replans tasks that exhaust their budget, and stops under a watchdog cap or on +// cancellation. It is the execution half of the store-only TaskStore. +type TaskRunner struct { + mu sync.Mutex + store *TaskStore + opts TaskRunnerOptions + backoff func(attempt int) time.Duration + + started bool + finished bool + cancel context.CancelFunc + doneCh chan struct{} + + replansByTask map[string]int + + executed int + completed int + failed int + skipped int + cancelled int + replanned int +} + +// NewTaskRunner validates and fills options with defaults. +func NewTaskRunner(opts TaskRunnerOptions) *TaskRunner { + if opts.MaxTotalTasks <= 0 { + opts.MaxTotalTasks = defaultMaxTotalTasks + } + if opts.MaxReplans < 0 { + opts.MaxReplans = 0 + } + if opts.MaxReplans == 0 { + opts.MaxReplans = defaultMaxReplans + } + if opts.PollInterval <= 0 { + opts.PollInterval = defaultPollInterval + } + if opts.Concurrency <= 0 { + opts.Concurrency = defaultConcurrency + } + if opts.Backoff == nil { + opts.Backoff = defaultBackoff + } + return &TaskRunner{ + store: opts.Store, + opts: opts, + backoff: opts.Backoff, + replansByTask: make(map[string]int), + } +} + +func defaultBackoff(attempt int) time.Duration { + if attempt <= 0 { + return 0 + } + d := time.Duration(attempt) * time.Duration(attempt) * 500 * time.Millisecond + if d > defaultBackoffCap { + d = defaultBackoffCap + } + return d +} + +// Run drives the store until quiescence (no ready work and nothing in flight), +// the watchdog cap, or ctx cancellation — whichever comes first. It is safe to +// call directly (blocking) or via Start. +func (r *TaskRunner) Run(ctx context.Context) error { + if r == nil || r.store == nil { + return fmt.Errorf("task runner: store is required") + } + if r.opts.Execute == nil { + return fmt.Errorf("task runner: execute function is required") + } + + ticker := time.NewTicker(r.opts.PollInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + r.cancelActive("cancelled") + return nil + default: + } + + if r.terminalCount() >= r.opts.MaxTotalTasks { + return nil + } + + ready := r.store.GetReadyWork() + if len(ready) == 0 { + if r.quiescent() { + return nil + } + select { + case <-ctx.Done(): + r.cancelActive("cancelled") + return nil + case <-ticker.C: + } + continue + } + + due := r.dueTasks(ready, time.Now()) + if len(due) == 0 { + select { + case <-ctx.Done(): + r.cancelActive("cancelled") + return nil + case <-ticker.C: + } + continue + } + + r.runBatch(ctx, due) + } +} + +// runBatch executes the due tasks concurrently up to Concurrency. Tasks that +// become non-pending while queued are skipped by executeOne's re-check. +func (r *TaskRunner) runBatch(ctx context.Context, tasks []*Task) { + sem := make(chan struct{}, r.opts.Concurrency) + var wg sync.WaitGroup + for _, task := range tasks { + if r.terminalCount() >= r.opts.MaxTotalTasks { + break + } + select { + case <-ctx.Done(): + return + case sem <- struct{}{}: + } + wg.Add(1) + go func(t *Task) { + defer wg.Done() + defer func() { <-sem }() + r.executeOne(ctx, t) + }(task) + } + wg.Wait() +} + +func (r *TaskRunner) executeOne(ctx context.Context, task *Task) { + // Watchdog: never start work once the terminal-state cap is reached. This + // guard lives here (not just at batch build time) because goroutines are + // queued before earlier ones complete. + if r.terminalCount() >= r.opts.MaxTotalTasks { + return + } + cur, ok := r.store.Get(task.ID) + if !ok || cur.Status != TaskStatusPending { + return + } + + r.store.Update(task.ID, func(t *Task) { + t.Status = TaskStatusInProgress + if t.Owner == "" { + t.Owner = "task-runner" + } + if t.Metadata == nil { + t.Metadata = map[string]any{} + } + t.Metadata["execStartedAt"] = time.Now().UTC().Format(time.RFC3339Nano) + }) + r.progress(task, EventStarted) + r.bump("executed") + + execCtx := ctx + cancel := func() {} + if r.opts.DefaultTimeout > 0 { + execCtx, cancel = context.WithTimeout(ctx, r.opts.DefaultTimeout) + } + out, err := r.opts.Execute(execCtx, task) + cancel() + + if err == nil { + r.store.Update(task.ID, func(t *Task) { + t.Status = TaskStatusCompleted + if t.Checkpoint == nil { + t.Checkpoint = map[string]any{} + } + t.Checkpoint["result"] = out + if t.Metadata == nil { + t.Metadata = map[string]any{} + } + t.Metadata["execFinishedAt"] = time.Now().UTC().Format(time.RFC3339Nano) + }) + r.progress(task, EventCompleted) + r.bump("completed") + return + } + + // The run itself was cancelled/stopped (not a per-task timeout): park the + // task as cancelled rather than consuming retry budget for an aborted run. + if ctx.Err() != nil { + if ok, _ := r.store.Cancel(task.ID, "run stopped"); ok { + r.bump("cancelled") + } + r.progress(task, EventCancelled) + return + } + + requeued, err := r.store.MarkFailed(task.ID, err.Error()) + if err != nil { + r.progress(task, EventFailed) + r.bump("failed") + return + } + if requeued { + r.progress(task, EventRetrying) + return + } + if r.tryReplan(ctx, task) { + r.progress(task, EventReplanned) + return + } + r.progress(task, EventFailed) + r.bump("failed") +} + +// tryReplan requeues a failed task with a fresh budget when the host's +// OnReplan hook approves and the per-task replan cap is not exhausted. +func (r *TaskRunner) tryReplan(ctx context.Context, task *Task) bool { + if r.opts.OnReplan == nil { + return false + } + r.mu.Lock() + n := r.replansByTask[task.ID] + r.mu.Unlock() + if n >= r.opts.MaxReplans { + return false + } + ok, err := r.opts.OnReplan(ctx, task) + if err != nil || !ok { + return false + } + if _, err := r.store.Requeue(task.ID); err != nil { + return false + } + r.mu.Lock() + r.replansByTask[task.ID]++ + r.replanned++ + r.mu.Unlock() + return true +} + +// dueTasks filters ready tasks to those whose retry backoff has elapsed. A task +// with no pending backoff tick is immediately due. +func (r *TaskRunner) dueTasks(tasks []*Task, now time.Time) []*Task { + var due []*Task + for _, t := range tasks { + tick := metaInt(t.Metadata, "retryBackoffTick") + if tick <= 0 { + due = append(due, t) + continue + } + if !now.Before(t.UpdatedAt.Add(r.backoff(tick))) { + due = append(due, t) + } + } + return due +} + +// quiescent reports whether no ready work exists and nothing is in flight. +// Tasks blocked behind failed/skipped/cancelled dependencies are not "active": +// they never become ready, so they must not keep the run alive forever. +func (r *TaskRunner) quiescent() bool { + if len(r.store.GetReadyWork()) > 0 { + return false + } + for _, t := range r.store.List() { + switch t.Status { + case TaskStatusInProgress, TaskStatusReviewing: + return false + } + } + return true +} + +// cancelActive parks all in-flight tasks as cancelled with the given reason. +func (r *TaskRunner) cancelActive(reason string) { + for _, t := range r.store.List() { + if t.Status != TaskStatusInProgress && t.Status != TaskStatusReviewing { + continue + } + if ok, _ := r.store.Cancel(t.ID, reason); ok { + r.bump("cancelled") + } + } +} + +func (r *TaskRunner) terminalCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.completed + r.failed + r.skipped + r.cancelled +} + +func (r *TaskRunner) bump(kind string) { + r.mu.Lock() + defer r.mu.Unlock() + switch kind { + case "executed": + r.executed++ + case "completed": + r.completed++ + case "failed": + r.failed++ + case "skipped": + r.skipped++ + case "cancelled": + r.cancelled++ + case "replanned": + r.replanned++ + } +} + +func (r *TaskRunner) progress(task *Task, event TaskRunnerEvent) { + if r.opts.OnProgress != nil { + r.opts.OnProgress(task, event) + } +} + +// Start launches Run in a background goroutine. Stop cancels it; Wait blocks +// until it finishes. +func (r *TaskRunner) Start(ctx context.Context) { + r.mu.Lock() + if r.started { + r.mu.Unlock() + return + } + r.started = true + r.finished = false + runCtx, cancel := context.WithCancel(ctx) + r.cancel = cancel + r.doneCh = make(chan struct{}) + r.mu.Unlock() + + go func() { + defer func() { + r.mu.Lock() + r.finished = true + r.mu.Unlock() + close(r.doneCh) + }() + _ = r.Run(runCtx) + }() +} + +// Stop cancels a background run. Idempotent; safe when never started. +func (r *TaskRunner) Stop() { + r.mu.Lock() + c := r.cancel + r.mu.Unlock() + if c != nil { + c() + } +} + +// Wait blocks until the background run finishes or ctx is done. +func (r *TaskRunner) Wait(ctx context.Context) error { + r.mu.Lock() + done := r.doneCh + r.mu.Unlock() + if done == nil { + return fmt.Errorf("task runner: not started") + } + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Status returns a point-in-time snapshot of runner activity. +func (r *TaskRunner) Status() TaskRunnerStats { + r.mu.Lock() + defer r.mu.Unlock() + return TaskRunnerStats{ + Running: r.started && !r.finished, + Executed: r.executed, + Completed: r.completed, + Failed: r.failed, + Skipped: r.skipped, + Cancelled: r.cancelled, + Replanned: r.replanned, + } +} + +// metaInt reads an integer metadata value that may have been round-tripped +// through JSON (float64) or set directly in memory (int). +func metaInt(m map[string]any, key string) int { + if m == nil { + return 0 + } + switch v := m[key].(type) { + case int: + return v + case int64: + return int(v) + case float64: + return int(v) + default: + return 0 + } +} diff --git a/internal/tool/task_executor_test.go b/internal/tool/task_executor_test.go new file mode 100644 index 00000000..0539e0f9 --- /dev/null +++ b/internal/tool/task_executor_test.go @@ -0,0 +1,402 @@ +package tool + +import ( + "context" + "fmt" + "strings" + "sync" + "testing" + "time" +) + +// fakeTaskExecutor records the tasks it runs and fails those on the fail list. +type fakeTaskExecutor struct { + mu sync.Mutex + ran []string + fail map[string]error + delay time.Duration + output map[string]string +} + +func newFakeExecutor(fail map[string]error) *fakeTaskExecutor { + return &fakeTaskExecutor{ + fail: fail, + output: map[string]string{}, + } +} + +func (f *fakeTaskExecutor) execute(ctx context.Context, t *Task) (string, error) { + f.mu.Lock() + f.ran = append(f.ran, t.ID) + err := f.fail[t.ID] + out := f.output[t.ID] + f.mu.Unlock() + if ctx != nil { + select { + case <-ctx.Done(): + return "", ctx.Err() + default: + } + } + if f.delay > 0 { + select { + case <-time.After(f.delay): + case <-ctx.Done(): + return "", ctx.Err() + } + } + if err != nil { + return "", err + } + if out == "" { + out = "done:" + t.ID + } + return out, nil +} + +func newStoreWith(tasks ...func(*Task)) *TaskStore { + s := &TaskStore{tasks: make(map[string]*Task)} + for _, setup := range tasks { + t := s.Create("subject", "description", "", nil) + setup(t) + } + return s +} + +func TestTaskRunnerCompletesReadyWork(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + a := s.Create("a", "do a", "", nil) + b := s.Create("b", "do b", "", nil) + + exec := newFakeExecutor(nil) + r := NewTaskRunner(TaskRunnerOptions{ + Store: s, + Execute: exec.execute, + PollInterval: time.Millisecond, + }) + if err := r.Run(context.Background()); err != nil { + t.Fatal(err) + } + + got, _ := s.Get(a.ID) + if got.Status != TaskStatusCompleted { + t.Fatalf("task a status = %q, want completed", got.Status) + } + gotB, _ := s.Get(b.ID) + if gotB.Status != TaskStatusCompleted { + t.Fatalf("task b status = %q, want completed", gotB.Status) + } + st := r.Status() + if st.Completed != 2 || st.Executed != 2 { + t.Fatalf("unexpected stats: %+v", st) + } + if got.Checkpoint["result"] != "done:task_1" { + t.Fatalf("expected result checkpoint, got %+v", got.Checkpoint) + } +} + +func TestTaskRunnerRespectsDependencies(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + a := s.Create("a", "do a", "", nil) + b := s.Create("b", "do b", "", nil) + s.Update(b.ID, func(t *Task) { + t.Dependencies = []TaskDependency{{TargetID: a.ID, Type: "blocks"}} + }) + + exec := newFakeExecutor(nil) + r := NewTaskRunner(TaskRunnerOptions{ + Store: s, + Execute: exec.execute, + PollInterval: time.Millisecond, + }) + if err := r.Run(context.Background()); err != nil { + t.Fatal(err) + } + + exec.mu.Lock() + defer exec.mu.Unlock() + if len(exec.ran) != 2 { + t.Fatalf("expected 2 executions, got %v", exec.ran) + } + if exec.ran[0] != a.ID || exec.ran[1] != b.ID { + t.Fatalf("dependency order violated: %v", exec.ran) + } +} + +func TestTaskRunnerRetriesWithinBudget(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + task := s.Create("flaky", "do flaky", "", nil) + s.Update(task.ID, func(t *Task) { t.MaxAttempts = 3 }) + + // Fail the first two attempts, succeed on the third. + attempts := map[string]int{} + var mu sync.Mutex + exec := func(_ context.Context, t *Task) (string, error) { + mu.Lock() + attempts[t.ID]++ + n := attempts[t.ID] + mu.Unlock() + if n < 3 { + return "", fmt.Errorf("transient failure %d", n) + } + return "ok", nil + } + + r := NewTaskRunner(TaskRunnerOptions{ + Store: s, + Execute: exec, + Backoff: func(int) time.Duration { return time.Millisecond }, + PollInterval: time.Millisecond, + }) + if err := r.Run(context.Background()); err != nil { + t.Fatal(err) + } + + got, _ := s.Get(task.ID) + if got.Status != TaskStatusCompleted { + t.Fatalf("expected completed after retries, got %q (lastError=%q)", got.Status, got.LastError) + } + if got.Attempts != 2 { + t.Fatalf("expected 2 failed attempts recorded, got %d", got.Attempts) + } +} + +func TestTaskRunnerParksFailedAfterBudget(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + task := s.Create("doomed", "do doomed", "", nil) + s.Update(task.ID, func(t *Task) { t.MaxAttempts = 2 }) + + exec := func(_ context.Context, t *Task) (string, error) { + return "", fmt.Errorf("always fails") + } + + r := NewTaskRunner(TaskRunnerOptions{ + Store: s, + Execute: exec, + Backoff: func(int) time.Duration { return time.Millisecond }, + PollInterval: time.Millisecond, + }) + if err := r.Run(context.Background()); err != nil { + t.Fatal(err) + } + + got, _ := s.Get(task.ID) + if got.Status != TaskStatusFailed { + t.Fatalf("expected failed after budget, got %q", got.Status) + } + if got.Attempts != 2 { + t.Fatalf("expected 2 attempts, got %d", got.Attempts) + } + st := r.Status() + if st.Failed != 1 { + t.Fatalf("expected 1 failed in stats, got %+v", st) + } +} + +func TestTaskRunnerReplansFailedTask(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + task := s.Create("replan", "do replan", "", nil) + s.Update(task.ID, func(t *Task) { t.MaxAttempts = 1 }) + + // Fail the first round (budget 1), then replan requeues it; succeed after. + round := 0 + exec := func(_ context.Context, t *Task) (string, error) { + if round == 0 { + round = 1 + return "", fmt.Errorf("first round fails") + } + return "ok", nil + } + replans := 0 + r := NewTaskRunner(TaskRunnerOptions{ + Store: s, + Execute: exec, + Backoff: func(int) time.Duration { return time.Millisecond }, + PollInterval: time.Millisecond, + OnReplan: func(_ context.Context, t *Task) (bool, error) { + replans++ + return true, nil + }, + }) + if err := r.Run(context.Background()); err != nil { + t.Fatal(err) + } + + got, _ := s.Get(task.ID) + if got.Status != TaskStatusCompleted { + t.Fatalf("expected completed after replan, got %q", got.Status) + } + if replans != 1 { + t.Fatalf("expected 1 replan, got %d", replans) + } + st := r.Status() + if st.Replanned != 1 { + t.Fatalf("expected 1 replanned in stats, got %+v", st) + } +} + +func TestTaskRunnerWatchdogStops(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + for i := 0; i < 10; i++ { + s.Create(fmt.Sprintf("t%d", i), "desc", "", nil) + } + + exec := newFakeExecutor(nil) + r := NewTaskRunner(TaskRunnerOptions{ + Store: s, + Execute: exec.execute, + PollInterval: time.Millisecond, + MaxTotalTasks: 3, + }) + if err := r.Run(context.Background()); err != nil { + t.Fatal(err) + } + + exec.mu.Lock() + n := len(exec.ran) + exec.mu.Unlock() + if n != 3 { + t.Fatalf("watchdog should stop after 3 tasks, ran %d", n) + } + st := r.Status() + if st.Completed != 3 { + t.Fatalf("expected 3 completed, got %+v", st) + } + // Remaining tasks stay pending. + remaining := 0 + for _, t := range s.List() { + if t.Status == TaskStatusPending { + remaining++ + } + } + if remaining != 7 { + t.Fatalf("expected 7 pending remaining, got %d", remaining) + } +} + +func TestTaskRunnerCancelsInFlightOnStop(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + task := s.Create("slow", "slow task", "", nil) + + started := make(chan struct{}) + exec := func(ctx context.Context, t *Task) (string, error) { + close(started) + <-ctx.Done() + return "", ctx.Err() + } + + r := NewTaskRunner(TaskRunnerOptions{ + Store: s, + Execute: exec, + PollInterval: time.Millisecond, + DefaultTimeout: 5 * time.Second, + }) + ctx := context.Background() + r.Start(ctx) + <-started + r.Stop() + if err := r.Wait(context.Background()); err != nil { + t.Fatal(err) + } + + got, _ := s.Get(task.ID) + if got.Status != TaskStatusCancelled { + t.Fatalf("expected cancelled on stop, got %q", got.Status) + } +} + +func TestTaskRunnerQuiescesBehindFailedBlocker(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + a := s.Create("a", "do a", "", nil) + b := s.Create("b", "depends on a", "", nil) + s.Update(b.ID, func(t *Task) { t.Dependencies = []TaskDependency{{TargetID: a.ID, Type: "blocks"}} }) + s.Update(a.ID, func(t *Task) { t.MaxAttempts = 1 }) + // Mark a failed so b can never become ready. + s.MarkFailed(a.ID, "cannot do a") + + exec := newFakeExecutor(nil) + r := NewTaskRunner(TaskRunnerOptions{ + Store: s, + Execute: exec.execute, + PollInterval: time.Millisecond, + }) + // Must terminate rather than spin forever behind the failed blocker. + done := make(chan error, 1) + go func() { done <- r.Run(context.Background()) }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("runner spun forever behind a failed blocker") + } +} + +func TestTaskRunnerProgressEvents(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + s.Create("a", "do a", "", nil) + + var events []string + var mu sync.Mutex + exec := newFakeExecutor(nil) + r := NewTaskRunner(TaskRunnerOptions{ + Store: s, + Execute: exec.execute, + PollInterval: time.Millisecond, + OnProgress: func(t *Task, e TaskRunnerEvent) { + mu.Lock() + events = append(events, string(e)) + mu.Unlock() + }, + }) + if err := r.Run(context.Background()); err != nil { + t.Fatal(err) + } + + mu.Lock() + defer mu.Unlock() + joined := strings.Join(events, ",") + if !strings.Contains(joined, string(EventStarted)) || !strings.Contains(joined, string(EventCompleted)) { + t.Fatalf("expected started+completed events, got %q", joined) + } +} + +func TestSkipAndCancel(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + task := s.Create("a", "do a", "", nil) + + if ok, _ := s.Skip(task.ID, "not needed"); !ok { + t.Fatal("skip should succeed") + } + got, _ := s.Get(task.ID) + if got.Status != TaskStatusSkipped || got.Metadata["skipReason"] != "not needed" { + t.Fatalf("unexpected after skip: %+v", got) + } + // Skipping a completed task is a no-op. + s.Update(task.ID, func(t *Task) { t.Status = TaskStatusPending }) + if ok, _ := s.Skip(task.ID, "x"); !ok { + t.Fatal("re-skip of pending should succeed") + } + s.Update(task.ID, func(t *Task) { t.Status = TaskStatusCompleted }) + if ok, _ := s.Cancel(task.ID, "too late"); ok { + t.Fatal("cancelling a completed task should be a no-op") + } + + c := s.Create("c", "do c", "", nil) + if ok, _ := s.Cancel(c.ID, "aborted"); !ok { + t.Fatal("cancel should succeed") + } + gotC, _ := s.Get(c.ID) + if gotC.Status != TaskStatusCancelled || gotC.Metadata["cancelReason"] != "aborted" { + t.Fatalf("unexpected after cancel: %+v", gotC) + } +} + +func TestTaskRunnerMissingExecute(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + r := NewTaskRunner(TaskRunnerOptions{Store: s}) + if err := r.Run(context.Background()); err == nil { + t.Fatal("expected error when execute is missing") + } +} diff --git a/internal/tool/task_run_tool.go b/internal/tool/task_run_tool.go new file mode 100644 index 00000000..ed5ccf58 --- /dev/null +++ b/internal/tool/task_run_tool.go @@ -0,0 +1,83 @@ +package tool + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +// TaskRunTool drives ready tasks through the task executor. It lets the agent +// (or a host) turn the validated task graph into execution: tasks run in +// dependency order, are retried up to their retry budget with backoff, and are +// parked failed when the budget is exhausted. It is the tool-level front door +// for TaskRunner. +type TaskRunTool struct{} + +func (TaskRunTool) Name() string { return "TaskRun" } +func (TaskRunTool) Aliases() []string { return []string{"task_run"} } +func (TaskRunTool) Description() string { + return "Execute all ready tasks (pending with no blockers) through the task executor. " + + "Tasks run in dependency order, are retried up to their retry budget, and are parked failed " + + "when the budget is exhausted. Returns a run summary." +} + +func (TaskRunTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "timeout_sec": map[string]interface{}{ + "type": "integer", + "description": "Per-task execution timeout in seconds (default 300)", + }, + "max_total_tasks": map[string]interface{}{ + "type": "integer", + "description": "Watchdog cap on distinct tasks that may reach a terminal state (default 50)", + }, + }, + } +} + +func (TaskRunTool) Execute(ctx context.Context, input json.RawMessage) (string, error) { + tc := GetToolContext(ctx) + if tc == nil || tc.TaskExecutor == nil { + return "", fmt.Errorf("TaskRun requires a task executor; none is configured for this session") + } + + var p struct { + TimeoutSec int `json:"timeout_sec"` + MaxTotalTasks int `json:"max_total_tasks"` + } + if input != nil { + _ = json.Unmarshal(input, &p) + } + timeout := time.Duration(p.TimeoutSec) * time.Second + if timeout <= 0 { + timeout = 300 * time.Second + } + + runner := NewTaskRunner(TaskRunnerOptions{ + Store: GetTaskStore(), + Execute: tc.TaskExecutor, + DefaultTimeout: timeout, + MaxTotalTasks: p.MaxTotalTasks, + }) + if err := runner.Run(ctx); err != nil { + return "", err + } + s := runner.Status() + parts := []string{ + fmt.Sprintf("%d executed", s.Executed), + fmt.Sprintf("%d completed", s.Completed), + fmt.Sprintf("%d failed", s.Failed), + fmt.Sprintf("%d replanned", s.Replanned), + } + if s.Skipped > 0 { + parts = append(parts, fmt.Sprintf("%d skipped", s.Skipped)) + } + if s.Cancelled > 0 { + parts = append(parts, fmt.Sprintf("%d cancelled", s.Cancelled)) + } + return "Task run finished: " + strings.Join(parts, ", "), nil +} diff --git a/internal/tool/task_run_tool_test.go b/internal/tool/task_run_tool_test.go new file mode 100644 index 00000000..ff4a68ac --- /dev/null +++ b/internal/tool/task_run_tool_test.go @@ -0,0 +1,69 @@ +package tool + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestTaskRunToolExecutesReadyTasks(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + s.Create("a", "do a", "", nil) + s.Create("b", "do b", "", nil) + // Replace the global store reference used by the tool for the test. + prev := globalTaskStore + globalTaskStore = s + defer func() { globalTaskStore = prev }() + + exec := newFakeExecutor(nil) + ctx := WithToolContext(context.Background(), &ToolContext{TaskExecutor: exec.execute}) + out, err := TaskRunTool{}.Execute(ctx, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(out, "2 completed") { + t.Fatalf("expected 2 completed in summary, got: %s", out) + } + got, _ := s.Get("task_1") + if got.Status != TaskStatusCompleted { + t.Fatalf("task_1 status = %q, want completed", got.Status) + } +} + +func TestTaskRunToolRequiresExecutor(t *testing.T) { + ctx := WithToolContext(context.Background(), &ToolContext{}) + _, err := TaskRunTool{}.Execute(ctx, nil) + if err == nil { + t.Fatal("expected error when no executor configured") + } + if !strings.Contains(err.Error(), "task executor") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestTaskRunToolCancelsOnContextTimeout(t *testing.T) { + s := &TaskStore{tasks: make(map[string]*Task)} + s.Create("slow", "slow task", "", nil) + prev := globalTaskStore + globalTaskStore = s + defer func() { globalTaskStore = prev }() + + exec := func(ctx context.Context, t *Task) (string, error) { + <-ctx.Done() + return "", ctx.Err() + } + + runCtx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + runCtx = WithToolContext(runCtx, &ToolContext{TaskExecutor: exec}) + _, err := TaskRunTool{}.Execute(runCtx, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got, _ := s.Get("task_1") + if got.Status != TaskStatusCancelled { + t.Fatalf("expected task cancelled on ctx timeout, got %q", got.Status) + } +} diff --git a/internal/tool/task_runner.go b/internal/tool/task_runner.go index 8cd5800f..59c6adae 100644 --- a/internal/tool/task_runner.go +++ b/internal/tool/task_runner.go @@ -213,6 +213,64 @@ func (s *TaskStore) Requeue(id string) (bool, error) { return true, nil } +// Skip parks a task as skipped without executing it. The reason is recorded in +// metadata for later inspection (skipReason). +func (s *TaskStore) Skip(id, reason string) (bool, error) { + s.mu.Lock() + t, ok := s.tasks[id] + if !ok { + s.mu.Unlock() + return false, fmt.Errorf("task %q not found", id) + } + if t.Status == TaskStatusCompleted || t.Status == TaskStatusSkipped { + s.mu.Unlock() + return false, nil + } + t.Status = TaskStatusSkipped + if t.Metadata == nil { + t.Metadata = make(map[string]any) + } + if reason != "" { + t.Metadata["skipReason"] = reason + } + t.UpdatedAt = time.Now() + persist := s.persist + s.mu.Unlock() + if persist != nil { + _ = s.Save("") + } + return true, nil +} + +// Cancel parks a task as cancelled (e.g. the run was aborted). The reason is +// recorded in metadata for later inspection (cancelReason). +func (s *TaskStore) Cancel(id, reason string) (bool, error) { + s.mu.Lock() + t, ok := s.tasks[id] + if !ok { + s.mu.Unlock() + return false, fmt.Errorf("task %q not found", id) + } + if t.Status == TaskStatusCompleted || t.Status == TaskStatusCancelled { + s.mu.Unlock() + return false, nil + } + t.Status = TaskStatusCancelled + if t.Metadata == nil { + t.Metadata = make(map[string]any) + } + if reason != "" { + t.Metadata["cancelReason"] = reason + } + t.UpdatedAt = time.Now() + persist := s.persist + s.mu.Unlock() + if persist != nil { + _ = s.Save("") + } + return true, nil +} + // Checkpoint merges resumable progress onto a task without changing its // status. A replan or resume reads the checkpoint to avoid starting from zero. func (s *TaskStore) Checkpoint(id string, data map[string]any) (bool, error) { diff --git a/internal/tool/task_schedule.go b/internal/tool/task_schedule.go index 826936cd..055141b5 100644 --- a/internal/tool/task_schedule.go +++ b/internal/tool/task_schedule.go @@ -41,7 +41,8 @@ func (s *TaskStore) Schedule() (TaskSchedule, error) { return TaskSchedule{}, fmt.Errorf("task schedule contains invalid task identity %q", id) } switch task.Status { - case TaskStatusPending, TaskStatusInProgress, TaskStatusCompleted, TaskStatusFailed: + case TaskStatusPending, TaskStatusInProgress, TaskStatusReviewing, + TaskStatusCompleted, TaskStatusFailed, TaskStatusSkipped, TaskStatusCancelled: default: return TaskSchedule{}, fmt.Errorf("task %q has invalid status %q", id, task.Status) } diff --git a/internal/tool/tool.go b/internal/tool/tool.go index 54696db9..ceff3e81 100644 --- a/internal/tool/tool.go +++ b/internal/tool/tool.go @@ -97,6 +97,9 @@ type ToolContext struct { // WorkingDir, when set, is used as cmd.Dir for Bash and as the preferred // workspace root for path tools (subagent worktree isolation). WorkingDir string + // TaskExecutor, when set, arms the TaskRun tool: it runs one task from the + // store (e.g. by spawning a sub-agent). Nil disables TaskRun. + TaskExecutor TaskExecutorFunc // Lint configures the optional post-write auto-lint cycle. The zero value // (Enabled=false) keeps linting off so users are not surprised. Lint lint.Config diff --git a/packaging/systemd/hawk-daemon.service b/packaging/systemd/hawk-daemon.service new file mode 100644 index 00000000..b5642435 --- /dev/null +++ b/packaging/systemd/hawk-daemon.service @@ -0,0 +1,42 @@ +[Unit] +Description=Hawk Daemon (code intelligence server) +Documentation=https://docs.hawkcode.ai +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +# Replace with your installed binary path. When installed via the package +# manager, the binary is typically at /usr/local/bin/hawk. +ExecStart=/usr/local/bin/hawk daemon start \ + --host 127.0.0.1 \ + --port 4590 \ + --api-key ${HAWK_DAEMON_API_KEY} \ + --log-level INFO +# For production, bind to 0.0.0.0 behind a TLS-terminating reverse proxy: +# ExecStart=/usr/local/bin/hawk daemon start --host 0.0.0.0 --port 4590 --api-key ${HAWK_DAEMON_API_KEY} --tls-cert /etc/hawk/tls.crt --tls-key /etc/hawk/tls.key +Restart=always +RestartSec=5 +TimeoutStopSec=10 + +# Security hardening (mirror the sandbox ethos at the process level). +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=%h/.hawk/state /tmp +# Read-only access to config and tool directories. +ReadOnlyPaths=%h/.hawk + +# Resource limits. +LimitNOFILE=65536 +MemoryMax=4G +CPUQuota=200% + +# Environment. +Environment=HAWK_DAEMON_API_KEY= +Environment=HAWK_CODE_ENABLE_TELEMETRY=0 +Environment=HAWK_DAEMON_MAX_CONCURRENT=4 + +[Install] +WantedBy=multi-user.target