From 93951f4e490268182ac1bccd25e05311e48ffd7b Mon Sep 17 00:00:00 2001 From: binrogithub Date: Sun, 8 Feb 2026 18:00:50 +0300 Subject: [PATCH] Add smoke validation script and docs --- documentation/smoke_validation.md | 111 ++++++++++++++++++++++ scripts/smoke_validation.sh | 147 ++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 documentation/smoke_validation.md create mode 100755 scripts/smoke_validation.sh diff --git a/documentation/smoke_validation.md b/documentation/smoke_validation.md new file mode 100644 index 0000000000..1acb928ac2 --- /dev/null +++ b/documentation/smoke_validation.md @@ -0,0 +1,111 @@ +# Smoke validation script + +This repository includes a minimal smoke validation script at +`scripts/smoke_validation.sh`. It avoids heavy test frameworks and is intended +for quick end-to-end checks. + +## Prerequisites + +* `rg` and `bash` available on the PATH. +* The CLI commands you want to exercise are available, or you can use the + built-in mock mode for a no-network, no-service run. + +## Commands and expected outputs + +### 1) Mocked run (no external services) + +This mode generates a local `report.md` and uses mock output for the preview +step. It is helpful for confirming the script wiring and output matching. + +```bash +SMOKE_USE_MOCK=1 \ +MCP_LIST_TOOLS_CMD='printf "tools: [hc_tools_search]\n"' \ +HC_TOOLS_SEARCH_CMD='printf "result: []\n"' \ +SMOKE_POLICY_BLOCK_CMD='printf "policy blocked for demo\n"; exit 1' \ +SMOKE_OPS_APPLY_CMD='printf "apply succeeded for ops\n"' \ +./scripts/smoke_validation.sh +``` + +**Expected output (abridged):** + +``` +==> Preview (expect run_id + report.md) +run_id: mock-123 +report.md: /tmp/.../report.md + +==> MCP list_tools +tools: [hc_tools_search] + +==> hc_tools_search +result: [] + +==> Policy block (demo tenant) +policy blocked for demo + +==> Apply allowed (ops tenant with approvals) +apply succeeded for ops + +==> Smoke validation complete +``` + +### 2) Real CLI run (replace commands as needed) + +Point the script at your real CLI commands. The example below assumes: + +* `hc preview` writes a report to the provided path and prints `run_id:`. +* `mcp list_tools` emits `hc_tools_search` in its output. +* `hc_tools_search` emits a `result` key in its output. +* `hc apply` fails with a policy-block message for the demo tenant. +* `hc apply` succeeds for the ops tenant when approvals are provided. + +```bash +export SMOKE_REPORT_PATH=/tmp/hc-smoke/report.md +export SMOKE_APPROVALS=/tmp/hc-smoke/approvals.json + +SMOKE_PREVIEW_CMD='hc preview --tenant demo --output "$SMOKE_REPORT_PATH"' \ +MCP_LIST_TOOLS_CMD='mcp list_tools' \ +HC_TOOLS_SEARCH_CMD='hc_tools_search' \ +SMOKE_POLICY_BLOCK_CMD='hc apply --tenant demo --plan "$SMOKE_REPORT_PATH"' \ +SMOKE_OPS_APPLY_CMD='hc apply --tenant ops --plan "$SMOKE_REPORT_PATH" --approvals "$SMOKE_APPROVALS"' \ +./scripts/smoke_validation.sh +``` + +**Expected output (abridged):** + +``` +==> Preview (expect run_id + report.md) +run_id: +report.md: /tmp/hc-smoke/report.md + +==> MCP list_tools +... hc_tools_search ... + +==> hc_tools_search +result: ... + +==> Policy block (demo tenant) +... policy blocked ... + +==> Apply allowed (ops tenant with approvals) +... apply succeeded ... + +==> Smoke validation complete +``` + +## Output matching customization + +If your CLI outputs differ, you can adjust expected patterns via environment +variables: + +* `SMOKE_MCP_TOOLS_PATTERN` (default: `hc_tools_search`) +* `SMOKE_TOOLS_SEARCH_PATTERN` (default: `result`) +* `SMOKE_POLICY_BLOCK_PATTERN` (default: `policy.*(blocked|denied)`) +* `SMOKE_APPLY_ALLOWED_PATTERN` (default: `apply.*(succeeded|allowed|ok)`) + +Example: + +```bash +SMOKE_POLICY_BLOCK_PATTERN='blocked by policy' \ +SMOKE_APPLY_ALLOWED_PATTERN='status: ok' \ +./scripts/smoke_validation.sh +``` diff --git a/scripts/smoke_validation.sh b/scripts/smoke_validation.sh new file mode 100755 index 0000000000..a9c04330cb --- /dev/null +++ b/scripts/smoke_validation.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +WORK_DIR=${SMOKE_WORK_DIR:-$(mktemp -d)} +REPORT_PATH=${SMOKE_REPORT_PATH:-"$WORK_DIR/report.md"} + +MCP_LIST_TOOLS_CMD=${MCP_LIST_TOOLS_CMD:-"mcp list_tools"} +HC_TOOLS_SEARCH_CMD=${HC_TOOLS_SEARCH_CMD:-"hc_tools_search"} +PREVIEW_CMD=${SMOKE_PREVIEW_CMD:-"hc preview --tenant demo --output $REPORT_PATH"} +POLICY_BLOCK_CMD=${SMOKE_POLICY_BLOCK_CMD:-"hc apply --tenant demo --plan $REPORT_PATH"} +OPS_APPLY_CMD=${SMOKE_OPS_APPLY_CMD:-"hc apply --tenant ops --plan $REPORT_PATH --approvals $SMOKE_APPROVALS"} + +MCP_TOOLS_PATTERN=${SMOKE_MCP_TOOLS_PATTERN:-"hc_tools_search"} +TOOLS_SEARCH_PATTERN=${SMOKE_TOOLS_SEARCH_PATTERN:-"result"} +POLICY_BLOCK_PATTERN=${SMOKE_POLICY_BLOCK_PATTERN:-"policy.*(blocked|denied)"} +APPLY_ALLOWED_PATTERN=${SMOKE_APPLY_ALLOWED_PATTERN:-"apply.*(succeeded|allowed|ok)"} + +log() { + printf '\n==> %s\n' "$1" +} + +run_cmd() { + local cmd=$1 + bash -c "$cmd" +} + +capture_cmd() { + local cmd=$1 + local output + output=$(bash -c "$cmd" 2>&1) + printf '%s' "$output" +} + +assert_contains() { + local label=$1 + local output=$2 + local pattern=$3 + + if ! printf '%s\n' "$output" | rg -q --pcre2 "$pattern"; then + printf '\n[ERROR] %s: expected output to match /%s/\n' "$label" "$pattern" >&2 + return 1 + fi +} + +assert_file() { + local path=$1 + if [[ ! -f "$path" ]]; then + printf '\n[ERROR] expected file not found: %s\n' "$path" >&2 + return 1 + fi +} + +parse_run_id() { + local output=$1 + printf '%s\n' "$output" | sed -n 's/.*run_id[:=][[:space:]]*\([A-Za-z0-9._-]\+\).*/\1/p' | head -n1 +} + +run_preview() { + log "Preview (expect run_id + report.md)" + + if [[ ${SMOKE_USE_MOCK:-0} -eq 1 ]]; then + mkdir -p "$(dirname "$REPORT_PATH")" + cat <<'MOCK' > "$REPORT_PATH" +# Mock report + +- status: ok +MOCK + printf 'run_id: mock-123\nreport.md: %s\n' "$REPORT_PATH" + return 0 + fi + + local output + output=$(capture_cmd "$PREVIEW_CMD") + printf '%s\n' "$output" + + local run_id + run_id=$(parse_run_id "$output") + if [[ -z "$run_id" ]]; then + printf '\n[ERROR] missing run_id in preview output\n' >&2 + return 1 + fi + + assert_file "$REPORT_PATH" +} + +run_mcp_list_tools() { + log "MCP list_tools" + + local output + output=$(capture_cmd "$MCP_LIST_TOOLS_CMD") + printf '%s\n' "$output" + assert_contains "list_tools" "$output" "$MCP_TOOLS_PATTERN" +} + +run_hc_tools_search() { + log "hc_tools_search" + + local output + output=$(capture_cmd "$HC_TOOLS_SEARCH_CMD") + printf '%s\n' "$output" + assert_contains "hc_tools_search" "$output" "$TOOLS_SEARCH_PATTERN" +} + +run_policy_block_demo() { + log "Policy block (demo tenant)" + + local output + set +e + output=$(capture_cmd "$POLICY_BLOCK_CMD") + local status=$? + set -e + + printf '%s\n' "$output" + + if [[ $status -eq 0 ]]; then + printf '\n[ERROR] expected demo tenant apply to fail\n' >&2 + return 1 + fi + + assert_contains "policy block" "$output" "$POLICY_BLOCK_PATTERN" +} + +run_ops_apply() { + log "Apply allowed (ops tenant with approvals)" + + local output + output=$(capture_cmd "$OPS_APPLY_CMD") + printf '%s\n' "$output" + + assert_contains "ops apply" "$output" "$APPLY_ALLOWED_PATTERN" +} + +main() { + log "Working directory: $WORK_DIR" + log "Report path: $REPORT_PATH" + + run_preview + run_mcp_list_tools + run_hc_tools_search + run_policy_block_demo + run_ops_apply + + log "Smoke validation complete" +} + +main "$@"