diff --git a/.github/workflows/sdk-proto-check.yml b/.github/workflows/sdk-proto-check.yml new file mode 100644 index 0000000000..019c8bb7ae --- /dev/null +++ b/.github/workflows/sdk-proto-check.yml @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: SDK Proto Check + +on: + merge_group: + types: [checks_requested] + push: + branches: + - "pull-request/[0-9]+" + workflow_dispatch: + +env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +permissions: + contents: read + packages: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pr_metadata: + name: Resolve PR metadata + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.gate.outputs.should_run }} + matrix: ${{ steps.config.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - id: gate + uses: ./.github/actions/pr-gate + + - id: config + name: Load SDK configuration + run: echo "matrix=$(jq -c '.include' tasks/sdk-sync-config.json)" >> "$GITHUB_OUTPUT" + + sdk_proto_drift: + name: Proto Drift (${{ matrix.sdk.name }}) + needs: pr_metadata + if: needs.pr_metadata.outputs.should_run == 'true' + runs-on: linux-amd64-cpu8 + timeout-minutes: 15 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + sdk: ${{ fromJSON(needs.pr_metadata.outputs.matrix) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install tools + run: mise install --locked + + - name: Check proto drift + id: drift + run: | + REPORT=$(mise run ${{ matrix.sdk.drift_task }} 2>"$RUNNER_TEMP/drift_stderr.log") || true + + if echo "$REPORT" | jq -e 'has("synced") and (.synced | type == "boolean")' >/dev/null 2>&1; then + SYNCED=$(echo "$REPORT" | jq -r '.synced') + DELIMITER="REPORT_EOF_$(openssl rand -hex 16)" + { + echo "report<<$DELIMITER" + echo "$REPORT" + echo "$DELIMITER" + } >> "$GITHUB_OUTPUT" + echo "synced=$SYNCED" >> "$GITHUB_OUTPUT" + else + echo "::warning::Proto drift check failed: unable to parse report" + echo "stderr: $(cat "$RUNNER_TEMP/drift_stderr.log")" + echo "synced=error" >> "$GITHUB_OUTPUT" + fi + + - name: Annotate drift warning + if: steps.drift.outputs.synced == 'false' + env: + DRIFT_REPORT: ${{ steps.drift.outputs.report }} + SDK_NAME: ${{ matrix.sdk.name }} + run: | + SUMMARY=$(echo "$DRIFT_REPORT" | jq -r '.summary') + FILES=$(echo "$DRIFT_REPORT" | jq -r '.files[] | select(.status != "synced") | " - \(.name) (\(.status), \(.diff_lines) lines changed)"' | sed ':a;N;$!ba;s/\n/%0A/g') + echo "::warning::SDK proto drift detected for ${SDK_NAME}: ${SUMMARY}%0A${FILES}" diff --git a/.github/workflows/sdk-sync-dashboard.yml b/.github/workflows/sdk-sync-dashboard.yml new file mode 100644 index 0000000000..962d882344 --- /dev/null +++ b/.github/workflows/sdk-sync-dashboard.yml @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: SDK Proto Sync + +on: + schedule: + - cron: "0 6 * * *" + workflow_dispatch: + +env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +permissions: + actions: read + contents: read + packages: read + issues: write + +concurrency: + group: sdk-proto-sync + cancel-in-progress: true + +jobs: + load_config: + name: Load SDK configuration + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + matrix: ${{ steps.config.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - id: config + run: echo "matrix=$(jq -c '.include' tasks/sdk-sync-config.json)" >> "$GITHUB_OUTPUT" + + sdk_sync_check: + name: Sync Check (${{ matrix.sdk.name }}) + needs: load_config + runs-on: linux-amd64-cpu8 + timeout-minutes: 30 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + sdk: ${{ fromJSON(needs.load_config.outputs.matrix) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install tools + run: mise install --locked + - name: Check proto drift and build + env: + SDK_NAME: ${{ matrix.sdk.name }} + DRIFT_TASK: ${{ matrix.sdk.drift_task }} + BUILD_CHECK_TASK: ${{ matrix.sdk.build_check_task }} + run: | + mkdir -p report + mise run "$DRIFT_TASK" > report/drift.json 2> report/drift.stderr || true + if ! jq -e 'has("synced") and (.synced | type == "boolean")' report/drift.json >/dev/null 2>&1; then + jq -n --arg sdk "$SDK_NAME" '{sdk:$sdk, has_drift:"error", build_failed:"false"}' > report/status.json + exit 0 + fi + SYNCED=$(jq -r '.synced' report/drift.json) + if [ "$SYNCED" = "true" ]; then + jq -n --arg sdk "$SDK_NAME" '{sdk:$sdk, has_drift:"false", build_failed:"false"}' > report/status.json + exit 0 + fi + if mise run "$BUILD_CHECK_TASK" > report/build.json 2> report/build.stderr; then + BUILD_FAILED=false + else + BUILD_FAILED=true + fi + jq -n --arg sdk "$SDK_NAME" --arg build_failed "$BUILD_FAILED" \ + '{sdk:$sdk, has_drift:"true", build_failed:$build_failed}' > report/status.json + - name: Upload SDK report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sdk-sync-${{ matrix.sdk.name }} + path: report/ + if-no-files-found: error + retention-days: 1 + + issue_management: + name: Manage Drift Issue (${{ matrix.sdk.name }}) + needs: [load_config, sdk_sync_check] + if: always() && needs.load_config.result == 'success' && needs.sdk_sync_check.result == 'success' + runs-on: linux-amd64-cpu8 + timeout-minutes: 5 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + sdk: ${{ fromJSON(needs.load_config.outputs.matrix) }} + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: sdk-sync-${{ matrix.sdk.name }} + path: report + - name: Read SDK status + id: status + run: | + echo "has_drift=$(jq -r '.has_drift' report/status.json)" >> "$GITHUB_OUTPUT" + echo "build_failed=$(jq -r '.build_failed' report/status.json)" >> "$GITHUB_OUTPUT" + - name: Warn on drift detection error + if: steps.status.outputs.has_drift == 'error' + run: | + echo "::error::Drift detection failed for ${{ matrix.sdk.name }} SDK" + cat report/drift.stderr + exit 1 + - if: steps.status.outputs.has_drift == 'true' && steps.status.outputs.build_failed == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install tools + if: steps.status.outputs.has_drift == 'true' && steps.status.outputs.build_failed == 'true' + run: mise install --locked + - name: Create or update drift issue + if: steps.status.outputs.has_drift == 'true' && steps.status.outputs.build_failed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + RESULT=$(uv run python tasks/scripts/sdk_sync.py manage-issue \ + --drift-report "$(cat report/drift.json)" \ + --build-report "$(cat report/build.json)" \ + --sdk "${{ matrix.sdk.name }}" \ + --repo "$GITHUB_REPOSITORY" \ + --label "${{ matrix.sdk.label }}") + echo "$RESULT" | jq . + ACTION=$(echo "$RESULT" | jq -r '.action // "unknown"') + if [ "$ACTION" = "error" ] || [ "$ACTION" = "unknown" ]; then + echo "::error::Issue management for ${{ matrix.sdk.name }} failed: $ACTION" + exit 1 + fi + - name: Close resolved drift issue + if: steps.status.outputs.has_drift == 'false' || (steps.status.outputs.has_drift == 'true' && steps.status.outputs.build_failed == 'false') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ISSUE=$(gh issue list --repo "$GITHUB_REPOSITORY" --label "${{ matrix.sdk.label }}" --state open --json number --jq '.[0].number') + if [ -n "$ISSUE" ]; then + gh issue close "$ISSUE" --repo "$GITHUB_REPOSITORY" --comment "SDK builds and tests pass after proto regeneration. Closing automatically." + echo "Closed issue #$ISSUE" + fi diff --git a/tasks/go.toml b/tasks/go.toml index 81091f2912..8ff9dc7af6 100644 --- a/tasks/go.toml +++ b/tasks/go.toml @@ -138,43 +138,17 @@ hide = true ["go:proto:check"] description = "Verify generated Go SDK proto files are up to date" dir = "sdk/go" -run = """ -#!/usr/bin/env bash -set -euo pipefail - -SDK_ROOT=$(pwd -P) -REPO_ROOT=$(cd ../.. && pwd -P) - -for tool in buf protoc-gen-go protoc-gen-go-grpc; do - if ! command -v "$tool" &>/dev/null; then - echo "ERROR: $tool not found. Run 'mise install' to install it." - exit 1 - fi -done - -WORK_DIR=$(mktemp -d) -trap 'rm -rf "$WORK_DIR"' EXIT - -if find proto -maxdepth 1 -name '*.proto' -print -quit | grep -q .; then - echo "ERROR: sdk/go/proto contains copied proto sources." - echo "Proto sources belong in the repository root proto/ directory." - exit 1 -fi - -# Generate to temp directory with adjusted output path -CHECK_TEMPLATE=$(sed 's|out: sdk/go|out: '"$WORK_DIR"'|' buf.gen.yaml) -(cd "$REPO_ROOT" && buf generate --template "$CHECK_TEMPLATE") - -DIFF_OUTPUT=$(diff -r "$WORK_DIR/proto" "$SDK_ROOT/proto" 2>&1) || true +run = 'bash ../../tasks/scripts/go_proto_check.sh text' +hide = true -if [ -n "$DIFF_OUTPUT" ]; then - echo "ERROR: Generated proto files are out of date." - echo "Run 'mise run go:proto:gen' to regenerate." - echo "" - echo "$DIFF_OUTPUT" - exit 1 -fi +["go:proto:drift"] +description = "Check Go SDK proto drift and output a JSON report" +dir = "sdk/go" +run = 'bash ../../tasks/scripts/go_proto_check.sh json' +hide = true -echo "Proto check passed: generated files are up to date." -""" +["go:proto:build-check"] +description = "Run full Go SDK proto sync, generate, build, and test pipeline" +dir = "sdk/go" +run = 'bash ../../tasks/scripts/sdk_build_check.sh go gen=go:proto:gen build=go:build test=go:test' hide = true diff --git a/tasks/scripts/go_proto_check.sh b/tasks/scripts/go_proto_check.sh new file mode 100755 index 0000000000..45a7aa2c15 --- /dev/null +++ b/tasks/scripts/go_proto_check.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +OUTPUT_FORMAT="${1:?Usage: go_proto_check.sh }" +if [ "$OUTPUT_FORMAT" != "text" ] && [ "$OUTPUT_FORMAT" != "json" ]; then + echo "ERROR: output format must be 'text' or 'json'." >&2 + exit 2 +fi + +SDK_ROOT=$(pwd -P) +REPO_ROOT=$(cd ../.. && pwd -P) + +emit_error() { + local message=$1 + if [ "$OUTPUT_FORMAT" = "json" ]; then + jq -n -c --arg message "$message" \ + '{sdk:"go", synced:false, files:[], summary:$message, error:$message}' + else + echo "ERROR: $message" >&2 + fi +} + +TOOLS=(buf protoc-gen-go protoc-gen-go-grpc) +if [ "$OUTPUT_FORMAT" = "json" ]; then + TOOLS+=(jq) +fi +for tool in "${TOOLS[@]}"; do + if ! command -v "$tool" &>/dev/null; then + if [ "$OUTPUT_FORMAT" = "json" ] && [ "$tool" = "jq" ]; then + echo '{"sdk":"go","synced":false,"files":[],"summary":"jq not found","error":"jq not found"}' + else + emit_error "$tool not found. Run 'mise install' to install it." + fi + exit 1 + fi +done + +if find proto -maxdepth 1 -name '*.proto' -print -quit | grep -q .; then + emit_error "sdk/go/proto contains copied proto sources; sources belong in the repository root proto/ directory." + exit 1 +fi + +WORK_DIR=$(mktemp -d) +RESULTS_FILE=$(mktemp) +GENERATION_LOG=$(mktemp) +trap 'rm -rf "$WORK_DIR"; rm -f "$RESULTS_FILE" "$GENERATION_LOG"' EXIT + +CHECK_TEMPLATE=$(sed 's|out: sdk/go|out: '"$WORK_DIR"'|' buf.gen.yaml) +if ! (cd "$REPO_ROOT" && buf generate --template "$CHECK_TEMPLATE") >"$GENERATION_LOG" 2>&1; then + if [ "$OUTPUT_FORMAT" = "text" ]; then + cat "$GENERATION_LOG" >&2 + fi + emit_error "buf generate failed" + exit 1 +fi + +while IFS= read -r generated; do + relative_path=${generated#"$WORK_DIR/proto/"} + committed="$SDK_ROOT/proto/$relative_path" + if [ ! -f "$committed" ]; then + printf '%s\t%s\t%s\n' "$relative_path" "added" "0" >>"$RESULTS_FILE" + continue + fi + + diff_lines=$(diff -u "$committed" "$generated" 2>/dev/null | wc -l | tr -d ' ') || true + if [ "$diff_lines" -gt 0 ]; then + printf '%s\t%s\t%s\n' "$relative_path" "modified" "$diff_lines" >>"$RESULTS_FILE" + fi +done < <(find "$WORK_DIR/proto" -name '*.go' -type f | sort) + +while IFS= read -r committed; do + relative_path=${committed#"$SDK_ROOT/proto/"} + if [ ! -f "$WORK_DIR/proto/$relative_path" ]; then + printf '%s\t%s\t%s\n' "$relative_path" "removed" "0" >>"$RESULTS_FILE" + fi +done < <(find "$SDK_ROOT/proto" -name '*.go' -type f | sort) + +if [ "$OUTPUT_FORMAT" = "text" ]; then + if [ ! -s "$RESULTS_FILE" ]; then + echo "Proto check passed: generated files are up to date." + exit 0 + fi + + echo "ERROR: Generated proto files are out of date." + echo "Run 'mise run go:proto:gen' to regenerate." + echo "" + while IFS=$'\t' read -r name status diff_lines; do + echo "$status: $name ($diff_lines diff lines)" + done <"$RESULTS_FILE" + exit 1 +fi + +REPORT=$(jq -R -c -s --arg sdk "go" ' + split("\n") | map(select(length > 0) | split("\t") | + {name: .[0], status: .[1], diff_lines: (.[2] | tonumber)}) | + {sdk: $sdk, synced: (length == 0), files: ., + summary: (if length == 0 then "all files synced" + else "\(length) file(s) drifted" end)} +' "$RESULTS_FILE") +echo "$REPORT" + +SYNCED=$(echo "$REPORT" | jq -r '.synced') +[ "$SYNCED" = "true" ] && exit 0 || exit 1 diff --git a/tasks/scripts/sdk_build_check.sh b/tasks/scripts/sdk_build_check.sh new file mode 100755 index 0000000000..b8cde79919 --- /dev/null +++ b/tasks/scripts/sdk_build_check.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SDK="${1:?Usage: sdk_build_check.sh [step2=task2] ...}" +shift + +PAIRS=("$@") +LOG_FILE=$(mktemp) +trap 'rm -f "$LOG_FILE"' EXIT + +FAILED_STEP="" +for pair in "${PAIRS[@]}"; do + STEP="${pair%%=*}" + TASK="${pair#*=}" + + if ! mise run "$TASK" >> "$LOG_FILE" 2>&1; then + FAILED_STEP="$STEP" + break + fi +done + +LOG_CONTENT=$(tail -500 "$LOG_FILE") + +if [ -z "$FAILED_STEP" ]; then + jq -n -c --arg sdk "$SDK" \ + '{sdk: $sdk, success: true, failed_step: null, log: ""}' +else + jq -n -c --arg sdk "$SDK" --arg step "$FAILED_STEP" --arg log "$LOG_CONTENT" \ + '{sdk: $sdk, success: false, failed_step: $step, log: $log}' + exit 1 +fi diff --git a/tasks/scripts/sdk_sync.py b/tasks/scripts/sdk_sync.py new file mode 100644 index 0000000000..8a0aca3bf5 --- /dev/null +++ b/tasks/scripts/sdk_sync.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDK proto sync utilities. + +Drift detection is handled by per-SDK mise tasks (go:proto:drift, +sdk:ts:proto:drift) which output JSON DriftReport objects. This CLI +provides the workflow integration layer: issue management when drift +is detected and auto-closing when it resolves. + +Subcommands: + manage-issue Create or update a GitHub drift issue (deduplicates by label) +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +def _load_sdk_configs() -> dict[str, dict]: + config_path = Path(__file__).resolve().parent.parent / "sdk-sync-config.json" + config = json.loads(config_path.read_text()) + return {entry["name"]: entry for entry in config["include"]} + + +SDK_CONFIGS = _load_sdk_configs() + +ISSUE_TEMPLATE = """\ +## Proto Drift Report + +**Summary**: {summary} + +{file_table} +{build_section} +## Fix Commands + +```bash +mise run {proto_task} # Regenerate bindings +mise run {build_task} # Verify build +mise run {test_task} # Run tests +``` + +## Agent Instructions + +This section is a ready-to-consume prompt for an AI agent. Copy it into your agent to produce a fix PR. + +
+Agent prompt (click to expand) + +{agent_section} +
+""" + +BUILD_SECTION_TEMPLATE = """\ +## Build Log + +**Failed step**: `{failed_step}` + +``` +{log} +``` + +""" + +AGENT_SECTION_TEMPLATE = """\ +Fix proto drift in the {display_name} SDK. + +## Context + +The root `proto/` directory has changed and the {display_name} SDK's generated bindings are out of sync. The drifted files are: {drifted_names}. +{build_context} + +## Steps + +1. **Regenerate bindings**: Run `mise run {proto_task}` to regenerate language-specific bindings from the updated protos. +2. **Fix compilation errors**: Read the build log above. Update the SDK source code to handle new/changed/removed proto fields: +{source_dirs} +3. **Fix test failures**: Update tests that assert on proto types that changed shape. +4. **Verify**: Run `mise run {build_task}` and `mise run {test_task}` until both pass. +5. **Create a PR**: Commit all changes and create a PR referencing this issue. + +## Scope + +- Only modify files under `sdk/{sdk}/`. Do not change root `proto/` files. +- Do not change the proto definitions. Adapt the SDK to match them. +- Keep changes minimal: only fix what the proto changes broke. +""" + + +def _sdk_display_name(sdk: str) -> str: + return SDK_CONFIGS[sdk]["display_name"] + + +def generate_issue_body( + drift_report: dict, + build_report: dict | None, + sdk: str, + max_log_lines: int = 500, +) -> str: + paths = SDK_CONFIGS[sdk] + files = drift_report.get("files", []) + drifted_files = [f for f in files if f.get("status") != "synced"] + return ISSUE_TEMPLATE.format( + summary=drift_report.get("summary", "unknown"), + file_table=_render_file_table(drifted_files), + build_section=_render_build_section(build_report, max_log_lines), + proto_task=paths["proto_task"], + build_task=paths["build_task"], + test_task=paths["test_task"], + agent_section=_render_agent_section(sdk, drifted_files, build_report), + ) + + +# --- helpers --- + + +def _render_file_table(files: list[dict]) -> str: + if not files: + return "" + lines = [ + "| File | Status | Diff Lines |", + "|------|--------|------------|", + ] + for f in files: + lines.append(f"| `{f['name']}` | {f['status']} | {f['diff_lines']} |") + return "\n".join(lines) + "\n\n" + + +def _render_build_section(build_report: dict | None, max_log_lines: int) -> str: + if not build_report or not build_report.get("failed_step"): + return "" + log_lines = build_report.get("log", "no log available").splitlines() + log = "\n".join(log_lines[-max_log_lines:]) + return BUILD_SECTION_TEMPLATE.format( + failed_step=build_report["failed_step"], + log=log, + ) + + +def _render_agent_section( + sdk: str, drifted_files: list[dict], build_report: dict | None +) -> str: + paths = SDK_CONFIGS[sdk] + display_name = _sdk_display_name(sdk) + failed_step = build_report.get("failed_step") if build_report else None + if drifted_files: + drifted_names = ", ".join(f"`{f['name']}`" for f in drifted_files) + elif sdk == "typescript": + drifted_names = ( + "not individually tracked (run `mise run sdk:ts:proto && " + "mise run sdk:ts:typecheck` to reproduce)" + ) + else: + drifted_names = "unknown" + + if failed_step: + build_context = ( + f"\nThe SDK build fails at the `{failed_step}` step after regenerating protos. " + "The build log above shows the exact error. Your job is to fix the " + f"{display_name} SDK code so it compiles and passes tests with the updated protos." + ) + else: + build_context = "\nThe SDK build status is unknown. Check if it compiles after regeneration." + + source_dirs = "\n".join(f" - `{path}`" for path in paths["source_dirs"]) + return AGENT_SECTION_TEMPLATE.format( + display_name=display_name, + drifted_names=drifted_names, + build_context=build_context, + proto_task=paths["proto_task"], + source_dirs=source_dirs, + build_task=paths["build_task"], + test_task=paths["test_task"], + sdk=sdk, + ) + + +def _run_cmd( + cmd: list[str], + cwd: str | None = None, + capture: bool = False, + stdin_data: str | None = None, + timeout: int = 60, +) -> subprocess.CompletedProcess: + try: + return subprocess.run( + cmd, + cwd=cwd, + capture_output=capture, + text=True, + input=stdin_data, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + command = " ".join(cmd) + raise RuntimeError( + f"Command timed out after {timeout} seconds: {command}" + ) from None + + +def _ensure_label(repo: str, label: str, description: str) -> None: + check = _run_cmd(["gh", "label", "view", label, "--repo", repo], capture=True) + if check.returncode != 0: + result = _run_cmd( + [ + "gh", + "label", + "create", + label, + "--repo", + repo, + "--description", + description, + "--color", + "D93F0B", + ], + capture=True, + ) + if result.returncode != 0: + details = result.stderr.strip() or "unknown error" + raise RuntimeError(f"Failed to create label '{label}': {details}") + + +def _find_open_issue(repo: str, label: str) -> dict | None: + result = _run_cmd( + [ + "gh", + "issue", + "list", + "--repo", + repo, + "--label", + label, + "--state", + "open", + "--json", + "url,number", + "--jq", + ".[0]", + ], + capture=True, + ) + if result.returncode == 0 and result.stdout.strip(): + try: + data = json.loads(result.stdout.strip()) + return {"url": data["url"], "number": str(data["number"])} + except (json.JSONDecodeError, KeyError, TypeError): + pass + return None + + +# --- public functions --- + + +def manage_issue( + drift_report: dict, + build_report: dict | None, + sdk: str, + repo: str, + label: str, +) -> dict: + try: + return _manage_issue(drift_report, build_report, sdk, repo, label) + except RuntimeError as error: + return { + "issue_url": "", + "action": "error", + "reason": str(error), + } + + +def _manage_issue( + drift_report: dict, + build_report: dict | None, + sdk: str, + repo: str, + label: str, +) -> dict: + try: + _ensure_label(repo, label, f"Proto drift detected for {sdk} SDK") + except RuntimeError as error: + return { + "issue_url": "", + "action": "error", + "reason": str(error), + } + + body = generate_issue_body(drift_report, build_report, sdk) + title = f"SDK proto drift: {sdk}" + + existing = _find_open_issue(repo, label) + if existing: + result = _run_cmd( + [ + "gh", + "issue", + "edit", + existing["number"], + "--repo", + repo, + "--body-file", + "-", + ], + capture=True, + stdin_data=body, + ) + if result.returncode == 0: + return {"issue_url": existing["url"], "action": "updated"} + return { + "issue_url": "", + "action": "error", + "reason": "Failed to update issue", + } + + result = _run_cmd( + [ + "gh", + "issue", + "create", + "--repo", + repo, + "--title", + title, + "--body-file", + "-", + "--label", + label, + ], + capture=True, + stdin_data=body, + ) + if result.returncode == 0: + url = result.stdout.strip() + return {"issue_url": url, "action": "created"} + return { + "issue_url": "", + "action": "error", + "reason": "Failed to create issue", + } + + +# --- CLI --- + + +def _load_json_arg(value: str) -> dict | list: + if value == "-": + return json.load(sys.stdin) + return json.loads(value) + + +def cmd_manage_issue(args: argparse.Namespace) -> int: + drift_report = _load_json_arg(args.drift_report) + build_report = None + if args.build_report: + build_report = _load_json_arg(args.build_report) + result = manage_issue(drift_report, build_report, args.sdk, args.repo, args.label) + print(json.dumps(result)) + return 1 if result.get("action") == "error" else 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="SDK proto sync utilities", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command", required=True) + + mi = sub.add_parser("manage-issue", help="Create or update a drift issue") + mi.add_argument("--drift-report", required=True, help="Drift report JSON") + mi.add_argument("--build-report", help="Build report JSON") + mi.add_argument( + "--sdk", + required=True, + choices=list(SDK_CONFIGS.keys()), + help="SDK name", + ) + mi.add_argument("--repo", required=True, help="GitHub repo (owner/name)") + mi.add_argument("--label", required=True, help="Issue label for deduplication") + + args = parser.parse_args() + handlers = { + "manage-issue": cmd_manage_issue, + } + return handlers[args.command](args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tasks/scripts/sdk_sync_test.py b/tasks/scripts/sdk_sync_test.py new file mode 100644 index 0000000000..a654d6bd18 --- /dev/null +++ b/tasks/scripts/sdk_sync_test.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for tasks/scripts/sdk_sync.py. + +Run via: uv run --no-project --with pytest pytest tasks/scripts/sdk_sync_test.py +""" + +from __future__ import annotations + +import subprocess +from unittest.mock import patch + +from sdk_sync import ( + _run_cmd, + generate_issue_body, + manage_issue, +) + + +def _mock_run(returncode=0, stdout="", stderr=""): + return subprocess.CompletedProcess([], returncode, stdout=stdout, stderr=stderr) + + +class TestGenerateIssueBody: + def test_go_sdk_issue_body(self): + drift = { + "sdk": "go", + "synced": False, + "files": [ + { + "name": "openshellv1/openshell.pb.go", + "status": "modified", + "diff_lines": 5, + } + ], + "summary": "1 file(s) drifted", + } + md = generate_issue_body(drift, None, "go") + assert "## Proto Drift Report" in md + assert "`openshellv1/openshell.pb.go`" in md + assert "## Fix Commands" in md + assert "mise run go:proto:gen" in md + + def test_typescript_sdk_issue_body(self): + drift = { + "sdk": "typescript", + "synced": False, + "files": [], + "summary": "typecheck failed after proto regeneration", + } + md = generate_issue_body(drift, None, "typescript") + assert "mise run sdk:ts:proto" in md + assert "mise run sdk:ts:build" in md + assert "mise run sdk:ts:test" in md + assert "sdk/typescript/src/" in md + assert "not individually tracked" in md + assert "mise run sdk:ts:proto && mise run sdk:ts:typecheck" in md + + def test_with_build_log(self): + drift = {"sdk": "go", "synced": False, "files": [], "summary": "drifted"} + build = { + "sdk": "go", + "success": False, + "failed_step": "build", + "log": "error here", + } + md = generate_issue_body(drift, build, "go") + assert "## Build Log" in md + assert "`build`" in md + assert "error here" in md + + def test_log_truncation(self): + long_log = "\n".join(f"line {i}" for i in range(1000)) + drift = {"sdk": "go", "synced": False, "files": [], "summary": "drifted"} + build = { + "sdk": "go", + "success": False, + "failed_step": "test", + "log": long_log, + } + md = generate_issue_body(drift, build, "go", max_log_lines=500) + log_section = md.split("```")[1] + assert log_section.strip().count("\n") <= 500 + assert "line 999" in md + assert "line 0" not in md + + def test_agent_instructions_present(self): + drift = { + "sdk": "go", + "synced": False, + "files": [ + { + "name": "openshellv1/openshell.pb.go", + "status": "modified", + "diff_lines": 5, + } + ], + "summary": "1 file(s) drifted", + } + build = { + "sdk": "go", + "success": False, + "failed_step": "build", + "log": "error", + } + md = generate_issue_body(drift, build, "go") + assert "## Agent Instructions" in md + assert "Agent prompt" in md + assert "mise run go:proto:gen" in md + assert "sdk/go/openshell/v1/internal/converter/" in md + assert "sdk/go/openshell/v1/types/" in md + assert "sdk/go/openshell/v1/" in md + assert "Create a PR" in md + + def test_agent_instructions_includes_failed_step(self): + drift = {"sdk": "go", "synced": False, "files": [], "summary": "drifted"} + build = { + "sdk": "go", + "success": False, + "failed_step": "test", + "log": "fail", + } + md = generate_issue_body(drift, build, "go") + agent_section = md.split("## Agent Instructions")[1] + assert "`test`" in agent_section + assert "fails at" in agent_section.lower() + + +class TestManageIssue: + @patch("sdk_sync.subprocess.run") + def test_command_timeout_uses_default_and_becomes_runtime_error(self, mock_run): + mock_run.side_effect = subprocess.TimeoutExpired(["gh", "issue", "list"], 60) + + try: + _run_cmd(["gh", "issue", "list"]) + raise AssertionError("expected timeout error") + except RuntimeError as error: + assert str(error) == ("Command timed out after 60 seconds: gh issue list") + assert mock_run.call_args.kwargs["timeout"] == 60 + + @patch("sdk_sync._ensure_label") + def test_command_timeout_returns_structured_error(self, mock_label): + mock_label.side_effect = RuntimeError( + "Command timed out after 60 seconds: gh label view" + ) + + drift = {"sdk": "go", "synced": False, "files": [], "summary": "drifted"} + result = manage_issue(drift, None, "go", "org/repo", "sdk:go:sync") + + assert result["action"] == "error" + assert "timed out after 60 seconds" in result["reason"] + + @patch("sdk_sync._find_open_issue") + @patch("sdk_sync._ensure_label") + def test_label_creation_failure_stops_issue_management(self, mock_label, mock_find): + mock_label.side_effect = RuntimeError( + "Failed to create label 'sdk:go:sync': permission denied" + ) + + drift = {"sdk": "go", "synced": False, "files": [], "summary": "drifted"} + result = manage_issue(drift, None, "go", "org/repo", "sdk:go:sync") + + assert result == { + "issue_url": "", + "action": "error", + "reason": "Failed to create label 'sdk:go:sync': permission denied", + } + mock_find.assert_not_called() + + @patch("sdk_sync._find_open_issue") + @patch("sdk_sync._ensure_label") + @patch("sdk_sync._run_cmd") + def test_create_new_issue(self, mock_run, _mock_label, mock_find): + mock_find.return_value = None + mock_run.return_value = _mock_run( + 0, stdout="https://github.com/org/repo/issues/42\n" + ) + + drift = {"sdk": "go", "synced": False, "files": [], "summary": "drifted"} + result = manage_issue(drift, None, "go", "org/repo", "sdk:go:sync") + assert result["action"] == "created" + assert "42" in result["issue_url"] + + @patch("sdk_sync._find_open_issue") + @patch("sdk_sync._ensure_label") + @patch("sdk_sync._run_cmd") + def test_update_existing_issue(self, mock_run, _mock_label, mock_find): + mock_find.return_value = { + "url": "https://github.com/org/repo/issues/10", + "number": "10", + } + mock_run.return_value = _mock_run(0) + + drift = {"sdk": "go", "synced": False, "files": [], "summary": "drifted"} + result = manage_issue(drift, None, "go", "org/repo", "sdk:go:sync") + assert result["action"] == "updated" + assert "10" in result["issue_url"] diff --git a/tasks/sdk-sync-config.json b/tasks/sdk-sync-config.json new file mode 100644 index 0000000000..6df1597be3 --- /dev/null +++ b/tasks/sdk-sync-config.json @@ -0,0 +1,32 @@ +{ + "include": [ + { + "name": "go", + "display_name": "Go", + "label": "sdk:go:sync", + "drift_task": "go:proto:drift", + "build_check_task": "go:proto:build-check", + "proto_task": "go:proto:gen", + "build_task": "go:build", + "test_task": "go:test", + "source_dirs": [ + "sdk/go/openshell/v1/internal/converter/", + "sdk/go/openshell/v1/types/", + "sdk/go/openshell/v1/" + ] + }, + { + "name": "typescript", + "display_name": "TypeScript", + "label": "sdk:typescript:sync", + "drift_task": "sdk:ts:proto:drift", + "build_check_task": "sdk:ts:proto:build-check", + "proto_task": "sdk:ts:proto", + "build_task": "sdk:ts:build", + "test_task": "sdk:ts:test", + "source_dirs": [ + "sdk/typescript/src/" + ] + } + ] +} diff --git a/tasks/typescript.toml b/tasks/typescript.toml index 823a881359..c30f941db0 100644 --- a/tasks/typescript.toml +++ b/tasks/typescript.toml @@ -67,6 +67,46 @@ depends = [ ] hide = true +["sdk:ts:proto:drift"] +description = "Check TypeScript SDK proto drift and output a JSON report" +dir = "sdk/typescript" +run = """ +#!/usr/bin/env bash +set -euo pipefail + +# TS SDK generated files (src/gen/) are gitignored, so there's no committed +# baseline to diff against. Instead, detect drift by regenerating + type +# checking: if the handwritten SDK code no longer compiles against the current +# proto definitions, that's drift. + +if ! command -v jq &>/dev/null; then + echo '{"sdk":"typescript","synced":false,"error":"jq not found"}' + exit 1 +fi + +LOG_FILE=$(mktemp) +trap 'rm -f "$LOG_FILE"' EXIT + +if ! mise run sdk:ts:proto > "$LOG_FILE" 2>&1; then + jq -n -c '{sdk:"typescript", synced:false, files:[], summary:"proto generation failed"}' + exit 1 +fi + +if ! mise run sdk:ts:typecheck >> "$LOG_FILE" 2>&1; then + jq -n -c '{sdk:"typescript", synced:false, files:[], summary:"typecheck failed after proto regeneration"}' + exit 1 +fi + +jq -n -c '{sdk:"typescript", synced:true, files:[], summary:"all files synced"}' +""" +hide = true + +["sdk:ts:proto:build-check"] +description = "Run full TypeScript SDK proto generate, typecheck, and test pipeline" +dir = "sdk/typescript" +run = 'bash ../../tasks/scripts/sdk_build_check.sh typescript gen=sdk:ts:proto typecheck=sdk:ts:typecheck test=sdk:ts:test' +hide = true + # Publish to the registry in package.json publishConfig. Set OPENSHELL_NPM_VERSION # to stamp the version from the release tag (release.py get-version --npm); the # package.json placeholder 0.0.0 is restored afterward, mirroring the Cargo