diff --git a/.github/.linkspector.yml b/.github/.linkspector.yml index 15ca806f41a..84ba43c9666 100644 --- a/.github/.linkspector.yml +++ b/.github/.linkspector.yml @@ -2,6 +2,7 @@ dirs: - . excludedFiles: - ./python/CHANGELOG.md + - "**/SKILL.md" ignorePatterns: - pattern: "/github/" - pattern: "./actions" @@ -26,7 +27,7 @@ ignorePatterns: - pattern: "https:\/\/dotnet.microsoft.com" - pattern: "https://github.com/Rel1cx/eslint-react" # excludedDirs: - # Folders which include links to localhost, since it's not ignored with regular expressions +# Folders which include links to localhost, since it's not ignored with regular expressions baseUrl: https://github.com/microsoft/agent-framework/ aliveStatusCodes: - 200 diff --git a/.github/actions/sample-validation-save-playbooks/action.yml b/.github/actions/sample-validation-save-playbooks/action.yml new file mode 100644 index 00000000000..5d1ffb15afb --- /dev/null +++ b/.github/actions/sample-validation-save-playbooks/action.yml @@ -0,0 +1,19 @@ +name: Save Sample Playbooks +description: > + Save the cached sample-validation playbooks. Split out from + sample-validation-setup (which only restores) so the save runs even when the + validation step fails. Combining restore+save via actions/cache would skip the + save on a failing job (post-if: success()), so freshly authored playbooks for + samples that failed validation would never persist. Invoke this with + 'if: not-cancelled' after the validation step in each job. + +runs: + using: "composite" + steps: + - name: Save sample playbooks cache + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + # Must match the restore path/key in sample-validation-setup/action.yml and the + # sample_validation --playbooks-dir default (samples/sample_validation/playbooks). + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml index c9d2d2d6ac6..9bb1b90f555 100644 --- a/.github/actions/sample-validation-setup/action.yml +++ b/.github/actions/sample-validation-setup/action.yml @@ -36,15 +36,31 @@ runs: shell: bash run: copilot --version && copilot -p "What can you do in one sentence?" + - name: Set up python and install the project + uses: ./.github/actions/python-setup + with: + python-version: ${{ inputs.python-version }} + os: ${{ inputs.os }} + + - name: Restore sample playbooks + # Restore-only. The matching save is a separate step in each job that runs with + # `if: ${{ !cancelled() }}` (see .github/actions/sample-validation-save-playbooks). + # A combined actions/cache would skip its post-job save on a failing job + # (post-if: success()), so playbooks authored for samples that failed validation + # would never persist. Keyed per job so each validate-* job keeps its own playbooks. + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + # Must match the sample_validation --playbooks-dir default, which resolves to + # samples/sample_validation/playbooks (see python/scripts/sample_validation/__main__.py). + # If a job overrides --playbooks-dir, update this path to match. + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Azure CLI Login uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2 with: client-id: ${{ inputs.azure-client-id }} tenant-id: ${{ inputs.azure-tenant-id }} subscription-id: ${{ inputs.azure-subscription-id }} - - - name: Set up python and install the project - uses: ./.github/actions/python-setup - with: - python-version: ${{ inputs.python-version }} - os: ${{ inputs.os }} diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index c99e63f4ef1..8c6a7db34e8 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -9,7 +9,7 @@ env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: claude-opus-4.6 + GITHUB_COPILOT_MODEL: auto COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} permissions: @@ -23,8 +23,8 @@ jobs: environment: integration env: # Required configuration for get-started samples - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -48,6 +48,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -57,12 +61,13 @@ jobs: validate-02-agents: name: Validate 02-agents + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: # Foundry configuration - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} @@ -70,12 +75,12 @@ jobs: AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME || vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }} # OpenAI configuration - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # GitHub MCP GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # Observability ENABLE_INSTRUMENTATION: "true" defaults: @@ -108,7 +113,11 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers --save-report --report-name 02-agents + cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers harness tools --save-report --report-name 02-agents + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -117,15 +126,105 @@ jobs: name: validation-report-02-agents path: python/samples/sample_validation/reports/ + validate-02-agents-harness: + name: Validate 02-agents/harness + if: false # Temporarily disabled - to free up Copilot quota for other jobs + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # Optional: enables the Foundry memory path in harness samples + FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }} + FOUNDRY_MEMORY_STORE: ${{ vars.FOUNDRY_MEMORY_STORE || '' }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + echo "FOUNDRY_EMBEDDING_MODEL=$FOUNDRY_EMBEDDING_MODEL" >> .env + echo "FOUNDRY_MEMORY_STORE=$FOUNDRY_MEMORY_STORE" >> .env + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/harness --save-report --report-name 02-agents-harness + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-02-agents-harness + path: python/samples/sample_validation/reports/ + + validate-02-agents-tools: + name: Validate 02-agents/tools + if: false # Temporarily disabled - to free up Copilot quota for other jobs + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/tools --save-report --report-name 02-agents-tools + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-02-agents-tools + path: python/samples/sample_validation/reports/ + validate-02-agents-openai: name: Validate 02-agents/providers/openai + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python @@ -151,6 +250,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -160,6 +263,7 @@ jobs: validate-02-agents-azure: name: Validate 02-agents/providers/azure + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -190,6 +294,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure --save-report --report-name 02-agents-azure + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -199,6 +307,7 @@ jobs: validate-02-agents-anthropic: name: Validate 02-agents/providers/anthropic + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -227,6 +336,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -236,6 +349,7 @@ jobs: validate-02-agents-github-copilot: name: Validate 02-agents/providers/github_copilot + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration defaults: @@ -256,6 +370,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -288,6 +406,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -320,6 +442,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -329,12 +455,12 @@ jobs: validate-02-agents-foundry: name: Validate 02-agents/providers/foundry - if: false # Temporarily disabled - provider folder also contains the local Foundry sample + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME || '' }} FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION || '' }} defaults: @@ -362,6 +488,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry --save-report --report-name 02-agents-foundry + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -404,6 +534,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -433,6 +567,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -442,11 +580,12 @@ jobs: validate-03-workflows: name: Validate 03-workflows + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -470,6 +609,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -477,14 +620,61 @@ jobs: name: validation-report-03-workflows path: python/samples/sample_validation/reports/ - validate-04-hosting: - name: Validate 04-hosting - if: false # Temporarily disabled because of sample complexity + validate-04-hosting-foundry-hosted-agents: + name: Validate 04-hosting (foundry-hosted-agents) runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # Foundry hosted agent configuration + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL }} + FOUNDRY_PROJECT_ID: ${{ vars.FOUNDRY_PROJECT_ID }} + AZURE_CONTAINER_REGISTRY_ENDPOINT: ${{ vars.AZURE_CONTAINER_REGISTRY_ENDPOINT }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + TOOLBOX_ENDPOINT: ${{ vars.TOOLBOX_ENDPOINT }} + FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_NAME }} + MEMORY_STORE_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_MEMORY_STORE }} + AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }} + AZURE_SEARCH_INDEX_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_SEARCH_INDEX_NAME }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + # Maximum parallel workers is set to 1 because all samples use the same port + run: | + cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-04-hosting-foundry-hosted-agents + path: python/samples/sample_validation/reports/ + + validate-04-hosting-other: + name: Validate 04-hosting (other) + if: false # Temporarily disabled - to free up Copilot quota for other jobs + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # A2A configuration A2A_AGENT_HOST: http://localhost:5001/ defaults: @@ -503,13 +693,17 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 04-hosting --save-report --report-name 04-hosting + cd scripts && uv run python -m sample_validation --subdir 04-hosting --exclude foundry-hosted-agents --save-report --report-name 04-hosting-other + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() with: - name: validation-report-04-hosting + name: validation-report-04-hosting-other path: python/samples/sample_validation/reports/ validate-05-end-to-end: @@ -518,8 +712,8 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} @@ -548,6 +742,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -560,16 +758,16 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # OpenAI configuration - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python @@ -594,9 +792,16 @@ jobs: echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env + - name: Pre-install AutoGen dependencies for migration samples + run: uv pip install "autogen-agentchat" "autogen-ext[openai]" + - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration + cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration --agent-timeout 600 + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -607,23 +812,25 @@ jobs: validate-semantic-kernel-migration: name: Validate semantic-kernel-migration + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration for AF AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration for SK AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} # OpenAI key - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # OpenAI configuration for SK - OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} + OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }} # Copilot Studio COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }} COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }} @@ -661,6 +868,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -675,6 +886,8 @@ jobs: needs: - validate-01-get-started - validate-02-agents + - validate-02-agents-harness + - validate-02-agents-tools - validate-02-agents-openai - validate-02-agents-azure - validate-02-agents-anthropic @@ -685,7 +898,8 @@ jobs: - validate-02-agents-copilotstudio - validate-02-agents-custom - validate-03-workflows - - validate-04-hosting + - validate-04-hosting-foundry-hosted-agents + - validate-04-hosting-other - validate-05-end-to-end - validate-autogen-migration - validate-semantic-kernel-migration diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py index 14ae940002e..d3a62997c90 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py @@ -878,6 +878,8 @@ class MagenticOrchestrator(BaseGroupChatOrchestrator): 5. The outer loop handles replanning and reenters the inner loop. """ + MANAGER_NAME: ClassVar[str] = "magentic_orchestrator" + def __init__( self, manager: MagenticManagerBase, @@ -894,7 +896,7 @@ def __init__( Keyword Args: require_plan_signoff: If True, requires human approval of the initial plan before proceeding. """ - super().__init__("magentic_orchestrator", participant_registry) + super().__init__(self.MANAGER_NAME, participant_registry) self._manager = manager self._require_plan_signoff = require_plan_signoff diff --git a/python/samples/03-workflows/orchestrations/magentic.py b/python/samples/03-workflows/orchestrations/magentic.py index 263408aa5b0..cc16eb96a05 100644 --- a/python/samples/03-workflows/orchestrations/magentic.py +++ b/python/samples/03-workflows/orchestrations/magentic.py @@ -14,6 +14,7 @@ ) from agent_framework.foundry import FoundryChatClient from agent_framework.orchestrations import GroupChatRequestSentEvent, MagenticBuilder, MagenticProgressLedger +from agent_framework_orchestrations import MagenticOrchestrator from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -113,17 +114,21 @@ async def main() -> None: print("\nStarting workflow execution...") # Keep track of the last executor to format output nicely in streaming mode - last_response_id: str | None = None + last_message_id: str | None = None output_event: WorkflowEvent | None = None async for event in workflow.run(task, stream=True): - if event.type in ("intermediate", "output") and isinstance(event.data, AgentResponseUpdate): - response_id = event.data.response_id - if response_id != last_response_id: - if last_response_id is not None: - print("\n") - print(f"- {event.executor_id}:", end=" ", flush=True) - last_response_id = response_id - print(event.data, end="", flush=True) + if event.type in ("intermediate", "output"): + if event.executor_id == MagenticOrchestrator.MANAGER_NAME: + output_event = event + else: + event_data = cast(AgentResponseUpdate, event.data) + message_id = event_data.message_id + if message_id != last_message_id: + if last_message_id is not None: + print("\n") + print(f"- {event.executor_id}:", end=" ", flush=True) + last_message_id = message_id + print(event_data, end="", flush=True) elif event.type == "magentic_orchestrator": print(f"\n[Magentic Orchestrator Event] Type: {event.data.event_type.name}") @@ -137,21 +142,20 @@ async def main() -> None: # Block to allow user to read the plan/progress before continuing # Note: this is for demonstration only and is not the recommended way to handle human interaction. # Please refer to `with_plan_review` for proper human interaction during planning phases. - await asyncio.get_event_loop().run_in_executor(None, input, "Press Enter to continue...") + # await asyncio.get_event_loop().run_in_executor(None, input, "Press Enter to continue...") elif event.type == "group_chat" and isinstance(event.data, GroupChatRequestSentEvent): print(f"\n[REQUEST SENT ({event.data.round_index})] to agent: {event.data.participant_name}") - elif event.type == "output": - output_event = event + if not output_event: + raise RuntimeError("Workflow did not produce a final output event.") - if output_event: - # The output of the magentic workflow is a collection of chat messages from all participants - outputs = cast(list[Message], output_event.data) - print("\n" + "=" * 80) - print("\nFinal Conversation Transcript:\n") - for message in outputs: - print(f"{message.author_name or message.role}: {message.text}\n") + print("\n\nWorkflow completed!") + print("Final Output:") + # The output of the Magentic workflow is an AgentResponse or AgentResponseUpdate, + # which contains the final message text. + output_message = cast(AgentResponseUpdate, output_event.data) + print(output_message.text) if __name__ == "__main__": diff --git a/python/samples/04-hosting/foundry-hosted-agents/README.md b/python/samples/04-hosting/foundry-hosted-agents/README.md index bdc4f52345c..0fc2c0db89a 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/README.md +++ b/python/samples/04-hosting/foundry-hosted-agents/README.md @@ -7,30 +7,38 @@ This directory contains samples that demonstrate how to use hosted [Agent Framew > The `agent-framework-foundry-hosting` Python API surface is intended to remain stable, but protocol 1.0.0 and 2.0.0 are incompatible. ## Samples +**Status legend:** + +| Icon | Status | Description | +|------|--------|-------------| +| ✅ | Most up to date | Demonstrates the current, recommended usage pattern. | +| ⚠️ | Legacy | Demonstrates an older pattern that has since been superseded and is no longer recommended. These samples are kept for reference and no longer maintained. | + ### Responses API -| # | Sample | Description | -|---|--------|-------------| -| 1 | [Basic](responses/01_basic/) | A minimal agent demonstrating basic request/response interaction and multi-turn conversations using `previous_response_id`. | -| 2 | [Tools](responses/02_tools/) | An agent with local tools (e.g., weather lookup), demonstrating how to register and invoke custom tool functions alongside the LLM. | -| 3 | [MCP](responses/03_mcp/) | An agent connected to a remote MCP server (GitHub), demonstrating external MCP tool provider integration. | -| 4 | [Foundry Toolbox](responses/04_foundry_toolbox/) | An agent using Azure Foundry Toolbox, demonstrating toolbox provisioning and querying available tools at runtime. | -| 5 | [Workflows](responses/05_workflows/) | An agent with a multi-step orchestrated workflow, demonstrating chaining prompts through an orchestrated flow. | -| 6 | [Files](responses/06_files/) | An agent demonstrating how to work with files in a hosted agent session, including uploading files to a hosted agent session and having the agent read and manipulate those files at runtime. | -| 7 | [Observability](responses/07_observability/) | A sample demonstrating how to enable observability for the agent deployed to Foundry. | -| 8 | [Azure AI Search RAG](responses/08_azure_search_rag/) | An agent with Retrieval Augmented Generation (RAG) capabilities backed by Azure AI Search, grounding answers in documents indexed in a pre-provisioned search index. | -| 9 | [Foundry Skills](responses/09_foundry_skills/) | An agent that uploads `SKILL.md` files to the Foundry Skills REST API and downloads them at startup, decoupling tone/policy guidelines from agent code. | -| 10 | [Foundry Memory](responses/10_foundry_memory/) | An agent with persistent semantic memory backed by a Microsoft Foundry Memory Store, using `FoundryMemoryProvider` to remember user facts across sessions. | -| 11 | [Monty CodeAct](responses/11_monty_codeact/) | An agent with a Monty-backed CodeAct context provider, exposing a single `execute_code` tool that runs Python in a [pydantic-monty](https://github.com/pydantic/monty) interpreter and invokes typed host tools (`compute`, `fetch_data`) from inside the sandbox. Uses the beta `agent-framework-monty` package. | -| 12 | [Foundry Toolbox MCP Skills](responses/12_foundry_toolbox_mcp_skills/) | An agent that discovers MCP-based skills attached to a Foundry Toolbox and serves them via `SkillsProvider(MCPSkillsSource(...))`, fetching `SKILL.md` bodies and supplementary resources on demand. | -| 13 | [Using deployed agent](responses/using_deployed_agent.py) | A sample demonstrating how to invoke an agent that has already been deployed to Foundry, showing how to interact with a hosted agent in code. | +| Status | # | Sample | Description | +|--------|---|--------|-------------| +| ✅ | 1 | [Basic](responses/01_basic/) | A minimal agent demonstrating basic request/response interaction and multi-turn conversations using `previous_response_id`. | +| ✅ | 2 | [Tools](responses/02_tools/) | An agent with local tools (e.g., weather lookup), demonstrating how to register and invoke custom tool functions alongside the LLM. | +| ✅ | 3 | [MCP](responses/03_mcp/) | An agent connected to a remote MCP server (GitHub), demonstrating external MCP tool provider integration. | +| ✅ | 4 | [Foundry Toolbox](responses/04_foundry_toolbox/) | An agent using Azure Foundry Toolbox, demonstrating toolbox provisioning and querying available tools at runtime. | +| ✅ | 5 | [Workflows](responses/05_workflows/) | An agent with a multi-step orchestrated workflow, demonstrating chaining prompts through an orchestrated flow. | +| ✅ | 6 | [Files](responses/06_files/) | An agent demonstrating how to work with files in a hosted agent session, including uploading files to a hosted agent session and having the agent read and manipulate those files at runtime. | +| ✅ | 7 | [Observability](responses/07_observability/) | A sample demonstrating how to enable observability for the agent deployed to Foundry. | +| ✅ | 8 | [Azure AI Search RAG](responses/08_azure_search_rag/) | An agent with Retrieval Augmented Generation (RAG) capabilities backed by Azure AI Search, grounding answers in documents indexed in a pre-provisioned search index. | +| ⚠️ | 9 | [Foundry Skills](responses/09_foundry_skills/) | An agent that uploads `SKILL.md` files to the Foundry Skills REST API and downloads them at startup, decoupling tone/policy guidelines from agent code. | +| ✅ | 10 | [Foundry Memory](responses/10_foundry_memory/) | An agent with persistent semantic memory backed by a Microsoft Foundry Memory Store, using `FoundryMemoryProvider` to remember user facts across sessions. | +| ✅ | 11 | [Monty CodeAct](responses/11_monty_codeact/) | An agent with a Monty-backed CodeAct context provider, exposing a single `execute_code` tool that runs Python in a [pydantic-monty](https://github.com/pydantic/monty) interpreter and invokes typed host tools (`compute`, `fetch_data`) from inside the sandbox. Uses the alpha `agent-framework-monty` package. | +| ✅ | 12 | [Foundry Toolbox MCP Skills](responses/12_foundry_toolbox_mcp_skills/) | An agent that discovers MCP-based skills attached to a Foundry Toolbox and serves them via `SkillsProvider(MCPSkillsSource(...))`, fetching `SKILL.md` bodies and supplementary resources on demand. | +| ✅ | 13 | [Using deployed agent](responses/using_deployed_agent.py) | A sample demonstrating how to invoke an agent that has already been deployed to Foundry, showing how to interact with a hosted agent in code. | ### Invocations API -| # | Sample | Description | -|---|--------|-------------| -| 1 | [Basic](invocations/01_basic/) | A minimal agent demonstrating basic request/response using the invocations protocol. | -| 2 | [Break Glass](invocations/02_break_glass/) | An agent demonstrating a "break glass" scenario where customizations of the API behaviors are needed, allowing for more direct control over how requests and responses are handled by the hosting layer. | +| Status | # | Sample | Description | +|--------|---|--------|-------------| +| ✅ | 1 | [Basic](invocations/01_basic/) | A minimal agent demonstrating basic request/response using the invocations protocol. | +| ✅ | 2 | [Break Glass](invocations/02_break_glass/) | An agent demonstrating a "break glass" scenario where customizations of the API behaviors are needed, allowing for more direct control over how requests and responses are handled by the hosting layer. | + ## Running the Agent Host Locally diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py b/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py index e5b59cc7753..a3378661d99 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py @@ -2,69 +2,16 @@ import asyncio import os -from collections.abc import Callable -from urllib.parse import urlsplit -import httpx -from agent_framework import Agent, MCPStreamableHTTPTool, tool -from agent_framework.foundry import FoundryChatClient, ResponsesHostServer -from azure.identity import DefaultAzureCredential, get_bearer_token_provider +from agent_framework import Agent, tool +from agent_framework.foundry import FoundryChatClient, FoundryToolbox, ResponsesHostServer +from azure.identity import DefaultAzureCredential from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() -def resolve_toolbox_endpoint() -> str: - """Resolve the toolbox MCP endpoint URL. - - Prefers the explicit ``TOOLBOX_ENDPOINT`` env var (set in ``agent.yaml`` or - ``agent.manifest.yaml`` and via ``azd env set TOOLBOX_ENDPOINT`` after the toolbox - is created); falls back to constructing the URL from ``FOUNDRY_PROJECT_ENDPOINT`` - and ``TOOLBOX_NAME``. - """ - if (endpoint := os.environ.get("TOOLBOX_ENDPOINT")) is not None: - if not endpoint: - raise ValueError("TOOLBOX_ENDPOINT is set but empty") - return endpoint - try: - project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"].rstrip("/") - toolbox_name = os.environ["TOOLBOX_NAME"] - except KeyError as e: - raise ValueError( - "Either set TOOLBOX_ENDPOINT, or set both FOUNDRY_PROJECT_ENDPOINT " - "and TOOLBOX_NAME to build the toolbox MCP endpoint." - ) from e - return f"{project_endpoint}/toolboxes/{toolbox_name}/mcp?api-version=v1" - - -def _toolbox_name_from_endpoint(endpoint: str) -> str: - """Extract the toolbox name from a toolbox MCP endpoint URL. - - Handles both the versioned (``.../toolboxes//versions//mcp``) and - unversioned (``.../toolboxes//mcp``) endpoint shapes that Foundry - produces. Falls back to ``"toolbox"`` when the path has no ``toolboxes`` - segment. - """ - segments = urlsplit(endpoint).path.split("/") - if "toolboxes" in segments: - idx = segments.index("toolboxes") - if idx + 1 < len(segments) and segments[idx + 1]: - return segments[idx + 1] - return "toolbox" - - -class ToolboxAuth(httpx.Auth): - """Injects a fresh bearer token on every request.""" - - def __init__(self, token_provider: Callable[[], str]): - self._get_token = token_provider - - def auth_flow(self, request: httpx.Request): - request.headers["Authorization"] = f"Bearer {self._get_token()}" - yield request - - @tool(description="Get the current working directory.", approval_mode="never_require") def get_cwd() -> str: """Get the current working directory.""" @@ -96,48 +43,35 @@ def read_file(file_path: str) -> str: async def main(): credential = DefaultAzureCredential() - # Create the toolbox - token_provider = get_bearer_token_provider(credential, "https://ai.azure.com/.default") - - # Resolve the endpoint once and derive a friendly tool name from it. When - # ``TOOLBOX_NAME`` isn't set, extract the toolbox name from the URL path so - # the tool's local name matches the upstream toolbox. - toolbox_endpoint = resolve_toolbox_endpoint() - toolbox_name = os.environ.get("TOOLBOX_NAME") or _toolbox_name_from_endpoint(toolbox_endpoint) - - async with httpx.AsyncClient( - auth=ToolboxAuth(token_provider), - timeout=120.0, - ) as http_client: - toolbox = MCPStreamableHTTPTool( - name=toolbox_name, - url=toolbox_endpoint, - http_client=http_client, - load_prompts=False, - ) - - # Create the chat client - client = FoundryChatClient( - project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], - credential=credential, - ) - - agent = Agent( - client=client, - instructions=( - "You are a friendly assistant. Keep your answers brief. " - "Make sure all mathematical calculations are performed using the code interpreter " - "instead of mental arithmetic." - ), - tools=[get_cwd, list_files, read_file, toolbox], - # History will be managed by the hosting infrastructure, thus there - # is no need to store history by the service. Learn more at: - # https://developers.openai.com/api/reference/resources/responses/methods/create - default_options={"store": False}, - ) - server = ResponsesHostServer(agent) - await server.run_async() + # FoundryToolbox resolves the toolbox endpoint from the environment + # (TOOLBOX_ENDPOINT, or FOUNDRY_PROJECT_ENDPOINT + TOOLBOX_NAME), authenticates + # every request with the credential, and transparently forwards the platform + # per-request call-id to the toolbox. The hosting server enters the agent, which + # connects the toolbox on first use and closes it at shutdown. + toolbox = FoundryToolbox(credential) + + # Create the chat client + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + credential=credential, + ) + + agent = Agent( + client=client, + instructions=( + "You are a friendly assistant. Keep your answers brief. " + "Make sure all mathematical calculations are performed using the code interpreter " + "instead of mental arithmetic." + ), + tools=[get_cwd, list_files, read_file, toolbox], + # History will be managed by the hosting infrastructure, thus there + # is no need to store history by the service. Learn more at: + # https://developers.openai.com/api/reference/resources/responses/methods/create + default_options={"store": False}, + ) + server = ResponsesHostServer(agent) + await server.run_async() if __name__ == "__main__": diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/09_foundry_skills/README.md b/python/samples/04-hosting/foundry-hosted-agents/responses/09_foundry_skills/README.md index d11ea8096db..5932d070085 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/09_foundry_skills/README.md +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/09_foundry_skills/README.md @@ -1,5 +1,7 @@ # What this sample demonstrates +> IMPORTANT: We recommend Foundry Hosted Agents to consume skills via Foundry Toolbox. Please see the [Foundry Toolbox MCP Skills](../12_foundry_toolbox_mcp_skills/README.md) sample for a more robust and production-ready approach to consuming skills in your hosted agent. + An [Agent Framework](https://github.com/microsoft/agent-framework) agent that loads its behavioral guidelines from [**Foundry Skills**](https://learn.microsoft.com/en-us/azure/foundry/agents/how-to/tools/skills?view=foundry&pivots=python) at startup, hosted using the **Responses protocol**. Skills are authored once as `SKILL.md` files, uploaded to your Foundry project through `AIProjectClient.beta.skills`, and downloaded by the agent on boot so updates ship without code changes. ## How It Works diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py index 9cef754ef3f..6f574b41252 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py @@ -4,8 +4,7 @@ import asyncio import os -from collections.abc import Mapping -from typing import Any, cast +from typing import cast from agent_framework import AgentSession from agent_framework.foundry import FoundryAgent @@ -35,47 +34,37 @@ """ -def get_hosted_session_agents(project_client: AIProjectClient) -> Any: - """Return the hosted-session operations for azure-ai-projects 2.2 or 2.3.""" - session_agents = cast(Any, project_client.agents) - if hasattr(session_agents, "create_session"): - return session_agents - return cast(Any, project_client.beta.agents) - - async def create_hosted_agent_session( *, agent: FoundryAgent, project_client: AIProjectClient, agent_name: str, agent_version: str | None, - isolation_key: str, ) -> AgentSession: """Create a hosted-agent service session and wrap it in an AgentSession.""" - create_session_kwargs: dict[str, Any] = { - "agent_name": agent_name, - "isolation_key": isolation_key, - } resolved_agent_version = agent_version if resolved_agent_version is None: - agent_details = await cast(Any, project_client.beta.agents).get( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] - agent_name=agent_name - ) - versions = getattr(agent_details, "versions", None) - if not isinstance(versions, Mapping): - raise ValueError("Hosted agent details did not include a versions mapping.") - latest_version = getattr(cast(Any, versions.get("latest")), "version", None) - if not isinstance(latest_version, str) or not latest_version: - raise ValueError("Hosted agent details did not include a latest version string.") - resolved_agent_version = latest_version + agent_details = await project_client.agents.get(agent_name) + resolved_agent_version = agent_details.versions.latest.version + + service_session = await project_client.agents.create_session( + agent_name, + version_indicator=VersionRefIndicator(agent_version=resolved_agent_version), + ) + return agent.get_session(service_session.agent_session_id) - create_session_kwargs["version_indicator"] = VersionRefIndicator(agent_version=resolved_agent_version) - service_session = await get_hosted_session_agents(project_client).create_session(**create_session_kwargs) - agent_session_id = getattr(service_session, "agent_session_id", None) - if not isinstance(agent_session_id, str) or not agent_session_id: - raise ValueError("Hosted agent session creation did not return a non-empty agent_session_id.") - return agent.get_session(agent_session_id) +async def delete_hosted_agent_session( + *, + project_client: AIProjectClient, + agent_name: str, + session: AgentSession, +) -> None: + """Delete a hosted-agent service session.""" + await project_client.agents.delete_session( + agent_name, + cast(str, session.service_session_id), + ) async def main() -> None: @@ -83,7 +72,6 @@ async def main() -> None: project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] agent_name = os.environ["FOUNDRY_AGENT_NAME"] agent_version = os.getenv("FOUNDRY_AGENT_VERSION") - isolation_key = "my-isolation-key" project_client = AIProjectClient( endpoint=project_endpoint, @@ -104,7 +92,6 @@ async def main() -> None: project_client=project_client, agent_name=agent_name, agent_version=agent_version, - isolation_key=isolation_key, ) try: @@ -132,12 +119,11 @@ async def main() -> None: if chunk.text: print(chunk.text, end="", flush=True) finally: - if isinstance(session.service_session_id, str): - await get_hosted_session_agents(project_client).delete_session( - agent_name=agent_name, - session_id=session.service_session_id, - isolation_key=isolation_key, - ) + await delete_hosted_agent_session( + project_client=project_client, + agent_name=agent_name, + session=session, + ) if __name__ == "__main__": diff --git a/python/samples/autogen-migration/orchestrations/04_magentic_one.py b/python/samples/autogen-migration/orchestrations/04_magentic_one.py index 9a7dec30eee..736d07408ce 100644 --- a/python/samples/autogen-migration/orchestrations/04_magentic_one.py +++ b/python/samples/autogen-migration/orchestrations/04_magentic_one.py @@ -10,7 +10,7 @@ Message, WorkflowEvent, ) -from agent_framework.orchestrations import MagenticProgressLedger +from agent_framework.orchestrations import MagenticOrchestrator, MagenticProgressLedger from dotenv import load_dotenv """AutoGen MagenticOneGroupChat vs Agent Framework MagenticBuilder. @@ -25,7 +25,6 @@ async def run_autogen() -> None: """AutoGen's MagenticOneGroupChat for orchestrated collaboration.""" - from autogen_agentchat.agents import AssistantAgent from autogen_agentchat.teams import MagenticOneGroupChat from autogen_agentchat.ui import Console @@ -62,7 +61,7 @@ async def run_autogen() -> None: team = MagenticOneGroupChat( participants=[researcher, coder, reviewer], model_client=client, # Coordinator uses this client - max_turns=20, + max_turns=10, max_stalls=3, ) @@ -109,7 +108,7 @@ async def run_agent_framework() -> None: instructions="You coordinate a team to complete complex tasks efficiently.", description="Orchestrator for team coordination", ), - max_round_count=20, + max_round_count=10, max_stall_count=3, max_reset_count=1, ).build() @@ -119,14 +118,18 @@ async def run_agent_framework() -> None: output_event: WorkflowEvent | None = None print("[Agent Framework] Magentic conversation:") async for event in workflow.run("Research Python async patterns and write a simple example", stream=True): - if event.type == "output" and isinstance(event.data, AgentResponseUpdate): - message_id = event.data.message_id - if message_id != last_message_id: - if last_message_id is not None: - print("\n") - print(f"- {event.executor_id}:", end=" ", flush=True) - last_message_id = message_id - print(event.data, end="", flush=True) + if event.type == "output": + if event.executor_id == MagenticOrchestrator.MANAGER_NAME: + output_event = event + else: + event_data = cast(AgentResponseUpdate, event.data) + message_id = event_data.message_id + if message_id != last_message_id: + if last_message_id is not None: + print("\n") + print(f"- {event.executor_id}:", end=" ", flush=True) + last_message_id = message_id + print(event_data, end="", flush=True) elif event.type == "magentic_orchestrator": print(f"\n[Magentic Orchestrator Event] Type: {event.data.event_type.name}") @@ -142,19 +145,15 @@ async def run_agent_framework() -> None: # Please refer to `with_plan_review` for proper human interaction during planning phases. await asyncio.get_event_loop().run_in_executor(None, input, "Press Enter to continue...") - elif event.type == "output": - output_event = event - if not output_event: raise RuntimeError("Workflow did not produce a final output event.") + print("\n\nWorkflow completed!") print("Final Output:") - # The output of the Magentic workflow is a list of ChatMessages with only one final message - # generated by the orchestrator. - output_messages = cast(list[Message], output_event.data) - if output_messages: - output = output_messages[-1].text - print(output) + # The output of the Magentic workflow is an AgentResponse or AgentResponseUpdate, + # which contains the final message text. + output_message = cast(AgentResponseUpdate, output_event.data) + print(output_message.text) async def main() -> None: diff --git a/python/samples/autogen-migration/single_agent/01_basic_agent.py b/python/samples/autogen-migration/single_agent/01_basic_agent.py index 204f50b93d5..a9bcb3da359 100644 --- a/python/samples/autogen-migration/single_agent/01_basic_agent.py +++ b/python/samples/autogen-migration/single_agent/01_basic_agent.py @@ -4,10 +4,11 @@ # "agent-framework-openai", # "autogen-agentchat", # "autogen-ext[openai]", +# "python-dotenv", # ] # /// # Run with any PEP 723 compatible runner, e.g.: -# uv run samples/autogen-migration/single_agent/01_basic_assistant_agent.py +# uv run samples/autogen-migration/single_agent/01_basic_agent.py # Copyright (c) Microsoft. All rights reserved. diff --git a/python/samples/autogen-migration/single_agent/02_agent_with_tool.py b/python/samples/autogen-migration/single_agent/02_agent_with_tool.py index 75cebac2f45..8cae2710e2f 100644 --- a/python/samples/autogen-migration/single_agent/02_agent_with_tool.py +++ b/python/samples/autogen-migration/single_agent/02_agent_with_tool.py @@ -1,3 +1,15 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "agent-framework-openai", +# "autogen-agentchat", +# "autogen-ext[openai]", +# "python-dotenv", +# ] +# /// +# Run with any PEP 723 compatible runner, e.g.: +# uv run samples/autogen-migration/single_agent/02_agent_with_tool.py + # Copyright (c) Microsoft. All rights reserved. import asyncio diff --git a/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py b/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py index 34549142b66..d4493b22a91 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py +++ b/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py b/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py index 7a5dfc6c279..21397b62c9a 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py +++ b/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py b/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py index 57789c09c94..b6fe966fe67 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py +++ b/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py b/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py index 363a92c54bd..240a03440dd 100644 --- a/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py +++ b/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py @@ -2,11 +2,21 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-copilotstudio", +# "python-dotenv", # "semantic-kernel", # ] # /// # Run with any PEP 723 compatible runner, e.g.: # uv run samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py +# +# NOTE: The metadata above resolves the Agent Framework half only. +# The Semantic Kernel half (run_semantic_kernel) requires the older +# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and +# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while +# Agent Framework requires the newer underscore-namespace SDK +# (microsoft_agents.copilotstudio.client, from +# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be +# installed in the same environment, so run each half in its own isolated env. # Copyright (c) Microsoft. All rights reserved. """Call a Copilot Studio agent with SK and Agent Framework.""" diff --git a/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py b/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py index 0b96492e057..991b4b5bf1c 100644 --- a/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py +++ b/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py @@ -2,11 +2,21 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-copilotstudio", +# "python-dotenv", # "semantic-kernel", # ] # /// # Run with any PEP 723 compatible runner, e.g.: # uv run samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py +# +# NOTE: The metadata above resolves the Agent Framework half only. +# The Semantic Kernel half (run_semantic_kernel) requires the older +# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and +# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while +# Agent Framework requires the newer underscore-namespace SDK +# (microsoft_agents.copilotstudio.client, from +# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be +# installed in the same environment, so run each half in its own isolated env. # Copyright (c) Microsoft. All rights reserved. """Stream responses from Copilot Studio agents in SK and AF.""" diff --git a/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py b/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py index 18b8d3ba4b6..b360da34e8f 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py +++ b/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py b/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py index cb4ffd83575..09b68fb7d00 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py +++ b/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py b/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py index d5e0e58e8e7..04fad5754ce 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py +++ b/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py b/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py index f5c6682c806..1a9470b7135 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py +++ b/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py index c13408bf316..8c64bcb4c13 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py +++ b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/handoff.py b/python/samples/semantic-kernel-migration/orchestrations/handoff.py index 3e34c8e51f3..08f77400611 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/handoff.py +++ b/python/samples/semantic-kernel-migration/orchestrations/handoff.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/magentic.py b/python/samples/semantic-kernel-migration/orchestrations/magentic.py index 3a9204f8c72..5e03f0a3037 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/magentic.py +++ b/python/samples/semantic-kernel-migration/orchestrations/magentic.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/sequential.py b/python/samples/semantic-kernel-migration/orchestrations/sequential.py index 59baa191651..c40792561d2 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/sequential.py +++ b/python/samples/semantic-kernel-migration/orchestrations/sequential.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py b/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py index a911eb3461a..4463de36c40 100644 --- a/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py +++ b/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-core", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/processes/nested_process.py b/python/samples/semantic-kernel-migration/processes/nested_process.py index 24572023e12..61a9aef7ee6 100644 --- a/python/samples/semantic-kernel-migration/processes/nested_process.py +++ b/python/samples/semantic-kernel-migration/processes/nested_process.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-core", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/scripts/sample_validation/__main__.py b/python/scripts/sample_validation/__main__.py index 948fed3a307..f0b93b2b67c 100644 --- a/python/scripts/sample_validation/__main__.py +++ b/python/scripts/sample_validation/__main__.py @@ -17,6 +17,7 @@ import argparse import asyncio +import logging import os import sys import time @@ -29,6 +30,8 @@ from sample_validation.report import save_report from sample_validation.workflow import ValidationConfig, create_validation_workflow +logging.basicConfig(level=logging.INFO) + def parse_arguments() -> argparse.Namespace: """Parse command line arguments.""" @@ -82,6 +85,33 @@ def parse_arguments() -> argparse.Namespace: help="Subdirectory paths to exclude (relative to the search directory set by --subdir)", ) + parser.add_argument( + "--playbooks-dir", + type=str, + default="./sample_validation/playbooks", + help=( + "Directory (relative to samples/) where cached per-sample playbooks are stored and " + "reused across runs (default: ./sample_validation/playbooks)" + ), + ) + + parser.add_argument( + "--no-cache", + action="store_true", + help="Ignore cached playbooks and always validate every sample with the agent", + ) + + parser.add_argument( + "--agent-timeout", + type=int, + default=120, + help=( + "Per-turn timeout in seconds for the GitHub Copilot agent while validating a sample. " + "Increase for long-running samples such as hosted agents that start a server and make " + "multiple calls (default: 120)" + ), + ) + return parser.parse_args() @@ -107,14 +137,23 @@ async def main() -> int: ) # Create validation config + playbooks_dir = (samples_dir / args.playbooks_dir).resolve() config = ValidationConfig( samples_dir=samples_dir, python_root=python_root, subdir=args.subdir, exclude=args.exclude, max_parallel_workers=max(1, args.max_parallel_workers), + playbooks_dir=playbooks_dir, + use_cache=not args.no_cache, + agent_timeout=max(1, args.agent_timeout), ) + if config.use_cache: + print(f"Playbook cache: {playbooks_dir}") + else: + print("Playbook cache: disabled (--no-cache)") + # Create and run the workflow workflow = create_validation_workflow(config) diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 6ebe25a8d4e..e7436cf64ad 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -3,10 +3,12 @@ import logging from collections import deque from dataclasses import dataclass +from pathlib import Path from agent_framework import ( Executor, Message, + SkillsProvider, Workflow, WorkflowBuilder, WorkflowContext, @@ -18,25 +20,47 @@ from copilot.session_events import PermissionRequest from pydantic import BaseModel from sample_validation.const import WORKER_COMPLETED -from sample_validation.discovery import DiscoveryResult from sample_validation.models import ( ExecutionResult, + ReplayResult, RunResult, RunStatus, SampleInfo, ValidationConfig, WorkflowCreationResult, ) +from sample_validation.playbook import ( + Playbook, + PlaybookStore, + compute_sample_hash, + sample_files, +) from typing_extensions import Never logger = logging.getLogger(__name__) +# Directory containing file-based skills used by the validation agents. +SKILLS_DIR = Path(__file__).parent / "skills" + + +class PlaybookScript(BaseModel): + """Reproducible Python validation script the agent returns so future runs can skip the agent. + + ``script`` is a complete, self-contained Python program that reproduces a successful + validation without any AI assistance. The harness runs it with the active interpreter from + the Python root and treats a zero exit code as success. + """ + + script: str + timeout: int = 120 + env: dict[str, str] = {} + class AgentResponseFormat(BaseModel): status: str output: str error: str - fix: str + playbook: PlaybookScript | None = None @dataclass @@ -58,17 +82,47 @@ class BatchCompletion: "Analyze the sample code and execute it as it is. Based on the execution result, determine " "if it runs successfully, fails, or is missing_setup. Use `missing_setup` if the sample reports " "missing required environment variables. The environment you're given should contain the necessary " - "variables. Don't create new environment variables nor modify the sample code.\n" + "variables. Don't create new environment variables nor modify the sample code, unless an available " + "skill instructs you to do so for the setup issue you detected. When a skill applies to the problem, " + "follow its guidance to resolve the setup and then re-run the sample.\n" "Feel free to install any required dependencies if needed.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " "based on your analysis of the code. You do not need to consult human on what to respond.\n" - "If the sample fails, investigate the error and suggest a fix.\n" + "Fail fast and do not attempt to fix the sample unless instructed by a skill.\n" + "When (and only when) the status is `success`, also return a `playbook`: a self-contained " + "Python script that reproduces this successful validation WITHOUT any AI assistance, so future " + "runs can replay it deterministically. `playbook.script` is a COMPLETE Python program with " + "these rules:\n" + " - It is run with the SAME Python interpreter, and its WORKING DIRECTORY is the python/ " + "directory (the repo's Python root, where the shared .env lives). Reference the sample with a " + 'path relative to python/, e.g. "samples/04-hosting/foundry-hosted-agents/responses/01_basic".\n' + " - It MUST exit 0 when validation passes and exit non-zero (raise or sys.exit(1)) when it " + "fails. The harness maps exit code 0 to success and anything else to failure.\n" + " - It may import ONLY: the sample's own already-installed dependencies, the Python standard " + "library, and `httpx`. Do not require any other third-party package.\n" + " - It must be non-interactive and deterministic (no prompts, no reliance on you).\n" + " - For a normal run-to-completion sample: launch it (e.g. subprocess.run([sys.executable, " + '"/main.py"], ...) feeding any needed stdin via input=), then assert on the exit code ' + "and/or expected output.\n" + " - For a LONG-LIVED SERVER sample (an HTTP server that never returns on its own, e.g. a " + "hosted agent listening on a port): start it in the BACKGROUND (subprocess.Popen), POLL until " + "the port accepts connections (bounded readiness loop), send the real HTTP request(s) with " + "httpx, and assert HTTP 200 plus a non-empty/expected response. When the sample keeps " + "conversational state, exercise MULTIPLE TURNS (e.g. turn 1 provides a fact like a name, turn 2 " + "asks it to be recalled) and assert the recall. ALWAYS terminate the server in a finally block " + "so no process or port is leaked (the harness also force-kills the process group as a backstop).\n" + " - If you had to edit any sample file to make it run, BAKE that edit into the script (apply it " + "in code before running); the harness restores sample files after replay.\n" + " - On failure, print the captured server/sample output before exiting non-zero to aid debugging.\n" + "Also set `playbook.timeout` (whole-run seconds; the harness caps it) and optional " + "`playbook.env` (extra environment overrides; the CI environment already provides the real " + "credentials, so usually leave this empty).\n" "Return ONLY valid JSON with this schema:\n" "{\n" ' "status": "success|failure|missing_setup",\n' ' "output": "short summary of the result and what you did if the sample was interactive",\n' ' "error": "error details or empty string",\n' - ' "fix": "suggested code fix if the sample failed, otherwise empty string"\n' + ' "playbook": {"script": "", "timeout": 120, "env": {}}\n' "}\n\n" ) @@ -116,9 +170,16 @@ class CustomAgentExecutor(Executor): # Retry in case GitHub Copilot agent encounters transient errors unrelated to the sample execution. RETRY_COUNT = 1 - def __init__(self, agent: GitHubCopilotAgent): + def __init__( + self, + agent: GitHubCopilotAgent, + store: PlaybookStore | None = None, + python_root: Path | None = None, + ): super().__init__(id=agent.id) self.agent = agent + self._store = store + self._python_root = python_root self._session = agent.create_session() @handler @@ -126,6 +187,9 @@ async def handle_task( self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult] ) -> None: """Execute one sample task and notify collector + coordinator.""" + # Snapshot the sample's pristine content so we can restore the working tree afterwards + # (the agent, or a baked-in playbook edit, may modify sample files while validating). + snapshot = self._snapshot_sample(sample) current_retry = 0 while True: try: @@ -144,8 +208,12 @@ async def handle_task( status=status_from_text(result_payload.status), output=result_payload.output, error=result_payload.error, - fix=result_payload.fix, ) + if result.status == RunStatus.SUCCESS: + # Restore the sample to its committed content before hashing so the playbook's + # sample_hash matches what a fresh checkout (and future staleness check) sees. + self._restore_snapshot(snapshot) + self._save_playbook(sample, result_payload) break except Exception as ex: if current_retry < self.RETRY_COUNT: @@ -167,7 +235,6 @@ async def handle_task( status=RunStatus.FAILURE, output="", error=f"Original error: {ex}. Restart error: {restart_ex}", - fix="", ) break @@ -177,15 +244,58 @@ async def handle_task( status=RunStatus.FAILURE, output="", error=str(ex), - fix="", ) break + # Always restore the working tree so a run never leaves the repository dirty + # (on success this is a no-op because we already restored above). + self._restore_snapshot(snapshot) + await ctx.send_message(result, target_id="collector") await ctx.send_message(WorkerFreed(worker_id=self.id), target_id="coordinator") await ctx.add_event(WorkflowEvent(WORKER_COMPLETED, sample)) # type: ignore + def _snapshot_sample(self, sample: SampleInfo) -> dict[Path, bytes]: + """Capture the current on-disk bytes of every file that makes up a sample.""" + snapshot: dict[Path, bytes] = {} + for file in sample_files(sample): + try: + snapshot[file] = file.read_bytes() + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not snapshot {file}: {ex}") + return snapshot + + def _restore_snapshot(self, snapshot: dict[Path, bytes]) -> None: + """Restore snapshotted files to their pristine bytes if the agent changed them.""" + for file, original in snapshot.items(): + try: + if file.read_bytes() != original: + file.write_bytes(original) + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not restore {file}: {ex}") + + def _save_playbook(self, sample: SampleInfo, payload: AgentResponseFormat) -> None: + """Persist a cached playbook for a sample the agent validated successfully.""" + if self._store is None or payload.playbook is None: + return + spec = payload.playbook + if not spec.script.strip(): + logger.warning(f"Agent returned no replay script for {sample.relative_path}; skipping playbook.") + return + try: + playbook = Playbook( + sample=sample.relative_path, + sample_hash=compute_sample_hash(sample), + script=spec.script, + timeout=int(spec.timeout), + env=dict(spec.env), + ) + path = self._store.save(playbook) + logger.info(f"Saved playbook for {sample.relative_path} -> {path}") + except Exception as ex: # pragma: no cover - defensive; never fail validation over caching + logger.warning(f"Could not save playbook for {sample.relative_path}: {ex}") + class BatchCoordinatorExecutor(Executor): """Dispatch sample tasks to worker executors in bounded batches.""" @@ -263,40 +373,60 @@ class CreateConcurrentValidationWorkflowExecutor(Executor): def __init__(self, config: ValidationConfig): super().__init__(id="create_dynamic_workflow") self.config = config + self._store = ( + PlaybookStore(config.playbooks_dir) + if config.use_cache and config.playbooks_dir is not None + else None + ) @handler async def create( self, - discovery: DiscoveryResult, + replay: ReplayResult, ctx: WorkflowContext[WorkflowCreationResult], ) -> None: """Create a nested workflow with a coordinator + worker fan-out/fan-in.""" - sample_count = len(discovery.samples) - print(f"\nCreating nested batched workflow for {sample_count} samples...") + samples = replay.remaining_samples + cached_results = replay.cached_results + sample_count = len(samples) + print( + f"\nCreating nested batched workflow for {sample_count} samples " + f"({len(cached_results)} served from cache)..." + ) if sample_count == 0: await ctx.send_message( - WorkflowCreationResult(samples=[], workflow=None, agents=[]) + WorkflowCreationResult( + samples=[], workflow=None, agents=[], cached_results=cached_results + ) ) return agents: list[GitHubCopilotAgent] = [] workers: list[CustomAgentExecutor] = [] - for index, sample in enumerate(discovery.samples, start=1): + for index, sample in enumerate(samples, start=1): agent_id = f"sample_validator_{index}({sample.relative_path})" agent = GitHubCopilotAgent( id=agent_id, name=agent_id, instructions=AgentInstruction, + context_providers=[SkillsProvider.from_paths( + skill_paths=str(SKILLS_DIR), + disable_load_skill_approval=True, + disable_read_skill_resource_approval=True, + disable_run_skill_script_approval=True, + )], default_options={ "on_permission_request": prompt_permission, - "timeout": 120, + "timeout": self.config.agent_timeout, }, # type: ignore ) agents.append(agent) - workers.append(CustomAgentExecutor(agent)) + workers.append( + CustomAgentExecutor(agent, store=self._store, python_root=self.config.python_root) + ) coordinator = BatchCoordinatorExecutor( worker_ids=[worker.id for worker in workers], @@ -314,8 +444,9 @@ async def create( await ctx.send_message( WorkflowCreationResult( - samples=discovery.samples, + samples=samples, workflow=nested_workflow, agents=agents, + cached_results=cached_results, ) ) diff --git a/python/scripts/sample_validation/discovery.py b/python/scripts/sample_validation/discovery.py index c5424dd6ee9..492843ada56 100644 --- a/python/scripts/sample_validation/discovery.py +++ b/python/scripts/sample_validation/discovery.py @@ -58,7 +58,7 @@ def discover_samples( exclude: list[str] | None = None, ) -> list[SampleInfo]: """ - Find all Python sample files in the samples directory. + Find all samples in the samples directory. Args: samples_dir: Root samples directory @@ -80,38 +80,48 @@ def discover_samples( # Resolve excluded paths to absolute for reliable comparison exclude_paths = {(search_dir / exc).resolve() for exc in (exclude or [])} - python_files: list[Path] = [] + samples: list[Path] = [] # Walk through all subdirectories and find .py files for root, dirs, files in os.walk(search_dir): - # Skip directories that start with _, __pycache__, or excluded paths + # Skip directories that start with _ or ., __pycache__, virtual envs, or excluded paths. + # Dot-directories (e.g. .venv) may be created in a sample folder during validation and + # must never be treated as samples. dirs[:] = [ d for d in dirs if not d.startswith("_") - and d != "__pycache__" + and not d.startswith(".") + and d not in ("__pycache__", "venv", "node_modules") and (Path(root) / d).resolve() not in exclude_paths ] + # If the whole directory is a sample, add the directory itself and do NOT descend into + # it: everything under a main.py/app.py entry point belongs to that one sample. + if any(file in ("main.py", "app.py") for file in files): + samples.append(Path(root)) + dirs[:] = [] + continue + for file in files: # Skip files that start with _ and include only scripts with a main entrypoint guard if file.endswith(".py") and not file.startswith("_"): file_path = Path(root) / file if _has_main_entrypoint_guard(file_path): - python_files.append(file_path) + samples.append(file_path) # Sort files for consistent execution order - python_files = sorted(python_files) + samples = sorted(samples) # Convert to SampleInfo objects - samples: list[SampleInfo] = [] - for path in python_files: + samples_info: list[SampleInfo] = [] + for path in samples: try: - samples.append(SampleInfo.from_path(path, samples_dir)) + samples_info.append(SampleInfo.from_path(path, samples_dir)) except Exception as e: print(f"Warning: Could not read {path}: {e}") - return samples + return samples_info class DiscoverSamplesExecutor(Executor): diff --git a/python/scripts/sample_validation/models.py b/python/scripts/sample_validation/models.py index ff45b5909ba..3740e4d491b 100644 --- a/python/scripts/sample_validation/models.py +++ b/python/scripts/sample_validation/models.py @@ -20,6 +20,9 @@ class ValidationConfig: subdir: str | None = None exclude: list[str] | None = None max_parallel_workers: int = 10 + playbooks_dir: Path | None = None + use_cache: bool = True + agent_timeout: int = 120 @dataclass @@ -28,7 +31,6 @@ class SampleInfo: path: Path relative_path: str - code: str @classmethod def from_path(cls, path: Path, samples_dir: Path) -> "SampleInfo": @@ -36,7 +38,6 @@ def from_path(cls, path: Path, samples_dir: Path) -> "SampleInfo": return cls( path=path, relative_path=str(path.relative_to(samples_dir)), - code=path.read_text(encoding="utf-8"), ) @@ -47,6 +48,19 @@ class DiscoveryResult: samples: list[SampleInfo] +@dataclass +class ReplayResult: + """Outcome of attempting to replay cached playbooks for discovered samples. + + Samples whose cached playbook replayed successfully are captured in + ``cached_results`` and skip the agent entirely. Everything else flows to the + agent-driven validation as ``remaining_samples``. + """ + + remaining_samples: list[SampleInfo] + cached_results: list["RunResult"] = field(default_factory=list) # type: ignore[assignment] + + @dataclass class WorkflowCreationResult: """Result of creating a nested per-sample concurrent workflow.""" @@ -54,6 +68,7 @@ class WorkflowCreationResult: samples: list[SampleInfo] workflow: Workflow | None agents: list[GitHubCopilotAgent] + cached_results: list["RunResult"] = field(default_factory=list) # type: ignore[assignment] class RunStatus(Enum): @@ -72,7 +87,6 @@ class RunResult: status: RunStatus output: str error: str - fix: str @dataclass @@ -154,7 +168,6 @@ def to_dict(self) -> dict[str, object]: "status": r.status.value, "output": r.output, "error": r.error, - "fix": r.fix, } for r in self.results ], diff --git a/python/scripts/sample_validation/playbook.py b/python/scripts/sample_validation/playbook.py new file mode 100644 index 00000000000..c35fbb0eae6 --- /dev/null +++ b/python/scripts/sample_validation/playbook.py @@ -0,0 +1,349 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Cached validation playbooks. + +A *playbook* is a small JSON recipe that captures exactly how a sample was +successfully validated so future runs can replay it deterministically without +invoking an LLM agent. The agent produces a playbook the first time it +validates a sample; subsequent runs replay it directly and only fall back to +the agent when the playbook is missing, stale (the sample changed), or the +replay fails. + +The playbook payload is an agent-authored **Python validation script**. Replaying +a playbook writes that script to a temporary file and runs it with the active +interpreter, treating a zero exit code as success. A single script can validate +any sample shape: a plain sample that runs to completion, or a long-lived server +sample (e.g. a hosted agent) that must be started in the background, exercised +over HTTP, asserted, and then torn down. + +The script is a cache-only artifact (it is never committed to the repository), so +it is exempt from the repository's Python style/type rules; the harness treats it +as an opaque, self-contained program. Any paths the script references should be +relative to the Python root (the ``python/`` directory), which is the working +directory used at replay time, so playbooks stay portable across machines and CI +runners. +""" + +import asyncio +import hashlib +import json +import logging +import os +import signal +import subprocess +import sys +import tempfile +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime +from pathlib import Path + +from sample_validation.models import RunResult, RunStatus, SampleInfo + +logger = logging.getLogger(__name__) + +# Bump when the on-disk playbook format changes in an incompatible way. +# v2: the payload is an agent-authored Python script (replaces the v1 RunSpec). +SCHEMA_VERSION = 2 + +# Hard cap on how long a replayed sample may run, regardless of the value the +# agent recorded, to protect the CI job from a runaway process. +MAX_REPLAY_TIMEOUT = 600 + + +@dataclass +class Playbook: + """A cached, replayable recipe for validating a single sample. + + The recipe is an agent-authored Python ``script`` that reproduces a + successful validation without any AI assistance. Replaying runs the script + with the active interpreter from the Python root and maps a zero exit code to + success. The script is self-contained: it launches/exercises the sample, + performs its own assertions, bakes in any edits it needs, and (for a server + sample) starts and tears down the server itself. + """ + + sample: str + sample_hash: str + script: str + timeout: int = 120 + env: dict[str, str] = field(default_factory=dict) # type: ignore + expected_status: str = RunStatus.SUCCESS.value + schema_version: int = SCHEMA_VERSION + generated_at: str = field(default_factory=lambda: datetime.now().isoformat()) + + def to_dict(self) -> dict[str, object]: + """Serialize to a JSON-friendly dict.""" + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, object]) -> "Playbook": + """Deserialize from a dict, tolerating unknown/missing optional keys. + + Tolerant of older schema versions (which lack ``script``) so + ``PlaybookStore.load`` can read them and then reject them on the schema + version check rather than raising. + """ + return cls( + sample=str(data["sample"]), + sample_hash=str(data["sample_hash"]), + script=str(data.get("script") or ""), + timeout=int(data.get("timeout") or 120), # type: ignore[arg-type] + env={str(k): str(v) for k, v in (data.get("env") or {}).items()}, # type: ignore[union-attr] + expected_status=str(data.get("expected_status") or RunStatus.SUCCESS.value), + schema_version=int(data.get("schema_version") or SCHEMA_VERSION), # type: ignore[arg-type] + generated_at=str(data.get("generated_at") or datetime.now().isoformat()), + ) + + +def sample_files(sample: SampleInfo) -> list[Path]: + """Return the files that make up a sample. + + For a single-file sample this is just that file. For a directory sample + (``main.py``/``app.py`` entrypoint) it is every ``.py`` file in the tree. + + Note: we only consider source code files, not data files, nor deployment + artifacts (Dockerfile, requirements.txt, etc.) for now. + """ + path = sample.path + if path.is_dir(): + return sorted(p for p in path.rglob("*.py") if "__pycache__" not in p.parts) + return [path] + + +def compute_sample_hash(sample: SampleInfo) -> str: + """Compute a stable content hash for a sample. + + For a single-file sample the hash covers that file. For a directory sample + (``main.py``/``app.py`` entrypoint) it covers every ``.py`` file in the + directory tree so any change to the sample invalidates the cached playbook. + """ + hasher = hashlib.sha256() + path = sample.path + for file in sample_files(sample): + try: + hasher.update(file.relative_to(path.parent).as_posix().encode("utf-8")) + hasher.update(b"\0") + hasher.update(file.read_bytes()) + hasher.update(b"\0") + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not hash {file}: {ex}") + + return f"sha256:{hasher.hexdigest()}" + + +class PlaybookStore: + """Loads and saves per-sample playbooks under a directory. + + Each sample gets one JSON file whose name is derived from the sample's + relative path (path separators replaced so it stays flat and filesystem + safe). + """ + + def __init__(self, directory: Path) -> None: + self.directory = directory + + @staticmethod + def _slug(relative_path: str) -> str: + return relative_path.replace("\\", "/").replace("/", "__") + ".json" + + def _path_for(self, relative_path: str) -> Path: + return self.directory / self._slug(relative_path) + + def load(self, sample: SampleInfo) -> Playbook | None: + """Return the stored playbook for a sample, or ``None`` if absent/invalid.""" + path = self._path_for(sample.relative_path) + if not path.exists(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + playbook = Playbook.from_dict(data) + except (OSError, ValueError, KeyError) as ex: + logger.warning(f"Ignoring unreadable playbook {path}: {ex}") + return None + if playbook.schema_version != SCHEMA_VERSION: + logger.info(f"Ignoring playbook with schema {playbook.schema_version} for {sample.relative_path}") + return None + return playbook + + def save(self, playbook: Playbook) -> Path: + """Persist a playbook to disk and return its path.""" + self.directory.mkdir(parents=True, exist_ok=True) + path = self._path_for(playbook.sample) + path.write_text(json.dumps(playbook.to_dict(), indent=2), encoding="utf-8") + return path + + def is_valid_for(self, sample: SampleInfo) -> Playbook | None: + """Return the stored playbook only if it matches the sample's current hash.""" + playbook = self.load(sample) + if playbook is None: + return None + current_hash = compute_sample_hash(sample) + if playbook.sample_hash != current_hash: + logger.info(f"Playbook for {sample.relative_path} is stale (sample changed); will re-validate.") + return None + return playbook + + +def _new_process_group_kwargs() -> dict[str, object]: + """Return subprocess kwargs that launch the child in its own process group. + + Isolating the replay in a new group/session lets us tear down the whole tree + (for example a background server the script spawned) even if the script exits + without cleaning up after itself. + """ + if os.name == "posix": + return {"start_new_session": True} + # Windows: a new process group so the whole tree can be terminated together. + return {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} + + +def _terminate_process_tree(proc: "asyncio.subprocess.Process", pgid: int | None) -> None: + """Best-effort kill of a replay process and any descendants it leaked. + + This always runs, even when the script itself already exited cleanly, because + the whole point of the backstop is to reap a background server the script may + have started but failed to tear down. On POSIX we signal the saved process + group id (captured at spawn time, since the leader may already be reaped); + on Windows we kill the process tree by PID. + """ + try: + if os.name == "posix": + if pgid is not None: + os.killpg(pgid, signal.SIGKILL) + elif proc.returncode is None: + proc.kill() + else: + # Kill by PID (not image name) including the whole child tree. + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], + capture_output=True, + check=False, + ) + except (ProcessLookupError, OSError) as ex: # pragma: no cover - defensive + logger.debug(f"Could not terminate replay process tree {proc.pid}: {ex}") + + +async def replay_playbook(playbook: Playbook, python_root: Path) -> RunResult: + """Deterministically replay a playbook and return a ``RunResult``. + + Writes the recorded Python script to a temporary file and runs it with the + active interpreter, using the Python root as the working directory. A zero + exit code maps to ``SUCCESS``; anything else yields a ``FAILURE`` so the + caller falls back to the agent. + + Sample files are snapshotted and restored so a replay never leaves the + repository dirty (the script may edit the sample to make it run), and the + replay process is launched in its own process group so a background server it + starts is always torn down afterwards, even if the script fails to clean up. + """ + sample_path = playbook.sample + sample_info = SampleInfo(path=python_root / sample_path, relative_path=sample_path) + + if not playbook.script.strip(): + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error="Playbook has an empty validation script.", + ) + + # Snapshot every file that makes up the sample so we can restore it byte-exact + # no matter how we exit (the script is allowed to edit the sample in place). + snapshot: dict[Path, bytes] = {} + for file in sample_files(sample_info): + try: + snapshot[file] = file.read_bytes() + except OSError: + pass + + script_file = tempfile.NamedTemporaryFile( # noqa: SIM115 - closed explicitly below + prefix="playbook_", suffix=".py", delete=False + ) + script_path = Path(script_file.name) + proc: asyncio.subprocess.Process | None = None + pgid: int | None = None + try: + script_file.write(playbook.script.encode("utf-8")) + script_file.close() + + env = {**os.environ, **playbook.env} + timeout = min(max(1, playbook.timeout), MAX_REPLAY_TIMEOUT) + + start = time.perf_counter() + try: + proc = await asyncio.create_subprocess_exec( + sys.executable, + str(script_path), + cwd=str(python_root), + env=env, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + **_new_process_group_kwargs(), # type: ignore[arg-type] + ) + except OSError as ex: + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay could not start the script: {ex}", + ) + + # Capture the process-group id now (== proc.pid, since we start a new + # session/group) so teardown can kill leaked descendants even after the + # leader process has exited and been reaped. + if os.name == "posix": + try: + pgid = os.getpgid(proc.pid) + except OSError: # pragma: no cover - defensive + pgid = None + + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except (TimeoutError, asyncio.TimeoutError): + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay timed out after {timeout}s.", + ) + + duration = time.perf_counter() - start + output_text = stdout.decode("utf-8", errors="replace") if stdout else "" + + if proc.returncode == 0: + return RunResult( + sample=sample_info, + status=RunStatus.SUCCESS, + output=f"Replayed cached playbook successfully in {duration:.1f}s.", + error="", + ) + + # Non-zero exit: surface a truncated tail to aid debugging; caller retries with the agent. + tail = output_text[-1000:] + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay exited with code {proc.returncode}. Output tail: {tail}", + ) + finally: + # Always tear down the process group (even if the script exited cleanly) + # so a background server it leaked is killed, then reap the leader. + if proc is not None: + _terminate_process_tree(proc, pgid) + try: + await proc.wait() + except Exception: # pragma: no cover - defensive + pass + for target, original in snapshot.items(): + try: + if target.read_bytes() != original: + target.write_bytes(original) + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not restore {target} after replay: {ex}") + try: + script_path.unlink() + except OSError: # pragma: no cover - defensive + pass diff --git a/python/scripts/sample_validation/replay_executor.py b/python/scripts/sample_validation/replay_executor.py new file mode 100644 index 00000000000..6806243d6d3 --- /dev/null +++ b/python/scripts/sample_validation/replay_executor.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Executor that replays cached playbooks before falling back to the agent.""" + +import asyncio +import logging + +from agent_framework import Executor, WorkflowContext, handler + +from sample_validation.models import ( + DiscoveryResult, + ReplayResult, + RunResult, + RunStatus, + SampleInfo, + ValidationConfig, +) +from sample_validation.playbook import PlaybookStore, replay_playbook + +logger = logging.getLogger(__name__) + + +class ReplayCachedPlaybooksExecutor(Executor): + """Replay cached playbooks and split samples into cached vs. agent-bound. + + For every discovered sample we look for a playbook whose recorded hash still + matches the sample's current content. Matching playbooks are replayed + deterministically (no LLM). Samples that pass are returned as cached + results; samples without a valid playbook, or whose replay fails, are passed + on for agent-driven validation (which will refresh the playbook). + """ + + def __init__(self, config: ValidationConfig) -> None: + super().__init__(id="replay_cached_playbooks") + self.config = config + self._store = PlaybookStore(config.playbooks_dir) if config.playbooks_dir else None + + @handler + async def replay(self, discovery: DiscoveryResult, ctx: WorkflowContext[ReplayResult]) -> None: + """Attempt cached replay for each sample, then forward the remainder.""" + samples = discovery.samples + + if not self.config.use_cache or self._store is None or not samples: + await ctx.send_message(ReplayResult(remaining_samples=samples, cached_results=[])) + return + + store = self._store + candidates: list[tuple[SampleInfo, object]] = [] + remaining: list[SampleInfo] = [] + + for sample in samples: + playbook = store.is_valid_for(sample) + if playbook is None: + remaining.append(sample) + else: + candidates.append((sample, playbook)) + + print( + f"\nCache check: {len(candidates)} sample(s) have a valid playbook, " + f"{len(remaining)} need the agent." + ) + + cached_results: list[RunResult] = [] + if candidates: + semaphore = asyncio.Semaphore(max(1, self.config.max_parallel_workers)) + + async def _run(sample: SampleInfo, playbook: object) -> tuple[SampleInfo, RunResult]: + async with semaphore: + result = await replay_playbook(playbook, self.config.python_root) # type: ignore[arg-type] + return sample, result + + for coro in asyncio.as_completed([_run(s, pb) for s, pb in candidates]): + sample, result = await coro + if result.status == RunStatus.SUCCESS: + cached_results.append(result) + print(f"[cache hit] {sample.relative_path}") + else: + # Replay failed: hand the sample to the agent to re-validate and refresh the playbook. + logger.info(f"Cached replay failed for {sample.relative_path}: {result.error}") + print(f"[cache miss] {sample.relative_path} (replay failed, will use agent)") + remaining.append(sample) + + await ctx.send_message(ReplayResult(remaining_samples=remaining, cached_results=cached_results)) diff --git a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py index c7244cff2a2..143a8214caf 100644 --- a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py +++ b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py @@ -36,8 +36,11 @@ async def run( self, creation: WorkflowCreationResult, ctx: WorkflowContext[ExecutionResult] ) -> None: """Run the nested workflow and emit execution results.""" + cached_results = list(creation.cached_results) + if creation.workflow is None: - await ctx.send_message(ExecutionResult(results=[])) + # Nothing left for the agent; emit whatever the cache produced. + await ctx.send_message(ExecutionResult(results=cached_results)) return print("\nRunning nested batched workflow...") @@ -50,10 +53,10 @@ async def run( CoordinatorStart(samples=creation.samples), stream=True ): if event.type == "output" and isinstance(event.data, ExecutionResult): - result = event.data # type: ignore - elif event.type == WORKER_COMPLETED and isinstance( + result = event.data + elif event.type == WORKER_COMPLETED and isinstance( # type: ignore event.data, SampleInfo - ): # type: ignore + ): remaining_sample_counts -= 1 print( f"Completed validation for sample: {event.data.relative_path:<80} | " @@ -61,7 +64,9 @@ async def run( ) if result is not None: - await ctx.send_message(result) + await ctx.send_message( + ExecutionResult(results=cached_results + result.results) + ) else: fallback_results = [ RunResult( @@ -69,10 +74,11 @@ async def run( status=RunStatus.FAILURE, output="", error="Nested workflow did not return an ExecutionResult.", - fix="", ) for sample in creation.samples ] - await ctx.send_message(ExecutionResult(results=fallback_results)) + await ctx.send_message( + ExecutionResult(results=cached_results + fallback_results) + ) finally: await stop_agents(creation.agents) diff --git a/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md new file mode 100644 index 00000000000..eb4e75746e0 --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md @@ -0,0 +1,52 @@ +--- +name: foundry-config-setup +description: Resolve missing setup caused by a hardcoded Foundry project endpoint or model in a sample. Use when a sample fails because it uses a placeholder/hardcoded project_endpoint (for example "https://your-project.services.ai.azure.com") or a hardcoded model instead of reading them from the environment. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +## Usage + +Some samples (notably those under `01-get-started`) hardcode the Foundry +project endpoint and model directly in the `FoundryChatClient` constructor +using placeholder values, for example: + +```python +client = FoundryChatClient( + project_endpoint="https://your-project.services.ai.azure.com", + model="gpt-4o", + credential=AzureCliCredential(), +) +``` + +These placeholder values are not real and the sample cannot run as written. +The validation environment provides the real values through environment +variables, so the sample must read them from the environment instead. + +When you detect a hardcoded/placeholder `project_endpoint` (or `model`) that +is causing the sample to fail with missing or invalid setup: + +1. Replace the hardcoded `project_endpoint` value with a read from the + `FOUNDRY_PROJECT_ENDPOINT` environment variable. +2. Replace the hardcoded `model` value with a read from the `FOUNDRY_MODEL` + environment variable. +3. Ensure `import os` is present at the top of the file. + +The corrected constructor should look like: + +```python +import os + +client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["FOUNDRY_MODEL"], + credential=AzureCliCredential(), +) +``` + +These samples are intentionally written with hardcoded placeholders, so this +is expected setup—not a defect in the sample. After applying the change, +re-run the sample and report the result as a `success` if it now runs. diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md new file mode 100644 index 00000000000..018953b75ca --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -0,0 +1,351 @@ +--- +name: foundry-hosted-agent-validation +description: > + Step-by-step process for validating a Python Foundry hosted agent sample + (under python/samples/04-hosting/foundry-hosted-agents/) end to end — running + it locally (native runtime and `azd ai agent run`) and after deploying it to + an Azure AI Foundry project with `azd`. Use this when asked to validate a hosted + agent sample. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +# Validating a Foundry Hosted Agent Sample + +A hosted agent sample is "validated" when it passes **three** independent +checks, plus cleanup: + +1. **Local, native runtime** — run the sample's own entry point + (`python main.py`) and invoke it over HTTP. +2. **Local, via `azd ai agent run`** — the `azd` local dev loop. +3. **Deployed** — `azd deploy` to Foundry, then invoke the hosted agent. + +Each check must succeed for **single-turn** and **multi-turn** (session / +`previous_response_id`) conversation. Always end with **cleanup** (delete the +deployed agent, remove the temp `azd` project, restore the sample dir). + +> Read the sample's own `README.md` and the parent +> `.../foundry-hosted-agents/README.md` first — they define the run/deploy +> commands and any sample-specific payload. This skill captures the process and +> the non-obvious gotchas the READMEs don't. + +--- + +## Automated script + +[`scripts/validate_hosted_agent.sh`](scripts/validate_hosted_agent.sh) runs all +three phases (and cleanup) non-interactively — use it for a full pass, and read +the phases below to interpret failures or validate a non-`responses` sample by +hand. Run `--help` for the full dependency list and options. + +```bash +# Full validation of the responses/01_basic sample (default sample dir): +python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh \ + --project-endpoint "https://.services.ai.azure.com/api/projects/" \ + --model "" \ + --acr-endpoint "" + +# Local-only (skip deploy), or point at another sample: +python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh --skip-deploy \ + --sample-dir python/samples/04-hosting/foundry-hosted-agents/responses/02_tools \ + --project-endpoint "..." --model "..." +``` + +Inputs may also come from env vars (`FOUNDRY_PROJECT_ENDPOINT`, +`AZURE_AI_MODEL_DEPLOYMENT_NAME`, `FOUNDRY_PROJECT_ID`, +`AZURE_CONTAINER_REGISTRY_ENDPOINT`). Phase flags: `--skip-native`, +`--skip-azd-local`, `--skip-deploy`; `--no-cleanup`/`--keep-agent` to inspect +afterward. The script encodes every gotcha below (model template fix, +pre-existing-agent removal, ACR reuse, port/temp cleanup). + +--- + +## Inputs you need before starting + +Gather these (ask the user if not provided): + +- **Foundry project endpoint**, e.g. + `https://.services.ai.azure.com/api/projects/`. +- **Foundry project resource id** (for non-interactive `azd ai agent init`): + `/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects/`. + Find it with `az cognitiveservices account list` + the project name. +- **A real, deployed model name** in that project (e.g. `gpt-4.1-mini`). This is + often **different** from the model id in `agent.manifest.yaml` — the actual + deployment name wins. +- **An existing ACR to reuse** for deployment (login server, e.g. + `myacr.azurecr.io`). Reusing one avoids `azd provision` creating resources. +- Whether a like-named agent **already exists** in the project (remove it first + for a clean validation — see below). + +## Tooling / auth + +- `az` (logged in: `az login`) and `azd` (logged in: `azd auth login`). +- `azd` **agents extension**: `azd extension list` should show + `azure.ai.agents`; install with `azd extension install azure.ai.agents`. +- `uv` for the native-Python local run. **`python` need not be on PATH** — `uv` + and `azd ai agent run` provision their own interpreter. +- Docker is **not** required when you reuse an ACR (`remoteBuild: true` builds + in ACR Tasks). + +--- + +## Phase 0 — Understand the sample + +A responses/invocations sample folder typically contains: +`main.py` (entry point + `ResponsesHostServer`/`InvocationsHostServer`), +`agent.manifest.yaml` (used by `azd ai agent init`), `agent.yaml` (the deployed +agent definition), `requirements.txt`, `Dockerfile`, `.env.example`. + +**The sample is the whole directory whose entry point is `main.py` — not every +`.py` file in it.** Other Python files in (or alongside) a sample folder are +**helper/companion scripts**, not standalone samples. Do **not** treat a helper +script as an individual sample — validate the sample via its `main.py` host, and +run a helper only when the sample's `README.md` calls for it as a setup. + +**Skip legacy samples — only validate "most up to date" samples.** The parent +`.../foundry-hosted-agents/README.md` sample tables have a **Status** column +(✅ Most up to date, ⚠️ Legacy). Samples marked **⚠️ Legacy** demonstrate an +older, superseded pattern and do **not** need validation — skip them and mark +them as success. Only validate samples marked **✅ Most up to date**. + +Note the **protocol** (`responses` or `invocations`) from `agent.yaml` / +manifest — it changes the invoke command (`--protocol invocations`) and the HTTP +path (`/responses` vs the invocations route). + +--- + +## Phase 1 — Local validation, native runtime (Python) + +Run from the sample directory. + +```bash +uv venv .venv --python 3.12 # 3.12 matches the sample Dockerfile +uv pip install --python .venv/... -r requirements.txt +``` + +Create `.env` from `.env.example` with the **real** values: + +``` +FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +AZURE_AI_MODEL_DEPLOYMENT_NAME="" +``` + +Start the server (`python main.py`) — it listens on `http://localhost:8088`. +`main.py` uses `DefaultAzureCredential`, so `az login` must be current. + +Invoke (single turn), capture the returned `response_id`, then reuse it for a +follow-up turn to confirm memory: + +```bash +curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" \ + -d '{"input": "My name is Tao. Remember it."}' +# take response_id from the JSON, then: +curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" \ + -d '{"input": "What is my name?", "previous_response_id": ""}' +``` + +PowerShell: use `Invoke-WebRequest -Uri http://localhost:8088/responses -Method POST -ContentType application/json -Body '...'`. + +**Pass:** HTTP 200, non-empty `output[].content[].text`, and the second turn +recalls the name. Stop the server afterward. + +### Recording the playbook (cached replay) + +The sample-validation harness caches a **playbook** so future runs replay your +result without an agent. For a hosted-agent sample the playbook is an +agent-authored **Python script** that reproduces **only this Phase 1 +native-local smoke test** — never the `azd`/deploy phases (those are +credential- and deployment-bound, non-deterministic, and must not enter the +replay cache). + +Emit a self-contained script that the harness runs with the active interpreter +from the `python/` directory (exit code 0 = success). It may import only the +sample's own installed deps, the Python stdlib, and `httpx`. It must: + +1. Start the sample server in the **background** with + `subprocess.Popen([sys.executable, "/main.py"])` (path relative to + `python/`; the shared `python/.env` and the process env supply credentials). +2. **Poll** `http://localhost:8088` until the port accepts a connection (bounded + readiness loop with a timeout), failing if it never comes up. +3. POST turn 1 to the protocol's route (`/responses` for `responses` samples; + the invocations route for `invocations` samples), capture the `response_id`, + then POST turn 2 with `previous_response_id` to confirm recall. +4. **Assert** HTTP 200, non-empty `output[].content[].text`, and that turn 2 + recalls the fact from turn 1. +5. **Always** terminate the server in a `finally` block so port 8088 is freed + (the harness force-kills the process group as a backstop, but the script must + still clean up). On any failure, print the captured server output before + exiting non-zero. + +Keep the deep three-phase validation (below) for a full manual/`azd` pass; it is +out of scope for the cached playbook. + +--- + +## Phase 2 — Local validation via `azd ai agent run` + +### Init the azd project (once) + +Run in an **empty temp directory outside the repo** (short path avoids Windows +path-length issues, e.g. `C:\afval\`). Point `-m` at the **local** +manifest so it validates the working-tree sample: + +```bash +azd ai agent init -m /agent.manifest.yaml \ + --project-id "" \ + --model-deployment "" \ + --agent-name "" \ + --no-prompt --force +``` + +`init` downloads the template into a **subfolder named after the agent**, so the +azd project root is `//`. `cd` there for all later `azd` +commands. + +> **Before init, remove any `.venv` you created in the sample dir** — `init` +> copies the entire manifest directory into `src/`. (`.venv` is excluded from +> deploy packaging by `.agentignore`/`.dockerignore`, so it is harmless but +> bloats/slows the copy.) + +### Fix the model deployment name (critical — see Gotcha 1) + +```bash +azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME "" +``` + +### Run and invoke locally + +```bash +azd ai agent run --no-inspector # auto-creates a uv venv + installs deps; listens on :8088 +azd ai agent invoke --local --new-session "My name is Tao. Remember it." +azd ai agent invoke --local "What is my name?" # same session is reused automatically +``` + +**Pass:** both invokes return text; the second recalls the name (same +`Session:` id). Stop the `run` process afterward. + +--- + +## Removing a pre-existing agent (do this before deploying) + +`init` prints a warning if the agent name already exists in the project. To +delete it, note that **`azd ai agent delete`/`show` resolve the deployed agent +name from an azd env var, not from the positional argument.** The var is +`AGENT_{SERVICEKEY}_NAME`, where `SERVICEKEY` = the `azure.yaml` service name +uppercased with `-`/spaces → `_`. + +Example for service `agent-framework-agent-basic-responses`: + +```bash +azd env set AGENT_AGENT_FRAMEWORK_AGENT_BASIC_RESPONSES_NAME agent-framework-agent-basic-responses +azd ai agent delete --force --no-prompt --output json +# -> {"object":"agent.deleted","name":"...","deleted":true} +``` + +(After a successful `azd deploy`, this var is set automatically, so later +`show`/`delete`/`invoke` work without setting it.) + +--- + +## Phase 3 — Deploy and validate + +### Reuse an existing ACR (avoid provisioning) + +For an existing project + model, **do not run `azd provision`/`azd up`** — the +generated `azure.yaml` has a `deployments` block for the manifest's model +(often an auto-selected `GlobalProvisionedManaged` PTU SKU) that provision would +try to create (costly / quota failures). Instead reuse an ACR: + +```bash +azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT # e.g. myacr.azurecr.io +azd deploy +``` + +`azd deploy` fails with _"could not determine container registry endpoint"_ if +this is unset and no ACR is provisioned. + +### Verify the deployed model env var, then invoke + +```bash +azd ai agent show --output json # check definition.environment_variables.AZURE_AI_MODEL_DEPLOYMENT_NAME +azd ai agent invoke --new-session "My name is Tao. Remember it." +azd ai agent invoke "What is my name?" +``` + +Use `--output raw` on invoke to see raw SSE events and any failure, e.g.: + +``` +event: response.failed +... "code": "DeploymentNotFound" ... 404 ... +``` + +`DeploymentNotFound` means the deployed `AZURE_AI_MODEL_DEPLOYMENT_NAME` points +at a model that isn't deployed → fix per Gotcha 1 and redeploy (creates a new +version). + +**Pass:** agent reaches `status: active`, invoke returns text (not empty, no +`response.failed`), and multi-turn recalls the name. + +--- + +## Cleanup (always) + +- Delete the deployed agent: `azd ai agent delete --force --no-prompt`. +- Delete the temp `azd` project directory. +- Remove `.env`/`.venv` you created in the sample dir; confirm the sample dir is + pristine (`git status --porcelain ` is empty — `.env`/`.venv` are + gitignored). +- Stop any leftover local server still holding port 8088: + `Get-NetTCPConnection -LocalPort 8088 -State Listen` → `Stop-Process -Id ` + (Linux/macOS: `lsof -ti:8088 | xargs kill`). Stopping the shell may leave the + child interpreter running. + +--- + +## Gotchas (the parts that waste the most time) + +1. **The deployed model name comes from `agent.yaml`, not the azd env.** + `azd ai agent init` ignores `--model-deployment` in `--no-prompt` mode and + writes the **manifest's** model id (e.g. `gpt-4.1-mini`) as a **literal** into + both the azd env `AZURE_AI_MODEL_DEPLOYMENT_NAME` and the generated + `src//agent.yaml` env var. Local runs read the azd env (so + `azd env set` fixes them), but **deployment injects `agent.yaml`'s value**. + Fix by setting the generated `agent.yaml` env var to the template + `value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME}` (what the repo sample already uses; + `init` flattens it) **and** `azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME `, + then redeploy. A literal `value: ` also works. + +2. **`azd ai agent delete/show` need the `AGENT_{SERVICEKEY}_NAME` env var** — a + bare positional agent name is treated as the _service_ name and the deployed + agent name is looked up from that env var (see "Removing a pre-existing + agent"). + +3. **`azd provision`/`azd up` will try to create the manifest's model + deployment** (from `azure.yaml`'s `deployments` block). Prefer `azd deploy` + with a reused ACR when the project + model already exist. + +4. **`python` on PATH is not required.** `uv venv` and `azd ai agent run` + provision their own interpreter and install `requirements.txt`. + +5. **`init` copies the whole manifest directory into `src/`.** Remove a local + `.venv` from the sample dir first to keep the copy clean/fast. + +6. **Port 8088 can stay bound after stopping the shell** — kill the interpreter + by PID (see Cleanup). + +--- + +## Success checklist + +- [ ] Native local run: 200 + non-empty text + multi-turn recall. +- [ ] `azd ai agent run` local: text returned + session reused across invokes. +- [ ] Pre-existing agent removed (if any). +- [ ] `azd deploy` succeeds; agent `status: active`. +- [ ] `azd ai agent show` confirms `AZURE_AI_MODEL_DEPLOYMENT_NAME` = the real + deployed model. +- [ ] Deployed invoke: text returned (no `response.failed`) + multi-turn recall. +- [ ] Cleanup done: agent deleted, temp project removed, sample dir pristine, + port 8088 free. diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh new file mode 100755 index 00000000000..1eab498da85 --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh @@ -0,0 +1,390 @@ +#!/usr/bin/env bash +# +# validate_hosted_agent.sh — automate validation of a Foundry hosted agent sample. +# +# Runs the three validation checks from the `foundry-hosted-agent-validation` +# skill and cleans up afterward: +# 1. Native local run (uv venv + `python main.py` + curl; responses protocol) +# 2. Local via azd (`azd ai agent init` -> `azd ai agent run` -> invoke) +# 3. Deployed (`azd deploy` to Foundry -> `azd ai agent invoke`) +# Each check exercises a single-turn and a multi-turn (memory) exchange. +# +# --------------------------------------------------------------------------- +# DEPENDENCIES (must be installed and on PATH in the runner / pipeline) +# --------------------------------------------------------------------------- +# bash >= 3.2 (no bash-4-only features are used) +# az Azure CLI, logged in (`az login`) — used to derive the project +# id when --project-id is omitted, and by DefaultAzureCredential. +# azd Azure Developer CLI, logged in (`azd auth login`), WITH the +# agents extension installed: +# azd extension install azure.ai.agents +# uv https://docs.astral.sh/uv/ — creates the native venv & installs +# requirements. (Note: a `python` interpreter on PATH is NOT +# required; uv and `azd ai agent run` provision their own.) +# curl native HTTP invocation of the local server. +# jq JSON parsing of responses / `azd ai agent show` output. +# coreutils awk, sed, grep, tr, printf, mktemp, sleep (standard on Linux/macOS). +# +# Optional but recommended: +# lsof OR fuser reliable freeing of the local port during cleanup. Without +# them, a child server process may linger on the port. +# Docker NOT needed when reusing an ACR (remote build runs in ACR +# Tasks). Only needed if you switch to a local docker build. +# +# --------------------------------------------------------------------------- +# REQUIRED INPUTS (flags or environment variables) +# --------------------------------------------------------------------------- +# --project-endpoint | FOUNDRY_PROJECT_ENDPOINT (native + all phases) +# --model | AZURE_AI_MODEL_DEPLOYMENT_NAME (a REAL deployed model) +# --project-id | FOUNDRY_PROJECT_ID (azd-local + deploy) +# --acr-endpoint | AZURE_CONTAINER_REGISTRY_ENDPOINT (deploy only) +# +# If --project-id is omitted it is derived from the endpoint via `az`. +# +# --------------------------------------------------------------------------- +# USAGE +# --------------------------------------------------------------------------- +# validate_hosted_agent.sh [options] +# +# --sample-dir DIR Sample folder (default: the responses/01_basic sample +# resolved relative to this script's repo). +# --project-endpoint URL Foundry project endpoint. +# --project-id ID Foundry project ARM resource id. +# --model NAME Real model deployment name in the project. +# --acr-endpoint HOST Existing ACR login server (e.g. myacr.azurecr.io). +# --agent-name NAME Override agent/service name (default: read agent.yaml). +# --port N Local port (default: 8088). +# --skip-native Skip phase 1 (native local run). +# --skip-azd-local Skip phase 2 (azd ai agent run). +# --skip-deploy Skip phase 3 (deploy to Foundry). +# --keep-agent Do NOT delete the deployed agent during cleanup. +# --no-cleanup Keep temp azd project, sample .env/.venv, and agent. +# -h | --help Show this help. +# +# EXIT CODES: 0 = all enabled checks passed; non-zero = a check or setup failed. +# +set -euo pipefail + +# --------------------------- pretty logging -------------------------------- +_c() { [ -t 1 ] && printf '%s' "$1" || printf ''; } +log() { printf '%s[validate]%s %s\n' "$(_c $'\033[1;34m')" "$(_c $'\033[0m')" "$*"; } +ok() { printf '%s[ pass ]%s %s\n' "$(_c $'\033[1;32m')" "$(_c $'\033[0m')" "$*"; } +warn() { printf '%s[ warn ]%s %s\n' "$(_c $'\033[1;33m')" "$(_c $'\033[0m')" "$*" >&2; } +die() { printf '%s[ FAIL ]%s %s\n' "$(_c $'\033[1;31m')" "$(_c $'\033[0m')" "$*" >&2; exit 1; } + +# --------------------------- defaults / args ------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# scripts -> foundry-hosted-agent-validation -> skills -> sample_validation -> scripts -> python +PYTHON_ROOT="$(cd "$SCRIPT_DIR/../../../../.." && pwd)" +DEFAULT_SAMPLE="$PYTHON_ROOT/samples/04-hosting/foundry-hosted-agents/responses/01_basic" + +SAMPLE_DIR="${DEFAULT_SAMPLE}" +PROJECT_ENDPOINT="${FOUNDRY_PROJECT_ENDPOINT:-}" +PROJECT_ID="${FOUNDRY_PROJECT_ID:-}" +MODEL="${AZURE_AI_MODEL_DEPLOYMENT_NAME:-}" +ACR_ENDPOINT="${AZURE_CONTAINER_REGISTRY_ENDPOINT:-}" +AGENT_NAME="" +PORT=8088 +DO_NATIVE=1; DO_AZD_LOCAL=1; DO_DEPLOY=1 +KEEP_AGENT=0; DO_CLEANUP=1 + +usage() { sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//; s/^#//' | sed '$d'; } + +while [ $# -gt 0 ]; do + case "$1" in + --sample-dir) SAMPLE_DIR="$2"; shift 2;; + --project-endpoint) PROJECT_ENDPOINT="$2"; shift 2;; + --project-id) PROJECT_ID="$2"; shift 2;; + --model) MODEL="$2"; shift 2;; + --acr-endpoint) ACR_ENDPOINT="$2"; shift 2;; + --agent-name) AGENT_NAME="$2"; shift 2;; + --port) PORT="$2"; shift 2;; + --skip-native) DO_NATIVE=0; shift;; + --skip-azd-local) DO_AZD_LOCAL=0; shift;; + --skip-deploy) DO_DEPLOY=0; shift;; + --keep-agent) KEEP_AGENT=1; shift;; + --no-cleanup) DO_CLEANUP=0; KEEP_AGENT=1; shift;; + -h|--help) usage; exit 0;; + *) die "unknown argument: $1 (use --help)";; + esac +done + +# --------------------------- state / cleanup ------------------------------- +WORK_DIR=""; PROJECT_ROOT=""; NATIVE_PID=""; AZD_RUN_PID="" +DEPLOYED=0; CREATED_ENV=0; CREATED_VENV=0 +PASS_COUNT=0; FAIL_COUNT=0; SUMMARY="" + +record() { # $1=pass|fail $2=text + if [ "$1" = pass ]; then PASS_COUNT=$((PASS_COUNT+1)); SUMMARY="${SUMMARY} [pass] $2"$'\n'; + else FAIL_COUNT=$((FAIL_COUNT+1)); SUMMARY="${SUMMARY} [FAIL] $2"$'\n'; fi +} + +free_port() { # $1=port + local port="$1" pids + if command -v lsof >/dev/null 2>&1; then + pids="$(lsof -ti "tcp:${port}" 2>/dev/null || true)" + [ -n "$pids" ] && kill -9 $pids 2>/dev/null || true + elif command -v fuser >/dev/null 2>&1; then + fuser -k "${port}/tcp" 2>/dev/null || true + fi +} + +stop_bg() { # $1=pid $2=port + local pid="${1:-}" port="${2:-}" + if [ -n "$pid" ]; then kill "$pid" 2>/dev/null || true; sleep 1; kill -9 "$pid" 2>/dev/null || true; fi + [ -n "$port" ] && free_port "$port" +} + +cleanup() { + local ec=$? + log "cleanup..." + stop_bg "$NATIVE_PID" "$PORT" + stop_bg "$AZD_RUN_PID" "$PORT" + if [ "$DO_CLEANUP" = 1 ]; then + if [ "$DEPLOYED" = 1 ] && [ "$KEEP_AGENT" = 0 ] && [ -n "$PROJECT_ROOT" ] && [ -d "$PROJECT_ROOT" ]; then + log "deleting deployed agent '$AGENT_NAME'" + ( cd "$PROJECT_ROOT" && azd ai agent delete "$AGENT_NAME" --force --no-prompt >/dev/null 2>&1 ) || \ + warn "could not delete deployed agent (delete it manually)" + fi + [ "$CREATED_ENV" = 1 ] && rm -f "$SAMPLE_DIR/.env" 2>/dev/null || true + [ "$CREATED_VENV" = 1 ] && rm -rf "$SAMPLE_DIR/.venv" 2>/dev/null || true + [ -n "$WORK_DIR" ] && rm -rf "$WORK_DIR" 2>/dev/null || true + else + warn "cleanup skipped (--no-cleanup). Temp project: ${WORK_DIR:-n/a}" + fi + if [ "$ec" -ne 0 ]; then printf '\n'; die "aborted (exit $ec)"; fi +} +trap cleanup EXIT INT TERM + +# --------------------------- preflight ------------------------------------- +require_cmd() { command -v "$1" >/dev/null 2>&1 || die "missing dependency '$1' — $2"; } + +log "preflight: checking dependencies" +require_cmd az "install Azure CLI: https://aka.ms/azcli" +require_cmd azd "install Azure Developer CLI: https://aka.ms/azd" +require_cmd uv "install uv: https://docs.astral.sh/uv/" +require_cmd curl "install curl" +require_cmd jq "install jq: https://jqlang.github.io/jq/" +require_cmd awk "install coreutils (awk)" +require_cmd mktemp "install coreutils (mktemp)" + +az account show >/dev/null 2>&1 || die "not logged in to Azure CLI — run: az login" +azd auth login --check-status >/dev/null 2>&1 || die "not logged in to azd — run: azd auth login" +azd extension list 2>/dev/null | grep -qi 'azure.ai.agents' || \ + die "azd agents extension missing — run: azd extension install azure.ai.agents" +command -v lsof >/dev/null 2>&1 || command -v fuser >/dev/null 2>&1 || \ + warn "neither lsof nor fuser found; port ${PORT} may not be freed cleanly on exit" + +[ -d "$SAMPLE_DIR" ] || die "sample dir not found: $SAMPLE_DIR" +[ -f "$SAMPLE_DIR/main.py" ] || die "no main.py in sample dir: $SAMPLE_DIR" +[ -f "$SAMPLE_DIR/agent.yaml" ] || die "no agent.yaml in sample dir: $SAMPLE_DIR" +[ -f "$SAMPLE_DIR/agent.manifest.yaml" ] || die "no agent.manifest.yaml in sample dir: $SAMPLE_DIR" +[ -n "$MODEL" ] || die "model deployment name required (--model or AZURE_AI_MODEL_DEPLOYMENT_NAME)" + +# derive agent name + protocol from agent.yaml +if [ -z "$AGENT_NAME" ]; then + AGENT_NAME="$(grep -E '^name:' "$SAMPLE_DIR/agent.yaml" | head -1 | sed -E 's/^name:[[:space:]]*//' | tr -d '"'\''\r')" +fi +[ -n "$AGENT_NAME" ] || die "could not determine agent name (pass --agent-name)" +PROTOCOL="$(grep -oE 'protocol:[[:space:]]*[a-zA-Z]+' "$SAMPLE_DIR/agent.yaml" | head -1 | sed -E 's/protocol:[[:space:]]*//')" +[ -n "$PROTOCOL" ] || PROTOCOL=responses + +WORK_DIR="$(mktemp -d 2>/dev/null || mktemp -d -t afval)" +log "sample=$SAMPLE_DIR" +log "agent=$AGENT_NAME protocol=$PROTOCOL model=$MODEL port=$PORT" +log "workdir=$WORK_DIR" + +# --------------------------- helpers --------------------------------------- +wait_http() { # $1=url $2=timeout_s + local url="$1" timeout="${2:-90}" i=0 code + while :; do + code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 "$url" 2>/dev/null || echo 000)" + [ "$code" != "000" ] && return 0 + i=$((i+1)); [ "$i" -ge "$timeout" ] && return 1; sleep 1 + done +} + +# assert an `azd ai agent invoke --output raw` SSE stream succeeded. +# $1 = raw output ; $2 = optional expected substring (case-insensitive) +check_sse() { + local raw="$1" expect="${2:-}" + printf '%s' "$raw" | grep -qi 'DeploymentNotFound' && { warn "DeploymentNotFound"; return 2; } + printf '%s' "$raw" | grep -q 'response.failed' && { warn "response.failed"; return 2; } + printf '%s' "$raw" | grep -q 'response.completed' || { warn "no response.completed"; return 3; } + if [ -n "$expect" ]; then printf '%s' "$raw" | grep -qi "$expect" || { warn "missing '$expect'"; return 4; }; fi + return 0 +} + +resolve_project_id() { + [ -n "$PROJECT_ID" ] && return 0 + [ -n "$PROJECT_ENDPOINT" ] || die "need --project-id or --project-endpoint to resolve the project" + # endpoint: https://.services.ai.azure.com/api/projects/ + local acct proj + acct="$(printf '%s' "$PROJECT_ENDPOINT" | sed -E 's#https?://([^.]+)\..*#\1#')" + proj="$(printf '%s' "$PROJECT_ENDPOINT" | sed -E 's#.*/projects/([^/?]+).*#\1#')" + [ -n "$acct" ] && [ -n "$proj" ] || die "could not parse account/project from endpoint" + local acct_id + acct_id="$(az cognitiveservices account list --query "[?name=='${acct}'].id | [0]" -o tsv 2>/dev/null || true)" + [ -n "$acct_id" ] || die "could not find Foundry account '$acct' via az (check subscription/login)" + PROJECT_ID="${acct_id}/projects/${proj}" + log "resolved project-id=$PROJECT_ID" +} + +# service-name -> AGENT_{KEY}_NAME env var used by `azd ai agent delete/show` +name_key() { # $1=service name + local k; k="$(printf '%s' "$1" | tr 'a-z' 'A-Z' | tr '-' '_' | tr ' ' '_')" + printf 'AGENT_%s_NAME' "$k" +} + +INITED=0 +ensure_azd_project() { + [ "$INITED" = 1 ] && return 0 + resolve_project_id + # Remove a local .venv from the sample dir so `init` doesn't copy it into src/. + [ -d "$SAMPLE_DIR/.venv" ] && [ "$CREATED_VENV" = 1 ] && { rm -rf "$SAMPLE_DIR/.venv"; CREATED_VENV=0; } + + log "azd ai agent init (this can take a few minutes)" + ( cd "$WORK_DIR" && azd ai agent init \ + -m "$SAMPLE_DIR/agent.manifest.yaml" \ + --project-id "$PROJECT_ID" \ + --model-deployment "$MODEL" \ + --agent-name "$AGENT_NAME" \ + --no-prompt --force ) >"$WORK_DIR/init.log" 2>&1 \ + || { tail -n 40 "$WORK_DIR/init.log" >&2; die "azd ai agent init failed"; } + + PROJECT_ROOT="$WORK_DIR/$AGENT_NAME" + [ -f "$PROJECT_ROOT/azure.yaml" ] || die "azd project not found at $PROJECT_ROOT after init" + + # GOTCHA fix: init hardcodes the manifest model into the generated agent.yaml. + # Restore the template so `azd deploy` substitutes the azd env value. + local ay="$PROJECT_ROOT/src/$AGENT_NAME/agent.yaml" + if [ -f "$ay" ]; then + awk ' + seen && /value:/ { sub(/value:.*/, "value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME}"); seen=0 } + /- name: AZURE_AI_MODEL_DEPLOYMENT_NAME/ { seen=1 } + { print } + ' "$ay" > "$ay.tmp" && mv "$ay.tmp" "$ay" + fi + + ( cd "$PROJECT_ROOT" && azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME "$MODEL" >/dev/null ) + INITED=1 +} + +# ========================= PHASE 1: native local =========================== +if [ "$DO_NATIVE" = 1 ]; then + [ "$PROTOCOL" = responses ] || { warn "phase 1 (native curl) supports 'responses'; sample is '$PROTOCOL' — skipping"; DO_NATIVE=0; } +fi +if [ "$DO_NATIVE" = 1 ]; then + log "=== Phase 1: native local run ===" + [ -n "$PROJECT_ENDPOINT" ] || die "native run needs --project-endpoint (FOUNDRY_PROJECT_ENDPOINT)" + + printf 'FOUNDRY_PROJECT_ENDPOINT="%s"\nAZURE_AI_MODEL_DEPLOYMENT_NAME="%s"\n' \ + "$PROJECT_ENDPOINT" "$MODEL" > "$SAMPLE_DIR/.env"; CREATED_ENV=1 + + log "creating venv + installing requirements (uv)" + uv venv "$SAMPLE_DIR/.venv" --python 3.12 >"$WORK_DIR/venv.log" 2>&1 || { cat "$WORK_DIR/venv.log" >&2; die "uv venv failed"; } + CREATED_VENV=1 + if [ -x "$SAMPLE_DIR/.venv/bin/python" ]; then VENV_PY="$SAMPLE_DIR/.venv/bin/python" + elif [ -x "$SAMPLE_DIR/.venv/Scripts/python.exe" ]; then VENV_PY="$SAMPLE_DIR/.venv/Scripts/python.exe" + else die "venv python not found under $SAMPLE_DIR/.venv"; fi + uv pip install --python "$VENV_PY" -r "$SAMPLE_DIR/requirements.txt" >"$WORK_DIR/pip.log" 2>&1 \ + || { tail -n 30 "$WORK_DIR/pip.log" >&2; die "uv pip install failed"; } + + log "starting server: python main.py (:$PORT)" + ( cd "$SAMPLE_DIR" && exec "$VENV_PY" main.py ) >"$WORK_DIR/native-server.log" 2>&1 & + NATIVE_PID=$! + wait_http "http://localhost:$PORT/responses" 90 || { tail -n 40 "$WORK_DIR/native-server.log" >&2; die "native server did not start on :$PORT"; } + + log "invoke: turn 1 (set name)" + b1="$(curl -sS -X POST "http://localhost:$PORT/responses" -H 'Content-Type: application/json' \ + -d '{"input":"My name is Tao. Please remember it."}')" + [ "$(printf '%s' "$b1" | jq -r '.status // empty')" = completed ] || die "native turn1 not completed: $b1" + rid="$(printf '%s' "$b1" | jq -r '.response_id // empty')" + [ -n "$rid" ] || die "native turn1 missing response_id" + + log "invoke: turn 2 (recall via previous_response_id)" + b2="$(curl -sS -X POST "http://localhost:$PORT/responses" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg id "$rid" '{input:"What is my name?", previous_response_id:$id}')")" + t2="$(printf '%s' "$b2" | jq -r '.output[0].content[0].text // empty')" + printf '%s' "$t2" | grep -qi 'Tao' || die "native multi-turn recall failed (got: '$t2')" + ok "native local: single + multi-turn (recall: '$t2')"; record pass "native local (python main.py)" + + stop_bg "$NATIVE_PID" "$PORT"; NATIVE_PID="" +else + log "skipping Phase 1 (native local)" +fi + +# ========================= PHASE 2: azd local ============================== +if [ "$DO_AZD_LOCAL" = 1 ]; then + log "=== Phase 2: local via 'azd ai agent run' ===" + ensure_azd_project + + log "starting: azd ai agent run --no-inspector (:$PORT)" + ( cd "$PROJECT_ROOT" && exec azd ai agent run --no-inspector --port "$PORT" ) >"$WORK_DIR/azd-run.log" 2>&1 & + AZD_RUN_PID=$! + wait_http "http://localhost:$PORT/responses" 240 || { tail -n 60 "$WORK_DIR/azd-run.log" >&2; die "azd ai agent run did not start on :$PORT"; } + + log "invoke --local: turn 1 (set name)" + r1="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke --local --protocol "$PROTOCOL" --output raw --new-session \ + "My name is Tao. Please remember it." ) 2>&1 || true )" + check_sse "$r1" || die "azd local turn1 failed" + log "invoke --local: turn 2 (recall, same session)" + r2="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke --local --protocol "$PROTOCOL" --output raw \ + "What is my name?" ) 2>&1 || true )" + check_sse "$r2" "Tao" || die "azd local multi-turn recall failed" + ok "azd local: single + multi-turn"; record pass "azd local (azd ai agent run)" + + stop_bg "$AZD_RUN_PID" "$PORT"; AZD_RUN_PID="" +else + log "skipping Phase 2 (azd local)" +fi + +# ========================= PHASE 3: deploy ================================= +if [ "$DO_DEPLOY" = 1 ]; then + log "=== Phase 3: deploy to Foundry ===" + [ -n "$ACR_ENDPOINT" ] || die "deploy needs --acr-endpoint (AZURE_CONTAINER_REGISTRY_ENDPOINT) to reuse an ACR" + ensure_azd_project + + ( cd "$PROJECT_ROOT" && azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT "$ACR_ENDPOINT" >/dev/null ) + + # Ensure a clean deploy: best-effort delete a pre-existing agent of this name. + log "removing any pre-existing agent named '$AGENT_NAME'" + NK="$(name_key "$AGENT_NAME")" + ( cd "$PROJECT_ROOT" && azd env set "$NK" "$AGENT_NAME" >/dev/null ) + ( cd "$PROJECT_ROOT" && azd ai agent delete "$AGENT_NAME" --force --no-prompt >/dev/null 2>&1 ) \ + && log "deleted pre-existing agent" || log "no pre-existing agent to delete" + + log "azd deploy (remote build via ACR; can take a few minutes)" + ( cd "$PROJECT_ROOT" && azd deploy ) >"$WORK_DIR/deploy.log" 2>&1 \ + || { tail -n 60 "$WORK_DIR/deploy.log" >&2; die "azd deploy failed"; } + DEPLOYED=1 + ok "azd deploy succeeded" + + log "verify deployed model env var" + shown="$( cd "$PROJECT_ROOT" && azd ai agent show "$AGENT_NAME" --output json 2>/dev/null || true )" + dep_model="$(printf '%s' "$shown" | jq -r '.definition.environment_variables.AZURE_AI_MODEL_DEPLOYMENT_NAME // empty')" + [ "$dep_model" = "$MODEL" ] || die "deployed model env var is '$dep_model', expected '$MODEL'" + status="$(printf '%s' "$shown" | jq -r '.status // empty')" + [ "$status" = active ] || warn "deployed agent status is '$status' (expected active)" + log "deployed model=$dep_model status=$status" + + log "invoke deployed: turn 1 (set name)" + d1="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke "$AGENT_NAME" --protocol "$PROTOCOL" --output raw --new-session \ + "My name is Tao. Please remember it." ) 2>&1 || true )" + check_sse "$d1" || die "deployed turn1 failed" + log "invoke deployed: turn 2 (recall, same session)" + d2="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke "$AGENT_NAME" --protocol "$PROTOCOL" --output raw \ + "What is my name?" ) 2>&1 || true )" + check_sse "$d2" "Tao" || die "deployed multi-turn recall failed" + ok "deployed: single + multi-turn"; record pass "deployed (azd deploy + invoke)" +else + log "skipping Phase 3 (deploy)" +fi + +# ============================= summary ===================================== +printf '\n===================== validation summary =====================\n' +printf '%s' "$SUMMARY" +printf '==============================================================\n' +printf 'passed: %d failed: %d\n' "$PASS_COUNT" "$FAIL_COUNT" +[ "$FAIL_COUNT" -eq 0 ] || exit 1 +log "all enabled checks passed" diff --git a/python/scripts/sample_validation/workflow.py b/python/scripts/sample_validation/workflow.py index 10187c069be..49ded23aa4a 100644 --- a/python/scripts/sample_validation/workflow.py +++ b/python/scripts/sample_validation/workflow.py @@ -12,6 +12,7 @@ CreateConcurrentValidationWorkflowExecutor, ) from sample_validation.discovery import DiscoverSamplesExecutor, ValidationConfig +from sample_validation.replay_executor import ReplayCachedPlaybooksExecutor from sample_validation.report import GenerateReportExecutor from sample_validation.run_dynamic_validation_workflow_executor import ( RunDynamicValidationWorkflowExecutor, @@ -31,13 +32,15 @@ def create_validation_workflow( Configured Workflow instance """ discover = DiscoverSamplesExecutor(config) + replay = ReplayCachedPlaybooksExecutor(config) create_dynamic_workflow = CreateConcurrentValidationWorkflowExecutor(config) run_dynamic_workflow = RunDynamicValidationWorkflowExecutor() generate = GenerateReportExecutor() return ( WorkflowBuilder(start_executor=discover) - .add_edge(discover, create_dynamic_workflow) + .add_edge(discover, replay) + .add_edge(replay, create_dynamic_workflow) .add_edge(create_dynamic_workflow, run_dynamic_workflow) .add_edge(run_dynamic_workflow, generate) .build()