From 9d98dbd38d93ca0ec4ecf41e49f228e8229d0132 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 25 Jun 2026 10:46:08 -0700 Subject: [PATCH 01/28] Add skill to replace hardcoded foundry project endpoint and model --- .../create_dynamic_workflow_executor.py | 10 +++- .../skills/foundry-config-setup/SKILL.md | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 6ebe25a8d4e..97b8524ec1e 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -3,10 +3,12 @@ import logging from collections import deque from dataclasses import dataclass +from pathlib import Path from agent_framework import ( Executor, Message, + SkillsProvider, Workflow, WorkflowBuilder, WorkflowContext, @@ -31,6 +33,9 @@ logger = logging.getLogger(__name__) +# Directory containing file-based skills used by the validation agents. +SKILLS_DIR = Path(__file__).parent / "skills" + class AgentResponseFormat(BaseModel): status: str @@ -58,7 +63,9 @@ class BatchCompletion: "Analyze the sample code and execute it as it is. Based on the execution result, determine " "if it runs successfully, fails, or is missing_setup. Use `missing_setup` if the sample reports " "missing required environment variables. The environment you're given should contain the necessary " - "variables. Don't create new environment variables nor modify the sample code.\n" + "variables. Don't create new environment variables nor modify the sample code, unless an available " + "skill instructs you to do so for the setup issue you detected. When a skill applies to the problem, " + "follow its guidance to resolve the setup and then re-run the sample.\n" "Feel free to install any required dependencies if needed.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " "based on your analysis of the code. You do not need to consult human on what to respond.\n" @@ -289,6 +296,7 @@ async def create( id=agent_id, name=agent_id, instructions=AgentInstruction, + context_providers=[SkillsProvider.from_paths(skill_paths=str(SKILLS_DIR))], default_options={ "on_permission_request": prompt_permission, "timeout": 120, diff --git a/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md new file mode 100644 index 00000000000..b8e3a8d85b0 --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md @@ -0,0 +1,53 @@ +--- +name: foundry-config-setup +description: Resolve missing setup caused by a hardcoded Foundry project endpoint or model in a sample. Use when a sample fails because it uses a placeholder/hardcoded project_endpoint (for example "https://your-project.services.ai.azure.com") or a hardcoded model instead of reading them from the environment. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +## Usage + +Some samples (notably those under `01-get-started`) hardcode the Foundry +project endpoint and model directly in the `FoundryChatClient` constructor +using placeholder values, for example: + +```python +client = FoundryChatClient( + project_endpoint="https://your-project.services.ai.azure.com", + model="gpt-4o", + credential=AzureCliCredential(), +) +``` + +These placeholder values are not real and the sample cannot run as written. +The validation environment provides the real values through environment +variables, so the sample must read them from the environment instead. + +When you detect a hardcoded/placeholder `project_endpoint` (or `model`) that +is causing the sample to fail with missing or invalid setup: + +1. Replace the hardcoded `project_endpoint` value with a read from the + `FOUNDRY_PROJECT_ENDPOINT` environment variable. +2. Replace the hardcoded `model` value with a read from the `FOUNDRY_MODEL` + environment variable. +3. Ensure `import os` is present at the top of the file. + +The corrected constructor should look like: + +```python +import os + +client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["FOUNDRY_MODEL"], + credential=AzureCliCredential(), +) +``` + +These samples are intentionally written with hardcoded placeholders, so this +is expected setup—not a defect in the sample. After applying the change, +re-run the sample and report the result as a `success` if it now runs. Do not +include a suggested `fix`. From ae13f79542c2ae8ac64683a53f24471f9c725704 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 25 Jun 2026 11:54:43 -0700 Subject: [PATCH 02/28] Include more samples and fix migration samples part 1 --- .../workflows/python-sample-validation.yml | 69 +++++++++---------- .../single_agent/01_basic_agent.py | 4 +- .../01_basic_chat_completion.py | 2 + .../02_chat_completion_with_tool.py | 2 + .../03_chat_completion_thread_and_stream.py | 2 + .../skills/hosting-sample-runner/SKILL.md | 50 ++++++++++++++ 6 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index bd76eb12d23..320f9c38b11 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -23,8 +23,8 @@ jobs: environment: integration env: # Required configuration for get-started samples - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -61,8 +61,8 @@ jobs: environment: integration env: # Foundry configuration - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} @@ -70,12 +70,12 @@ jobs: AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME || vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }} # OpenAI configuration - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # GitHub MCP GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # Observability ENABLE_INSTRUMENTATION: "true" defaults: @@ -122,10 +122,10 @@ jobs: runs-on: ubuntu-latest environment: integration env: - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python @@ -329,12 +329,11 @@ jobs: validate-02-agents-foundry: name: Validate 02-agents/providers/foundry - if: false # Temporarily disabled - provider folder also contains the local Foundry sample runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME || '' }} FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION || '' }} defaults: @@ -445,8 +444,8 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -479,12 +478,11 @@ jobs: validate-04-hosting: name: Validate 04-hosting - if: false # Temporarily disabled because of sample complexity runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # A2A configuration A2A_AGENT_HOST: http://localhost:5001/ defaults: @@ -518,8 +516,8 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} @@ -560,16 +558,16 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # OpenAI configuration - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python @@ -610,20 +608,21 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration for AF AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration for SK AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} # OpenAI key - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # OpenAI configuration for SK - OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} + OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }} # Copilot Studio COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }} COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }} diff --git a/python/samples/autogen-migration/single_agent/01_basic_agent.py b/python/samples/autogen-migration/single_agent/01_basic_agent.py index bfa0c915eae..a9bcb3da359 100644 --- a/python/samples/autogen-migration/single_agent/01_basic_agent.py +++ b/python/samples/autogen-migration/single_agent/01_basic_agent.py @@ -1,12 +1,14 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", # "autogen-agentchat", # "autogen-ext[openai]", +# "python-dotenv", # ] # /// # Run with any PEP 723 compatible runner, e.g.: -# uv run samples/autogen-migration/single_agent/01_basic_assistant_agent.py +# uv run samples/autogen-migration/single_agent/01_basic_agent.py # Copyright (c) Microsoft. All rights reserved. diff --git a/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py b/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py index ebd10122dc3..d4493b22a91 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py +++ b/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py b/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py index d5b12035184..21397b62c9a 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py +++ b/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py b/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py index f656220f204..b6fe966fe67 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py +++ b/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md b/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md new file mode 100644 index 00000000000..58412c6450a --- /dev/null +++ b/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md @@ -0,0 +1,50 @@ +--- +name: hosting-sample-runner +description: Decide how to validate a hosting sample (for example anything under 04-hosting that starts a server, function host, or deployed agent). Use when a sample hosts an agent or workflow and is run by starting a local server/host and calling it, or by deploying to a cloud provider. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +## Usage + +Hosting samples are different from ordinary scripts: instead of running to +completion, they typically start a server, function host, or hosted agent and +are exercised by a separate client call. Each hosting sample includes a +`README.md` that documents how to set up and run it. + +When validating a hosting sample: + +1. Read the sample's `README.md` (and any sibling READMEs in parent + directories) to understand how the sample is meant to be run. +2. Decide whether the sample can be run **locally** — that is, fully exercised + on this machine without deploying to a cloud provider (for example Azure + Functions deployment, an Azure Container App, or a Foundry hosted-agent + publish step). A sample is locally runnable when its README describes a + local launch path, such as starting a local server (e.g. Hypercorn, + `uv run python app.py`, the Functions Core Tools `func start`, or a durable + task worker) and then calling it from a local client/HTTP request. + +### If the sample can be run locally + +Follow the README's local setup and run instructions: + +1. Install any required dependencies it lists. +2. Start the host process in the background (it will not exit on its own). +3. Exercise it as the README describes — run the companion client script, + send the documented HTTP request, or otherwise drive a single end-to-end + interaction. +4. If the interaction succeeds, stop the host process and mark the sample as + `success`. +5. If the host fails to start or the interaction errors, treat it as a + `failure` and investigate the error. + +### If the sample cannot be run locally + +If the README only documents a cloud deployment path (for example deploying +to Azure Functions, publishing a Foundry hosted agent, or otherwise requiring +provisioned cloud infrastructure to exercise the sample), do not attempt to +deploy it. Mark the sample as `missing_setup` and note in the output that it +requires cloud deployment that cannot be performed locally. From 0ffeb715cb767df0d1206e4bac4b309cb4820ddd Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 25 Jun 2026 13:32:38 -0700 Subject: [PATCH 03/28] Fix migration samples --- .github/workflows/python-sample-validation.yml | 2 +- .../copilot_studio/01_basic_copilot_studio_agent.py | 11 +++++++++++ .../copilot_studio/02_copilot_studio_streaming.py | 11 +++++++++++ .../openai_responses/01_basic_responses_agent.py | 2 ++ .../openai_responses/02_responses_agent_with_tool.py | 2 ++ .../03_responses_agent_structured_output.py | 2 ++ .../orchestrations/concurrent_basic.py | 3 +++ .../orchestrations/group_chat.py | 3 +++ .../orchestrations/handoff.py | 3 +++ .../orchestrations/magentic.py | 3 +++ .../orchestrations/sequential.py | 3 +++ .../processes/fan_out_fan_in_process.py | 2 ++ .../processes/nested_process.py | 2 ++ 13 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 320f9c38b11..116c1595b7d 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -9,7 +9,7 @@ env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: claude-opus-4.6 + GITHUB_COPILOT_MODEL: claude-opus-4.8 COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} permissions: diff --git a/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py b/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py index a477181b264..240a03440dd 100644 --- a/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py +++ b/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py @@ -1,11 +1,22 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-copilotstudio", +# "python-dotenv", # "semantic-kernel", # ] # /// # Run with any PEP 723 compatible runner, e.g.: # uv run samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py +# +# NOTE: The metadata above resolves the Agent Framework half only. +# The Semantic Kernel half (run_semantic_kernel) requires the older +# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and +# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while +# Agent Framework requires the newer underscore-namespace SDK +# (microsoft_agents.copilotstudio.client, from +# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be +# installed in the same environment, so run each half in its own isolated env. # Copyright (c) Microsoft. All rights reserved. """Call a Copilot Studio agent with SK and Agent Framework.""" diff --git a/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py b/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py index 97ef158c532..991b4b5bf1c 100644 --- a/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py +++ b/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py @@ -1,11 +1,22 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-copilotstudio", +# "python-dotenv", # "semantic-kernel", # ] # /// # Run with any PEP 723 compatible runner, e.g.: # uv run samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py +# +# NOTE: The metadata above resolves the Agent Framework half only. +# The Semantic Kernel half (run_semantic_kernel) requires the older +# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and +# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while +# Agent Framework requires the newer underscore-namespace SDK +# (microsoft_agents.copilotstudio.client, from +# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be +# installed in the same environment, so run each half in its own isolated env. # Copyright (c) Microsoft. All rights reserved. """Stream responses from Copilot Studio agents in SK and AF.""" diff --git a/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py b/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py index d74487d1e8f..b360da34e8f 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py +++ b/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py b/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py index 01b783aff90..09b68fb7d00 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py +++ b/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py b/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py index cbfbf470a08..04fad5754ce 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py +++ b/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py b/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py index 11140aa8752..1a9470b7135 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py +++ b/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py index 89613072d8c..8c64bcb4c13 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py +++ b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/handoff.py b/python/samples/semantic-kernel-migration/orchestrations/handoff.py index 5313c2943f8..08f77400611 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/handoff.py +++ b/python/samples/semantic-kernel-migration/orchestrations/handoff.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/magentic.py b/python/samples/semantic-kernel-migration/orchestrations/magentic.py index 4ce62492e21..5e03f0a3037 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/magentic.py +++ b/python/samples/semantic-kernel-migration/orchestrations/magentic.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/sequential.py b/python/samples/semantic-kernel-migration/orchestrations/sequential.py index 22a2be6f23e..c40792561d2 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/sequential.py +++ b/python/samples/semantic-kernel-migration/orchestrations/sequential.py @@ -1,6 +1,9 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-openai", +# "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py b/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py index 37e210e80b8..4463de36c40 100644 --- a/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py +++ b/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-core", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/processes/nested_process.py b/python/samples/semantic-kernel-migration/processes/nested_process.py index ee8d889229a..61a9aef7ee6 100644 --- a/python/samples/semantic-kernel-migration/processes/nested_process.py +++ b/python/samples/semantic-kernel-migration/processes/nested_process.py @@ -1,6 +1,8 @@ # /// script # requires-python = ">=3.10" # dependencies = [ +# "agent-framework-core", +# "python-dotenv", # "semantic-kernel", # ] # /// From 02611676d625a4a823f1d78035775f58270dbd95 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 7 Jul 2026 14:06:19 -0700 Subject: [PATCH 04/28] Replace Foundry hosted agent validation skill --- .../workflows/python-sample-validation.yml | 4 + .../create_dynamic_workflow_executor.py | 8 +- python/scripts/sample_validation/models.py | 2 - ...un_dynamic_validation_workflow_executor.py | 7 +- .../skills/foundry-config-setup/SKILL.md | 3 +- .../foundry-hosted-agent-validation/SKILL.md | 309 ++++++++++++++ .../scripts/validate_hosted_agent.sh | 389 ++++++++++++++++++ .../skills/hosting-sample-runner/SKILL.md | 50 --- 8 files changed, 708 insertions(+), 64 deletions(-) create mode 100644 python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md create mode 100755 python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh delete mode 100644 python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 116c1595b7d..531d96800a8 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -483,6 +483,10 @@ jobs: env: FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # Foundry hosted agent configuration + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL }} + FOUNDRY_PROJECT_ID: ${{ vars.FOUNDRY_PROJECT_ID }} + AZURE_CONTAINER_REGISTRY_ENDPOINT: ${{ vars.AZURE_CONTAINER_REGISTRY_ENDPOINT }} # A2A configuration A2A_AGENT_HOST: http://localhost:5001/ defaults: diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 97b8524ec1e..3a2a4e3a318 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -68,14 +68,13 @@ class BatchCompletion: "follow its guidance to resolve the setup and then re-run the sample.\n" "Feel free to install any required dependencies if needed.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " - "based on your analysis of the code. You do not need to consult human on what to respond.\n" - "If the sample fails, investigate the error and suggest a fix.\n" + "based on your analysis of the code. You do not need to consult human on what to respond.\n" \ + "Fail fast and do not attempt to fix the sample unless instructed by a skill.\n" "Return ONLY valid JSON with this schema:\n" "{\n" ' "status": "success|failure|missing_setup",\n' ' "output": "short summary of the result and what you did if the sample was interactive",\n' ' "error": "error details or empty string",\n' - ' "fix": "suggested code fix if the sample failed, otherwise empty string"\n' "}\n\n" ) @@ -151,7 +150,6 @@ async def handle_task( status=status_from_text(result_payload.status), output=result_payload.output, error=result_payload.error, - fix=result_payload.fix, ) break except Exception as ex: @@ -174,7 +172,6 @@ async def handle_task( status=RunStatus.FAILURE, output="", error=f"Original error: {ex}. Restart error: {restart_ex}", - fix="", ) break @@ -184,7 +181,6 @@ async def handle_task( status=RunStatus.FAILURE, output="", error=str(ex), - fix="", ) break diff --git a/python/scripts/sample_validation/models.py b/python/scripts/sample_validation/models.py index ff45b5909ba..e41de324eba 100644 --- a/python/scripts/sample_validation/models.py +++ b/python/scripts/sample_validation/models.py @@ -72,7 +72,6 @@ class RunResult: status: RunStatus output: str error: str - fix: str @dataclass @@ -154,7 +153,6 @@ def to_dict(self) -> dict[str, object]: "status": r.status.value, "output": r.output, "error": r.error, - "fix": r.fix, } for r in self.results ], diff --git a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py index c7244cff2a2..a895e5ddc83 100644 --- a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py +++ b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py @@ -50,10 +50,10 @@ async def run( CoordinatorStart(samples=creation.samples), stream=True ): if event.type == "output" and isinstance(event.data, ExecutionResult): - result = event.data # type: ignore - elif event.type == WORKER_COMPLETED and isinstance( + result = event.data + elif event.type == WORKER_COMPLETED and isinstance( # type: ignore event.data, SampleInfo - ): # type: ignore + ): remaining_sample_counts -= 1 print( f"Completed validation for sample: {event.data.relative_path:<80} | " @@ -69,7 +69,6 @@ async def run( status=RunStatus.FAILURE, output="", error="Nested workflow did not return an ExecutionResult.", - fix="", ) for sample in creation.samples ] diff --git a/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md index b8e3a8d85b0..eb4e75746e0 100644 --- a/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md +++ b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md @@ -49,5 +49,4 @@ client = FoundryChatClient( These samples are intentionally written with hardcoded placeholders, so this is expected setup—not a defect in the sample. After applying the change, -re-run the sample and report the result as a `success` if it now runs. Do not -include a suggested `fix`. +re-run the sample and report the result as a `success` if it now runs. diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md new file mode 100644 index 00000000000..ed70c965e9b --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -0,0 +1,309 @@ +--- +name: foundry-hosted-agent-validation +description: > + Step-by-step process for validating a Python Foundry hosted agent sample + (under python/samples/04-hosting/foundry-hosted-agents/) end to end — running + it locally (native runtime and `azd ai agent run`) and after deploying it to + an Azure AI Foundry project with `azd`. Use this when asked to validate, smoke + test, or verify a hosted agent sample works locally and/or deployed, or when + deploying one of these samples to Foundry. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +# Validating a Foundry Hosted Agent Sample + +A hosted agent sample is "validated" when it passes **three** independent +checks, plus cleanup: + +1. **Local, native runtime** — run the sample's own entry point + (`python main.py`) and invoke it over HTTP. +2. **Local, via `azd ai agent run`** — the `azd` local dev loop. +3. **Deployed** — `azd deploy` to Foundry, then invoke the hosted agent. + +Each check must succeed for **single-turn** and **multi-turn** (session / +`previous_response_id`) conversation. Always end with **cleanup** (delete the +deployed agent, remove the temp `azd` project, restore the sample dir). + +> Read the sample's own `README.md` and the parent +> `.../foundry-hosted-agents/README.md` first — they define the run/deploy +> commands and any sample-specific payload. This skill captures the process and +> the non-obvious gotchas the READMEs don't. + +--- + +## Automated script + +[`scripts/validate_hosted_agent.sh`](scripts/validate_hosted_agent.sh) runs all +three phases (and cleanup) non-interactively — use it for a full pass, and read +the phases below to interpret failures or validate a non-`responses` sample by +hand. Run `--help` for the full dependency list and options. + +```bash +# Full validation of the responses/01_basic sample (default sample dir): +python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh \ + --project-endpoint "https://.services.ai.azure.com/api/projects/" \ + --model "" \ + --acr-endpoint "" + +# Local-only (skip deploy), or point at another sample: +python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh --skip-deploy \ + --sample-dir python/samples/04-hosting/foundry-hosted-agents/responses/02_tools \ + --project-endpoint "..." --model "..." +``` + +Inputs may also come from env vars (`FOUNDRY_PROJECT_ENDPOINT`, +`AZURE_AI_MODEL_DEPLOYMENT_NAME`, `FOUNDRY_PROJECT_ID`, +`AZURE_CONTAINER_REGISTRY_ENDPOINT`). Phase flags: `--skip-native`, +`--skip-azd-local`, `--skip-deploy`; `--no-cleanup`/`--keep-agent` to inspect +afterward. The script encodes every gotcha below (model template fix, +pre-existing-agent removal, ACR reuse, port/temp cleanup). + +--- + +## Inputs you need before starting + +Gather these (ask the user if not provided): + +- **Foundry project endpoint**, e.g. + `https://.services.ai.azure.com/api/projects/`. +- **Foundry project resource id** (for non-interactive `azd ai agent init`): + `/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects/`. + Find it with `az cognitiveservices account list` + the project name. +- **A real, deployed model name** in that project (e.g. `gpt-4.1-mini`). This is + often **different** from the model id in `agent.manifest.yaml` — the actual + deployment name wins. +- **An existing ACR to reuse** for deployment (login server, e.g. + `myacr.azurecr.io`). Reusing one avoids `azd provision` creating resources. +- Whether a like-named agent **already exists** in the project (remove it first + for a clean validation — see below). + +## Tooling / auth + +- `az` (logged in: `az login`) and `azd` (logged in: `azd auth login`). +- `azd` **agents extension**: `azd extension list` should show + `azure.ai.agents`; install with `azd extension install azure.ai.agents`. +- `uv` for the native-Python local run. **`python` need not be on PATH** — `uv` + and `azd ai agent run` provision their own interpreter. +- Docker is **not** required when you reuse an ACR (`remoteBuild: true` builds + in ACR Tasks). + +--- + +## Phase 0 — Understand the sample + +A responses/invocations sample folder typically contains: +`main.py` (entry point + `ResponsesHostServer`/`InvocationsHostServer`), +`agent.manifest.yaml` (used by `azd ai agent init`), `agent.yaml` (the deployed +agent definition), `requirements.txt`, `Dockerfile`, `.env.example`. + +Note the **protocol** (`responses` or `invocations`) from `agent.yaml` / +manifest — it changes the invoke command (`--protocol invocations`) and the HTTP +path (`/responses` vs the invocations route). + +--- + +## Phase 1 — Local validation, native runtime (Python) + +Run from the sample directory. + +```bash +uv venv .venv --python 3.12 # 3.12 matches the sample Dockerfile +uv pip install --python .venv/... -r requirements.txt +``` + +Create `.env` from `.env.example` with the **real** values: + +``` +FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +AZURE_AI_MODEL_DEPLOYMENT_NAME="" +``` + +Start the server (`python main.py`) — it listens on `http://localhost:8088`. +`main.py` uses `DefaultAzureCredential`, so `az login` must be current. + +Invoke (single turn), capture the returned `response_id`, then reuse it for a +follow-up turn to confirm memory: + +```bash +curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" \ + -d '{"input": "My name is Tao. Remember it."}' +# take response_id from the JSON, then: +curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" \ + -d '{"input": "What is my name?", "previous_response_id": ""}' +``` + +PowerShell: use `Invoke-WebRequest -Uri http://localhost:8088/responses -Method POST -ContentType application/json -Body '...'`. + +**Pass:** HTTP 200, non-empty `output[].content[].text`, and the second turn +recalls the name. Stop the server afterward. + +--- + +## Phase 2 — Local validation via `azd ai agent run` + +### Init the azd project (once) + +Run in an **empty temp directory outside the repo** (short path avoids Windows +path-length issues, e.g. `C:\afval\`). Point `-m` at the **local** +manifest so it validates the working-tree sample: + +```bash +azd ai agent init -m /agent.manifest.yaml \ + --project-id "" \ + --model-deployment "" \ + --agent-name "" \ + --no-prompt --force +``` + +`init` downloads the template into a **subfolder named after the agent**, so the +azd project root is `//`. `cd` there for all later `azd` +commands. + +> **Before init, remove any `.venv` you created in the sample dir** — `init` +> copies the entire manifest directory into `src/`. (`.venv` is excluded from +> deploy packaging by `.agentignore`/`.dockerignore`, so it is harmless but +> bloats/slows the copy.) + +### Fix the model deployment name (critical — see Gotcha 1) + +```bash +azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME "" +``` + +### Run and invoke locally + +```bash +azd ai agent run --no-inspector # auto-creates a uv venv + installs deps; listens on :8088 +azd ai agent invoke --local --new-session "My name is Tao. Remember it." +azd ai agent invoke --local "What is my name?" # same session is reused automatically +``` + +**Pass:** both invokes return text; the second recalls the name (same +`Session:` id). Stop the `run` process afterward. + +--- + +## Removing a pre-existing agent (do this before deploying) + +`init` prints a warning if the agent name already exists in the project. To +delete it, note that **`azd ai agent delete`/`show` resolve the deployed agent +name from an azd env var, not from the positional argument.** The var is +`AGENT_{SERVICEKEY}_NAME`, where `SERVICEKEY` = the `azure.yaml` service name +uppercased with `-`/spaces → `_`. + +Example for service `agent-framework-agent-basic-responses`: + +```bash +azd env set AGENT_AGENT_FRAMEWORK_AGENT_BASIC_RESPONSES_NAME agent-framework-agent-basic-responses +azd ai agent delete --force --no-prompt --output json +# -> {"object":"agent.deleted","name":"...","deleted":true} +``` + +(After a successful `azd deploy`, this var is set automatically, so later +`show`/`delete`/`invoke` work without setting it.) + +--- + +## Phase 3 — Deploy and validate + +### Reuse an existing ACR (avoid provisioning) + +For an existing project + model, **do not run `azd provision`/`azd up`** — the +generated `azure.yaml` has a `deployments` block for the manifest's model +(often an auto-selected `GlobalProvisionedManaged` PTU SKU) that provision would +try to create (costly / quota failures). Instead reuse an ACR: + +```bash +azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT # e.g. myacr.azurecr.io +azd deploy +``` + +`azd deploy` fails with _"could not determine container registry endpoint"_ if +this is unset and no ACR is provisioned. + +### Verify the deployed model env var, then invoke + +```bash +azd ai agent show --output json # check definition.environment_variables.AZURE_AI_MODEL_DEPLOYMENT_NAME +azd ai agent invoke --new-session "My name is Tao. Remember it." +azd ai agent invoke "What is my name?" +``` + +Use `--output raw` on invoke to see raw SSE events and any failure, e.g.: + +``` +event: response.failed +... "code": "DeploymentNotFound" ... 404 ... +``` + +`DeploymentNotFound` means the deployed `AZURE_AI_MODEL_DEPLOYMENT_NAME` points +at a model that isn't deployed → fix per Gotcha 1 and redeploy (creates a new +version). + +**Pass:** agent reaches `status: active`, invoke returns text (not empty, no +`response.failed`), and multi-turn recalls the name. + +--- + +## Cleanup (always) + +- Delete the deployed agent: `azd ai agent delete --force --no-prompt`. +- Delete the temp `azd` project directory. +- Remove `.env`/`.venv` you created in the sample dir; confirm the sample dir is + pristine (`git status --porcelain ` is empty — `.env`/`.venv` are + gitignored). +- Stop any leftover local server still holding port 8088: + `Get-NetTCPConnection -LocalPort 8088 -State Listen` → `Stop-Process -Id ` + (Linux/macOS: `lsof -ti:8088 | xargs kill`). Stopping the shell may leave the + child interpreter running. + +--- + +## Gotchas (the parts that waste the most time) + +1. **The deployed model name comes from `agent.yaml`, not the azd env.** + `azd ai agent init` ignores `--model-deployment` in `--no-prompt` mode and + writes the **manifest's** model id (e.g. `gpt-4.1-mini`) as a **literal** into + both the azd env `AZURE_AI_MODEL_DEPLOYMENT_NAME` and the generated + `src//agent.yaml` env var. Local runs read the azd env (so + `azd env set` fixes them), but **deployment injects `agent.yaml`'s value**. + Fix by setting the generated `agent.yaml` env var to the template + `value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME}` (what the repo sample already uses; + `init` flattens it) **and** `azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME `, + then redeploy. A literal `value: ` also works. + +2. **`azd ai agent delete/show` need the `AGENT_{SERVICEKEY}_NAME` env var** — a + bare positional agent name is treated as the _service_ name and the deployed + agent name is looked up from that env var (see "Removing a pre-existing + agent"). + +3. **`azd provision`/`azd up` will try to create the manifest's model + deployment** (from `azure.yaml`'s `deployments` block). Prefer `azd deploy` + with a reused ACR when the project + model already exist. + +4. **`python` on PATH is not required.** `uv venv` and `azd ai agent run` + provision their own interpreter and install `requirements.txt`. + +5. **`init` copies the whole manifest directory into `src/`.** Remove a local + `.venv` from the sample dir first to keep the copy clean/fast. + +6. **Port 8088 can stay bound after stopping the shell** — kill the interpreter + by PID (see Cleanup). + +--- + +## Success checklist + +- [ ] Native local run: 200 + non-empty text + multi-turn recall. +- [ ] `azd ai agent run` local: text returned + session reused across invokes. +- [ ] Pre-existing agent removed (if any). +- [ ] `azd deploy` succeeds; agent `status: active`. +- [ ] `azd ai agent show` confirms `AZURE_AI_MODEL_DEPLOYMENT_NAME` = the real + deployed model. +- [ ] Deployed invoke: text returned (no `response.failed`) + multi-turn recall. +- [ ] Cleanup done: agent deleted, temp project removed, sample dir pristine, + port 8088 free. diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh new file mode 100755 index 00000000000..23e385ffc9a --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh @@ -0,0 +1,389 @@ +#!/usr/bin/env bash +# +# validate_hosted_agent.sh — automate validation of a Foundry hosted agent sample. +# +# Runs the three validation checks from the `foundry-hosted-agent-validation` +# skill and cleans up afterward: +# 1. Native local run (uv venv + `python main.py` + curl; responses protocol) +# 2. Local via azd (`azd ai agent init` -> `azd ai agent run` -> invoke) +# 3. Deployed (`azd deploy` to Foundry -> `azd ai agent invoke`) +# Each check exercises a single-turn and a multi-turn (memory) exchange. +# +# --------------------------------------------------------------------------- +# DEPENDENCIES (must be installed and on PATH in the runner / pipeline) +# --------------------------------------------------------------------------- +# bash >= 3.2 (no bash-4-only features are used) +# az Azure CLI, logged in (`az login`) — used to derive the project +# id when --project-id is omitted, and by DefaultAzureCredential. +# azd Azure Developer CLI, logged in (`azd auth login`), WITH the +# agents extension installed: +# azd extension install azure.ai.agents +# uv https://docs.astral.sh/uv/ — creates the native venv & installs +# requirements. (Note: a `python` interpreter on PATH is NOT +# required; uv and `azd ai agent run` provision their own.) +# curl native HTTP invocation of the local server. +# jq JSON parsing of responses / `azd ai agent show` output. +# coreutils awk, sed, grep, tr, printf, mktemp, sleep (standard on Linux/macOS). +# +# Optional but recommended: +# lsof OR fuser reliable freeing of the local port during cleanup. Without +# them, a child server process may linger on the port. +# Docker NOT needed when reusing an ACR (remote build runs in ACR +# Tasks). Only needed if you switch to a local docker build. +# +# --------------------------------------------------------------------------- +# REQUIRED INPUTS (flags or environment variables) +# --------------------------------------------------------------------------- +# --project-endpoint | FOUNDRY_PROJECT_ENDPOINT (native + all phases) +# --model | AZURE_AI_MODEL_DEPLOYMENT_NAME (a REAL deployed model) +# --project-id | FOUNDRY_PROJECT_ID (azd-local + deploy) +# --acr-endpoint | AZURE_CONTAINER_REGISTRY_ENDPOINT (deploy only) +# +# If --project-id is omitted it is derived from the endpoint via `az`. +# +# --------------------------------------------------------------------------- +# USAGE +# --------------------------------------------------------------------------- +# validate_hosted_agent.sh [options] +# +# --sample-dir DIR Sample folder (default: the responses/01_basic sample +# resolved relative to this script's repo). +# --project-endpoint URL Foundry project endpoint. +# --project-id ID Foundry project ARM resource id. +# --model NAME Real model deployment name in the project. +# --acr-endpoint HOST Existing ACR login server (e.g. myacr.azurecr.io). +# --agent-name NAME Override agent/service name (default: read agent.yaml). +# --port N Local port (default: 8088). +# --skip-native Skip phase 1 (native local run). +# --skip-azd-local Skip phase 2 (azd ai agent run). +# --skip-deploy Skip phase 3 (deploy to Foundry). +# --keep-agent Do NOT delete the deployed agent during cleanup. +# --no-cleanup Keep temp azd project, sample .env/.venv, and agent. +# -h | --help Show this help. +# +# EXIT CODES: 0 = all enabled checks passed; non-zero = a check or setup failed. +# +set -euo pipefail + +# --------------------------- pretty logging -------------------------------- +_c() { [ -t 1 ] && printf '%s' "$1" || printf ''; } +log() { printf '%s[validate]%s %s\n' "$(_c $'\033[1;34m')" "$(_c $'\033[0m')" "$*"; } +ok() { printf '%s[ pass ]%s %s\n' "$(_c $'\033[1;32m')" "$(_c $'\033[0m')" "$*"; } +warn() { printf '%s[ warn ]%s %s\n' "$(_c $'\033[1;33m')" "$(_c $'\033[0m')" "$*" >&2; } +die() { printf '%s[ FAIL ]%s %s\n' "$(_c $'\033[1;31m')" "$(_c $'\033[0m')" "$*" >&2; exit 1; } + +# --------------------------- defaults / args ------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# scripts -> foundry-hosted-agent-validation -> skills -> sample_validation -> scripts -> python +PYTHON_ROOT="$(cd "$SCRIPT_DIR/../../../../.." && pwd)" +DEFAULT_SAMPLE="$PYTHON_ROOT/samples/04-hosting/foundry-hosted-agents/responses/01_basic" + +SAMPLE_DIR="${DEFAULT_SAMPLE}" +PROJECT_ENDPOINT="${FOUNDRY_PROJECT_ENDPOINT:-}" +PROJECT_ID="${FOUNDRY_PROJECT_ID:-}" +MODEL="${AZURE_AI_MODEL_DEPLOYMENT_NAME:-}" +ACR_ENDPOINT="${AZURE_CONTAINER_REGISTRY_ENDPOINT:-}" +AGENT_NAME="" +PORT=8088 +DO_NATIVE=1; DO_AZD_LOCAL=1; DO_DEPLOY=1 +KEEP_AGENT=0; DO_CLEANUP=1 + +usage() { sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//; s/^#//' | sed '$d'; } + +while [ $# -gt 0 ]; do + case "$1" in + --sample-dir) SAMPLE_DIR="$2"; shift 2;; + --project-endpoint) PROJECT_ENDPOINT="$2"; shift 2;; + --project-id) PROJECT_ID="$2"; shift 2;; + --model) MODEL="$2"; shift 2;; + --acr-endpoint) ACR_ENDPOINT="$2"; shift 2;; + --agent-name) AGENT_NAME="$2"; shift 2;; + --port) PORT="$2"; shift 2;; + --skip-native) DO_NATIVE=0; shift;; + --skip-azd-local) DO_AZD_LOCAL=0; shift;; + --skip-deploy) DO_DEPLOY=0; shift;; + --keep-agent) KEEP_AGENT=1; shift;; + --no-cleanup) DO_CLEANUP=0; KEEP_AGENT=1; shift;; + -h|--help) usage; exit 0;; + *) die "unknown argument: $1 (use --help)";; + esac +done + +# --------------------------- state / cleanup ------------------------------- +WORK_DIR=""; PROJECT_ROOT=""; NATIVE_PID=""; AZD_RUN_PID="" +DEPLOYED=0; CREATED_ENV=0; CREATED_VENV=0 +PASS_COUNT=0; FAIL_COUNT=0; SUMMARY="" + +record() { # $1=pass|fail $2=text + if [ "$1" = pass ]; then PASS_COUNT=$((PASS_COUNT+1)); SUMMARY="${SUMMARY} [pass] $2"$'\n'; + else FAIL_COUNT=$((FAIL_COUNT+1)); SUMMARY="${SUMMARY} [FAIL] $2"$'\n'; fi +} + +free_port() { # $1=port + local port="$1" pids + if command -v lsof >/dev/null 2>&1; then + pids="$(lsof -ti "tcp:${port}" 2>/dev/null || true)" + [ -n "$pids" ] && kill -9 $pids 2>/dev/null || true + elif command -v fuser >/dev/null 2>&1; then + fuser -k "${port}/tcp" 2>/dev/null || true + fi +} + +stop_bg() { # $1=pid $2=port + local pid="${1:-}" port="${2:-}" + if [ -n "$pid" ]; then kill "$pid" 2>/dev/null || true; sleep 1; kill -9 "$pid" 2>/dev/null || true; fi + [ -n "$port" ] && free_port "$port" +} + +cleanup() { + local ec=$? + log "cleanup..." + stop_bg "$NATIVE_PID" "$PORT" + stop_bg "$AZD_RUN_PID" "$PORT" + if [ "$DO_CLEANUP" = 1 ]; then + if [ "$DEPLOYED" = 1 ] && [ "$KEEP_AGENT" = 0 ] && [ -n "$PROJECT_ROOT" ] && [ -d "$PROJECT_ROOT" ]; then + log "deleting deployed agent '$AGENT_NAME'" + ( cd "$PROJECT_ROOT" && azd ai agent delete "$AGENT_NAME" --force --no-prompt >/dev/null 2>&1 ) || \ + warn "could not delete deployed agent (delete it manually)" + fi + [ "$CREATED_ENV" = 1 ] && rm -f "$SAMPLE_DIR/.env" 2>/dev/null || true + [ "$CREATED_VENV" = 1 ] && rm -rf "$SAMPLE_DIR/.venv" 2>/dev/null || true + [ -n "$WORK_DIR" ] && rm -rf "$WORK_DIR" 2>/dev/null || true + else + warn "cleanup skipped (--no-cleanup). Temp project: ${WORK_DIR:-n/a}" + fi + if [ "$ec" -ne 0 ]; then printf '\n'; die "aborted (exit $ec)"; fi +} +trap cleanup EXIT INT TERM + +# --------------------------- preflight ------------------------------------- +require_cmd() { command -v "$1" >/dev/null 2>&1 || die "missing dependency '$1' — $2"; } + +log "preflight: checking dependencies" +require_cmd az "install Azure CLI: https://aka.ms/azcli" +require_cmd azd "install Azure Developer CLI: https://aka.ms/azd" +require_cmd uv "install uv: https://docs.astral.sh/uv/" +require_cmd curl "install curl" +require_cmd jq "install jq: https://jqlang.github.io/jq/" +require_cmd awk "install coreutils (awk)" +require_cmd mktemp "install coreutils (mktemp)" + +az account show >/dev/null 2>&1 || die "not logged in to Azure CLI — run: az login" +azd auth login --check-status >/dev/null 2>&1 || die "not logged in to azd — run: azd auth login" +azd extension list 2>/dev/null | grep -qi 'azure.ai.agents' || \ + die "azd agents extension missing — run: azd extension install azure.ai.agents" +command -v lsof >/dev/null 2>&1 || command -v fuser >/dev/null 2>&1 || \ + warn "neither lsof nor fuser found; port ${PORT} may not be freed cleanly on exit" + +[ -d "$SAMPLE_DIR" ] || die "sample dir not found: $SAMPLE_DIR" +[ -f "$SAMPLE_DIR/main.py" ] || die "no main.py in sample dir: $SAMPLE_DIR" +[ -f "$SAMPLE_DIR/agent.yaml" ] || die "no agent.yaml in sample dir: $SAMPLE_DIR" +[ -n "$MODEL" ] || die "model deployment name required (--model or AZURE_AI_MODEL_DEPLOYMENT_NAME)" + +# derive agent name + protocol from agent.yaml +if [ -z "$AGENT_NAME" ]; then + AGENT_NAME="$(grep -E '^name:' "$SAMPLE_DIR/agent.yaml" | head -1 | sed -E 's/^name:[[:space:]]*//' | tr -d '"'\''\r')" +fi +[ -n "$AGENT_NAME" ] || die "could not determine agent name (pass --agent-name)" +PROTOCOL="$(grep -oE 'protocol:[[:space:]]*[a-zA-Z]+' "$SAMPLE_DIR/agent.yaml" | head -1 | sed -E 's/protocol:[[:space:]]*//')" +[ -n "$PROTOCOL" ] || PROTOCOL=responses + +WORK_DIR="$(mktemp -d 2>/dev/null || mktemp -d -t afval)" +log "sample=$SAMPLE_DIR" +log "agent=$AGENT_NAME protocol=$PROTOCOL model=$MODEL port=$PORT" +log "workdir=$WORK_DIR" + +# --------------------------- helpers --------------------------------------- +wait_http() { # $1=url $2=timeout_s + local url="$1" timeout="${2:-90}" i=0 code + while :; do + code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 3 "$url" 2>/dev/null || echo 000)" + [ "$code" != "000" ] && return 0 + i=$((i+1)); [ "$i" -ge "$timeout" ] && return 1; sleep 1 + done +} + +# assert an `azd ai agent invoke --output raw` SSE stream succeeded. +# $1 = raw output ; $2 = optional expected substring (case-insensitive) +check_sse() { + local raw="$1" expect="${2:-}" + printf '%s' "$raw" | grep -qi 'DeploymentNotFound' && { warn "DeploymentNotFound"; return 2; } + printf '%s' "$raw" | grep -q 'response.failed' && { warn "response.failed"; return 2; } + printf '%s' "$raw" | grep -q 'response.completed' || { warn "no response.completed"; return 3; } + if [ -n "$expect" ]; then printf '%s' "$raw" | grep -qi "$expect" || { warn "missing '$expect'"; return 4; }; fi + return 0 +} + +resolve_project_id() { + [ -n "$PROJECT_ID" ] && return 0 + [ -n "$PROJECT_ENDPOINT" ] || die "need --project-id or --project-endpoint to resolve the project" + # endpoint: https://.services.ai.azure.com/api/projects/ + local acct proj + acct="$(printf '%s' "$PROJECT_ENDPOINT" | sed -E 's#https?://([^.]+)\..*#\1#')" + proj="$(printf '%s' "$PROJECT_ENDPOINT" | sed -E 's#.*/projects/([^/?]+).*#\1#')" + [ -n "$acct" ] && [ -n "$proj" ] || die "could not parse account/project from endpoint" + local acct_id + acct_id="$(az cognitiveservices account list --query "[?name=='${acct}'].id | [0]" -o tsv 2>/dev/null || true)" + [ -n "$acct_id" ] || die "could not find Foundry account '$acct' via az (check subscription/login)" + PROJECT_ID="${acct_id}/projects/${proj}" + log "resolved project-id=$PROJECT_ID" +} + +# service-name -> AGENT_{KEY}_NAME env var used by `azd ai agent delete/show` +name_key() { # $1=service name + local k; k="$(printf '%s' "$1" | tr 'a-z' 'A-Z' | tr '-' '_' | tr ' ' '_')" + printf 'AGENT_%s_NAME' "$k" +} + +INITED=0 +ensure_azd_project() { + [ "$INITED" = 1 ] && return 0 + resolve_project_id + # Remove a local .venv from the sample dir so `init` doesn't copy it into src/. + [ -d "$SAMPLE_DIR/.venv" ] && [ "$CREATED_VENV" = 1 ] && { rm -rf "$SAMPLE_DIR/.venv"; CREATED_VENV=0; } + + log "azd ai agent init (this can take a few minutes)" + ( cd "$WORK_DIR" && azd ai agent init \ + -m "$SAMPLE_DIR/agent.manifest.yaml" \ + --project-id "$PROJECT_ID" \ + --model-deployment "$MODEL" \ + --agent-name "$AGENT_NAME" \ + --no-prompt --force ) >"$WORK_DIR/init.log" 2>&1 \ + || { tail -n 40 "$WORK_DIR/init.log" >&2; die "azd ai agent init failed"; } + + PROJECT_ROOT="$WORK_DIR/$AGENT_NAME" + [ -f "$PROJECT_ROOT/azure.yaml" ] || die "azd project not found at $PROJECT_ROOT after init" + + # GOTCHA fix: init hardcodes the manifest model into the generated agent.yaml. + # Restore the template so `azd deploy` substitutes the azd env value. + local ay="$PROJECT_ROOT/src/$AGENT_NAME/agent.yaml" + if [ -f "$ay" ]; then + awk ' + seen && /value:/ { sub(/value:.*/, "value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME}"); seen=0 } + /- name: AZURE_AI_MODEL_DEPLOYMENT_NAME/ { seen=1 } + { print } + ' "$ay" > "$ay.tmp" && mv "$ay.tmp" "$ay" + fi + + ( cd "$PROJECT_ROOT" && azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME "$MODEL" >/dev/null ) + INITED=1 +} + +# ========================= PHASE 1: native local =========================== +if [ "$DO_NATIVE" = 1 ]; then + [ "$PROTOCOL" = responses ] || { warn "phase 1 (native curl) supports 'responses'; sample is '$PROTOCOL' — skipping"; DO_NATIVE=0; } +fi +if [ "$DO_NATIVE" = 1 ]; then + log "=== Phase 1: native local run ===" + [ -n "$PROJECT_ENDPOINT" ] || die "native run needs --project-endpoint (FOUNDRY_PROJECT_ENDPOINT)" + + printf 'FOUNDRY_PROJECT_ENDPOINT="%s"\nAZURE_AI_MODEL_DEPLOYMENT_NAME="%s"\n' \ + "$PROJECT_ENDPOINT" "$MODEL" > "$SAMPLE_DIR/.env"; CREATED_ENV=1 + + log "creating venv + installing requirements (uv)" + uv venv "$SAMPLE_DIR/.venv" --python 3.12 >"$WORK_DIR/venv.log" 2>&1 || { cat "$WORK_DIR/venv.log" >&2; die "uv venv failed"; } + CREATED_VENV=1 + if [ -x "$SAMPLE_DIR/.venv/bin/python" ]; then VENV_PY="$SAMPLE_DIR/.venv/bin/python" + elif [ -x "$SAMPLE_DIR/.venv/Scripts/python.exe" ]; then VENV_PY="$SAMPLE_DIR/.venv/Scripts/python.exe" + else die "venv python not found under $SAMPLE_DIR/.venv"; fi + uv pip install --python "$VENV_PY" -r "$SAMPLE_DIR/requirements.txt" >"$WORK_DIR/pip.log" 2>&1 \ + || { tail -n 30 "$WORK_DIR/pip.log" >&2; die "uv pip install failed"; } + + log "starting server: python main.py (:$PORT)" + ( cd "$SAMPLE_DIR" && exec "$VENV_PY" main.py ) >"$WORK_DIR/native-server.log" 2>&1 & + NATIVE_PID=$! + wait_http "http://localhost:$PORT/responses" 90 || { tail -n 40 "$WORK_DIR/native-server.log" >&2; die "native server did not start on :$PORT"; } + + log "invoke: turn 1 (set name)" + b1="$(curl -sS -X POST "http://localhost:$PORT/responses" -H 'Content-Type: application/json' \ + -d '{"input":"My name is Tao. Please remember it."}')" + [ "$(printf '%s' "$b1" | jq -r '.status // empty')" = completed ] || die "native turn1 not completed: $b1" + rid="$(printf '%s' "$b1" | jq -r '.response_id // empty')" + [ -n "$rid" ] || die "native turn1 missing response_id" + + log "invoke: turn 2 (recall via previous_response_id)" + b2="$(curl -sS -X POST "http://localhost:$PORT/responses" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg id "$rid" '{input:"What is my name?", previous_response_id:$id}')")" + t2="$(printf '%s' "$b2" | jq -r '.output[0].content[0].text // empty')" + printf '%s' "$t2" | grep -qi 'Tao' || die "native multi-turn recall failed (got: '$t2')" + ok "native local: single + multi-turn (recall: '$t2')"; record pass "native local (python main.py)" + + stop_bg "$NATIVE_PID" "$PORT"; NATIVE_PID="" +else + log "skipping Phase 1 (native local)" +fi + +# ========================= PHASE 2: azd local ============================== +if [ "$DO_AZD_LOCAL" = 1 ]; then + log "=== Phase 2: local via 'azd ai agent run' ===" + ensure_azd_project + + log "starting: azd ai agent run --no-inspector (:$PORT)" + ( cd "$PROJECT_ROOT" && exec azd ai agent run --no-inspector --port "$PORT" ) >"$WORK_DIR/azd-run.log" 2>&1 & + AZD_RUN_PID=$! + wait_http "http://localhost:$PORT/responses" 240 || { tail -n 60 "$WORK_DIR/azd-run.log" >&2; die "azd ai agent run did not start on :$PORT"; } + + log "invoke --local: turn 1 (set name)" + r1="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke --local --protocol "$PROTOCOL" --output raw --new-session \ + "My name is Tao. Please remember it." ) 2>&1 || true )" + check_sse "$r1" || die "azd local turn1 failed" + log "invoke --local: turn 2 (recall, same session)" + r2="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke --local --protocol "$PROTOCOL" --output raw \ + "What is my name?" ) 2>&1 || true )" + check_sse "$r2" "Tao" || die "azd local multi-turn recall failed" + ok "azd local: single + multi-turn"; record pass "azd local (azd ai agent run)" + + stop_bg "$AZD_RUN_PID" "$PORT"; AZD_RUN_PID="" +else + log "skipping Phase 2 (azd local)" +fi + +# ========================= PHASE 3: deploy ================================= +if [ "$DO_DEPLOY" = 1 ]; then + log "=== Phase 3: deploy to Foundry ===" + [ -n "$ACR_ENDPOINT" ] || die "deploy needs --acr-endpoint (AZURE_CONTAINER_REGISTRY_ENDPOINT) to reuse an ACR" + ensure_azd_project + + ( cd "$PROJECT_ROOT" && azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT "$ACR_ENDPOINT" >/dev/null ) + + # Ensure a clean deploy: best-effort delete a pre-existing agent of this name. + log "removing any pre-existing agent named '$AGENT_NAME'" + NK="$(name_key "$AGENT_NAME")" + ( cd "$PROJECT_ROOT" && azd env set "$NK" "$AGENT_NAME" >/dev/null ) + ( cd "$PROJECT_ROOT" && azd ai agent delete "$AGENT_NAME" --force --no-prompt >/dev/null 2>&1 ) \ + && log "deleted pre-existing agent" || log "no pre-existing agent to delete" + + log "azd deploy (remote build via ACR; can take a few minutes)" + ( cd "$PROJECT_ROOT" && azd deploy ) >"$WORK_DIR/deploy.log" 2>&1 \ + || { tail -n 60 "$WORK_DIR/deploy.log" >&2; die "azd deploy failed"; } + DEPLOYED=1 + ok "azd deploy succeeded" + + log "verify deployed model env var" + shown="$( cd "$PROJECT_ROOT" && azd ai agent show "$AGENT_NAME" --output json 2>/dev/null || true )" + dep_model="$(printf '%s' "$shown" | jq -r '.definition.environment_variables.AZURE_AI_MODEL_DEPLOYMENT_NAME // empty')" + [ "$dep_model" = "$MODEL" ] || die "deployed model env var is '$dep_model', expected '$MODEL'" + status="$(printf '%s' "$shown" | jq -r '.status // empty')" + [ "$status" = active ] || warn "deployed agent status is '$status' (expected active)" + log "deployed model=$dep_model status=$status" + + log "invoke deployed: turn 1 (set name)" + d1="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke "$AGENT_NAME" --protocol "$PROTOCOL" --output raw --new-session \ + "My name is Tao. Please remember it." ) 2>&1 || true )" + check_sse "$d1" || die "deployed turn1 failed" + log "invoke deployed: turn 2 (recall, same session)" + d2="$( ( cd "$PROJECT_ROOT" && azd ai agent invoke "$AGENT_NAME" --protocol "$PROTOCOL" --output raw \ + "What is my name?" ) 2>&1 || true )" + check_sse "$d2" "Tao" || die "deployed multi-turn recall failed" + ok "deployed: single + multi-turn"; record pass "deployed (azd deploy + invoke)" +else + log "skipping Phase 3 (deploy)" +fi + +# ============================= summary ===================================== +printf '\n===================== validation summary =====================\n' +printf '%s' "$SUMMARY" +printf '==============================================================\n' +printf 'passed: %d failed: %d\n' "$PASS_COUNT" "$FAIL_COUNT" +[ "$FAIL_COUNT" -eq 0 ] || exit 1 +log "all enabled checks passed" diff --git a/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md b/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md deleted file mode 100644 index 58412c6450a..00000000000 --- a/python/scripts/sample_validation/skills/hosting-sample-runner/SKILL.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: hosting-sample-runner -description: Decide how to validate a hosting sample (for example anything under 04-hosting that starts a server, function host, or deployed agent). Use when a sample hosts an agent or workflow and is run by starting a local server/host and calling it, or by deploying to a cloud provider. -license: MIT -compatibility: Works with any model that supports tool use. -metadata: - author: agent-framework-samples - version: "1.0" ---- - -## Usage - -Hosting samples are different from ordinary scripts: instead of running to -completion, they typically start a server, function host, or hosted agent and -are exercised by a separate client call. Each hosting sample includes a -`README.md` that documents how to set up and run it. - -When validating a hosting sample: - -1. Read the sample's `README.md` (and any sibling READMEs in parent - directories) to understand how the sample is meant to be run. -2. Decide whether the sample can be run **locally** — that is, fully exercised - on this machine without deploying to a cloud provider (for example Azure - Functions deployment, an Azure Container App, or a Foundry hosted-agent - publish step). A sample is locally runnable when its README describes a - local launch path, such as starting a local server (e.g. Hypercorn, - `uv run python app.py`, the Functions Core Tools `func start`, or a durable - task worker) and then calling it from a local client/HTTP request. - -### If the sample can be run locally - -Follow the README's local setup and run instructions: - -1. Install any required dependencies it lists. -2. Start the host process in the background (it will not exit on its own). -3. Exercise it as the README describes — run the companion client script, - send the documented HTTP request, or otherwise drive a single end-to-end - interaction. -4. If the interaction succeeds, stop the host process and mark the sample as - `success`. -5. If the host fails to start or the interaction errors, treat it as a - `failure` and investigate the error. - -### If the sample cannot be run locally - -If the README only documents a cloud deployment path (for example deploying -to Azure Functions, publishing a Foundry hosted agent, or otherwise requiring -provisioned cloud infrastructure to exercise the sample), do not attempt to -deploy it. Mark the sample as `missing_setup` and note in the output that it -requires cloud deployment that cannot be performed locally. From 2958f19df34a9553110fca938e40a96010fc64bb Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 7 Jul 2026 14:09:10 -0700 Subject: [PATCH 05/28] Fix hosted agent file sample --- .../responses/06_files/main.py | 130 +++++------------- 1 file changed, 32 insertions(+), 98 deletions(-) diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py b/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py index 43a1d241735..c47addd4a2e 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/06_files/main.py @@ -2,70 +2,17 @@ import asyncio import os -from collections.abc import Callable -from urllib.parse import urlsplit -import httpx -from agent_framework import Agent, MCPStreamableHTTPTool, tool +from agent_framework import Agent, tool from agent_framework.foundry import FoundryChatClient -from agent_framework_foundry_hosting import ResponsesHostServer -from azure.identity import DefaultAzureCredential, get_bearer_token_provider +from agent_framework_foundry_hosting import FoundryToolbox, ResponsesHostServer +from azure.identity import DefaultAzureCredential from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() -def resolve_toolbox_endpoint() -> str: - """Resolve the toolbox MCP endpoint URL. - - Prefers the explicit ``TOOLBOX_ENDPOINT`` env var (set in ``agent.yaml`` or - ``agent.manifest.yaml`` and via ``azd env set TOOLBOX_ENDPOINT`` after the toolbox - is created); falls back to constructing the URL from ``FOUNDRY_PROJECT_ENDPOINT`` - and ``TOOLBOX_NAME``. - """ - if (endpoint := os.environ.get("TOOLBOX_ENDPOINT")) is not None: - if not endpoint: - raise ValueError("TOOLBOX_ENDPOINT is set but empty") - return endpoint - try: - project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"].rstrip("/") - toolbox_name = os.environ["TOOLBOX_NAME"] - except KeyError as e: - raise ValueError( - "Either set TOOLBOX_ENDPOINT, or set both FOUNDRY_PROJECT_ENDPOINT " - "and TOOLBOX_NAME to build the toolbox MCP endpoint." - ) from e - return f"{project_endpoint}/toolboxes/{toolbox_name}/mcp?api-version=v1" - - -def _toolbox_name_from_endpoint(endpoint: str) -> str: - """Extract the toolbox name from a toolbox MCP endpoint URL. - - Handles both the versioned (``.../toolboxes//versions//mcp``) and - unversioned (``.../toolboxes//mcp``) endpoint shapes that Foundry - produces. Falls back to ``"toolbox"`` when the path has no ``toolboxes`` - segment. - """ - segments = urlsplit(endpoint).path.split("/") - if "toolboxes" in segments: - idx = segments.index("toolboxes") - if idx + 1 < len(segments) and segments[idx + 1]: - return segments[idx + 1] - return "toolbox" - - -class ToolboxAuth(httpx.Auth): - """Injects a fresh bearer token on every request.""" - - def __init__(self, token_provider: Callable[[], str]): - self._get_token = token_provider - - def auth_flow(self, request: httpx.Request): - request.headers["Authorization"] = f"Bearer {self._get_token()}" - yield request - - @tool(description="Get the current working directory.", approval_mode="never_require") def get_cwd() -> str: """Get the current working directory.""" @@ -97,48 +44,35 @@ def read_file(file_path: str) -> str: async def main(): credential = DefaultAzureCredential() - # Create the toolbox - token_provider = get_bearer_token_provider(credential, "https://ai.azure.com/.default") - - # Resolve the endpoint once and derive a friendly tool name from it. When - # ``TOOLBOX_NAME`` isn't set, extract the toolbox name from the URL path so - # the tool's local name matches the upstream toolbox. - toolbox_endpoint = resolve_toolbox_endpoint() - toolbox_name = os.environ.get("TOOLBOX_NAME") or _toolbox_name_from_endpoint(toolbox_endpoint) - - async with httpx.AsyncClient( - auth=ToolboxAuth(token_provider), - timeout=120.0, - ) as http_client: - toolbox = MCPStreamableHTTPTool( - name=toolbox_name, - url=toolbox_endpoint, - http_client=http_client, - load_prompts=False, - ) - - # Create the chat client - client = FoundryChatClient( - project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], - credential=credential, - ) - - agent = Agent( - client=client, - instructions=( - "You are a friendly assistant. Keep your answers brief. " - "Make sure all mathematical calculations are performed using the code interpreter " - "instead of mental arithmetic." - ), - tools=[get_cwd, list_files, read_file, toolbox], - # History will be managed by the hosting infrastructure, thus there - # is no need to store history by the service. Learn more at: - # https://developers.openai.com/api/reference/resources/responses/methods/create - default_options={"store": False}, - ) - server = ResponsesHostServer(agent) - await server.run_async() + # FoundryToolbox resolves the toolbox endpoint from the environment + # (TOOLBOX_ENDPOINT, or FOUNDRY_PROJECT_ENDPOINT + TOOLBOX_NAME), authenticates + # every request with the credential, and transparently forwards the platform + # per-request call-id to the toolbox. The hosting server enters the agent, which + # connects the toolbox on first use and closes it at shutdown. + toolbox = FoundryToolbox(credential) + + # Create the chat client + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + credential=credential, + ) + + agent = Agent( + client=client, + instructions=( + "You are a friendly assistant. Keep your answers brief. " + "Make sure all mathematical calculations are performed using the code interpreter " + "instead of mental arithmetic." + ), + tools=[get_cwd, list_files, read_file, toolbox], + # History will be managed by the hosting infrastructure, thus there + # is no need to store history by the service. Learn more at: + # https://developers.openai.com/api/reference/resources/responses/methods/create + default_options={"store": False}, + ) + server = ResponsesHostServer(agent) + await server.run_async() if __name__ == "__main__": From 6b3b89b9d1a81c7ef7375f8e39341c1477cd88c8 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 7 Jul 2026 14:50:23 -0700 Subject: [PATCH 06/28] Fix agent result format --- .../sample_validation/create_dynamic_workflow_executor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 3a2a4e3a318..ce245b8e7df 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -41,7 +41,6 @@ class AgentResponseFormat(BaseModel): status: str output: str error: str - fix: str @dataclass From d63c00895bc2709ab57278271008a64312747d42 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 7 Jul 2026 16:58:32 -0700 Subject: [PATCH 07/28] Reorganize jobs --- .../workflows/python-sample-validation.yml | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 531d96800a8..83ec7aeac77 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -9,7 +9,7 @@ env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache # GitHub Copilot configuration - GITHUB_COPILOT_MODEL: claude-opus-4.8 + GITHUB_COPILOT_MODEL: auto COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} permissions: @@ -199,6 +199,7 @@ jobs: validate-02-agents-anthropic: name: Validate 02-agents/providers/anthropic + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -236,6 +237,7 @@ jobs: validate-02-agents-github-copilot: name: Validate 02-agents/providers/github_copilot + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration defaults: @@ -476,8 +478,8 @@ jobs: name: validation-report-03-workflows path: python/samples/sample_validation/reports/ - validate-04-hosting: - name: Validate 04-hosting + validate-04-hosting-foundry-hosted-agents: + name: Validate 04-hosting (foundry-hosted-agents) runs-on: ubuntu-latest environment: integration env: @@ -487,6 +489,40 @@ jobs: AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL }} FOUNDRY_PROJECT_ID: ${{ vars.FOUNDRY_PROJECT_ID }} AZURE_CONTAINER_REGISTRY_ENDPOINT: ${{ vars.AZURE_CONTAINER_REGISTRY_ENDPOINT }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-04-hosting-foundry-hosted-agents + path: python/samples/sample_validation/reports/ + + validate-04-hosting-other: + name: Validate 04-hosting (other) + if: false # Temporarily disabled - to free up Copilot quota for other jobs + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # A2A configuration A2A_AGENT_HOST: http://localhost:5001/ defaults: @@ -505,13 +541,13 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 04-hosting --save-report --report-name 04-hosting + cd scripts && uv run python -m sample_validation --subdir 04-hosting --exclude foundry-hosted-agents --save-report --report-name 04-hosting-other - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() with: - name: validation-report-04-hosting + name: validation-report-04-hosting-other path: python/samples/sample_validation/reports/ validate-05-end-to-end: @@ -688,7 +724,8 @@ jobs: - validate-02-agents-copilotstudio - validate-02-agents-custom - validate-03-workflows - - validate-04-hosting + - validate-04-hosting-foundry-hosted-agents + - validate-04-hosting-other - validate-05-end-to-end - validate-autogen-migration - validate-semantic-kernel-migration From a6fbc3c23b56e2d4be8ccd93db84c2236025c601 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 9 Jul 2026 09:52:43 -0700 Subject: [PATCH 08/28] Update discovery heuristic for apps --- .../workflows/python-sample-validation.yml | 3 ++- python/scripts/sample_validation/discovery.py | 21 ++++++++++++------- python/scripts/sample_validation/models.py | 2 -- .../foundry-hosted-agent-validation/SKILL.md | 6 ++++++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 83ec7aeac77..82d9f5f96eb 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -505,8 +505,9 @@ jobs: os: ${{ runner.os }} - name: Run sample validation + # Maximum parallel workers is set to 1 because all samples use the same port run: | - cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents + cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/python/scripts/sample_validation/discovery.py b/python/scripts/sample_validation/discovery.py index c5424dd6ee9..1b613c1f5b5 100644 --- a/python/scripts/sample_validation/discovery.py +++ b/python/scripts/sample_validation/discovery.py @@ -58,7 +58,7 @@ def discover_samples( exclude: list[str] | None = None, ) -> list[SampleInfo]: """ - Find all Python sample files in the samples directory. + Find all samples in the samples directory. Args: samples_dir: Root samples directory @@ -80,7 +80,7 @@ def discover_samples( # Resolve excluded paths to absolute for reliable comparison exclude_paths = {(search_dir / exc).resolve() for exc in (exclude or [])} - python_files: list[Path] = [] + samples: list[Path] = [] # Walk through all subdirectories and find .py files for root, dirs, files in os.walk(search_dir): @@ -93,25 +93,30 @@ def discover_samples( and (Path(root) / d).resolve() not in exclude_paths ] + # If the whole directory is a sample, add the directory itself and skip its subdirectories + if any(file in ("main.py", "app.py") for file in files): + samples.append(Path(root)) + continue + for file in files: # Skip files that start with _ and include only scripts with a main entrypoint guard if file.endswith(".py") and not file.startswith("_"): file_path = Path(root) / file if _has_main_entrypoint_guard(file_path): - python_files.append(file_path) + samples.append(file_path) # Sort files for consistent execution order - python_files = sorted(python_files) + samples = sorted(samples) # Convert to SampleInfo objects - samples: list[SampleInfo] = [] - for path in python_files: + samples_info: list[SampleInfo] = [] + for path in samples: try: - samples.append(SampleInfo.from_path(path, samples_dir)) + samples_info.append(SampleInfo.from_path(path, samples_dir)) except Exception as e: print(f"Warning: Could not read {path}: {e}") - return samples + return samples_info class DiscoverSamplesExecutor(Executor): diff --git a/python/scripts/sample_validation/models.py b/python/scripts/sample_validation/models.py index e41de324eba..f5e6c40066d 100644 --- a/python/scripts/sample_validation/models.py +++ b/python/scripts/sample_validation/models.py @@ -28,7 +28,6 @@ class SampleInfo: path: Path relative_path: str - code: str @classmethod def from_path(cls, path: Path, samples_dir: Path) -> "SampleInfo": @@ -36,7 +35,6 @@ def from_path(cls, path: Path, samples_dir: Path) -> "SampleInfo": return cls( path=path, relative_path=str(path.relative_to(samples_dir)), - code=path.read_text(encoding="utf-8"), ) diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md index ed70c965e9b..8609120166e 100644 --- a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -100,6 +100,12 @@ A responses/invocations sample folder typically contains: `agent.manifest.yaml` (used by `azd ai agent init`), `agent.yaml` (the deployed agent definition), `requirements.txt`, `Dockerfile`, `.env.example`. +**The sample is the whole directory whose entry point is `main.py` — not every +`.py` file in it.** Other Python files in (or alongside) a sample folder are +**helper/companion scripts**, not standalone samples. Do **not** treat a helper +script as an individual sample — validate the sample via its `main.py` host, and +run a helper only when the sample's `README.md` calls for it as a setup. + Note the **protocol** (`responses` or `invocations`) from `agent.yaml` / manifest — it changes the invoke command (`--protocol invocations`) and the HTTP path (`/responses` vs the invocations route). From 5f9406f761f7eedda919dec70fbb5f02261cad72 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 9 Jul 2026 16:42:54 -0700 Subject: [PATCH 09/28] Split agents into even more jobs --- .../workflows/python-sample-validation.yml | 83 ++++++++++++++++++- .../foundry-hosted-agent-validation/SKILL.md | 5 +- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 82d9f5f96eb..711f06ee13b 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -108,7 +108,7 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers --save-report --report-name 02-agents + cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers harness tools --save-report --report-name 02-agents - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -117,6 +117,85 @@ jobs: name: validation-report-02-agents path: python/samples/sample_validation/reports/ + validate-02-agents-harness: + name: Validate 02-agents/harness + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # Optional: enables the Foundry memory path in harness samples + FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }} + FOUNDRY_MEMORY_STORE: ${{ vars.FOUNDRY_MEMORY_STORE || '' }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + echo "FOUNDRY_EMBEDDING_MODEL=$FOUNDRY_EMBEDDING_MODEL" >> .env + echo "FOUNDRY_MEMORY_STORE=$FOUNDRY_MEMORY_STORE" >> .env + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/harness --save-report --report-name 02-agents-harness + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-02-agents-harness + path: python/samples/sample_validation/reports/ + + validate-02-agents-tools: + name: Validate 02-agents/tools + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/tools --save-report --report-name 02-agents-tools + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-02-agents-tools + path: python/samples/sample_validation/reports/ + validate-02-agents-openai: name: Validate 02-agents/providers/openai runs-on: ubuntu-latest @@ -715,6 +794,8 @@ jobs: needs: - validate-01-get-started - validate-02-agents + - validate-02-agents-harness + - validate-02-agents-tools - validate-02-agents-openai - validate-02-agents-azure - validate-02-agents-anthropic diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md index 8609120166e..e57c05cf384 100644 --- a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -4,9 +4,8 @@ description: > Step-by-step process for validating a Python Foundry hosted agent sample (under python/samples/04-hosting/foundry-hosted-agents/) end to end — running it locally (native runtime and `azd ai agent run`) and after deploying it to - an Azure AI Foundry project with `azd`. Use this when asked to validate, smoke - test, or verify a hosted agent sample works locally and/or deployed, or when - deploying one of these samples to Foundry. + an Azure AI Foundry project with `azd`. Use this when asked to validate a hosted + agent sample. license: MIT compatibility: Works with any model that supports tool use. metadata: From 16b2c1e354198766301f80b9334220ac0561a564 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Mon, 20 Jul 2026 14:12:15 -0700 Subject: [PATCH 10/28] Add toolbox endpoint --- .github/workflows/python-sample-validation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 5fa8da82e6b..1448aed6382 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -564,6 +564,7 @@ jobs: env: FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + TOOLBOX_ENDPOINT: ${{ vars.TOOLBOX_ENDPOINT }} # Foundry hosted agent configuration AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL }} FOUNDRY_PROJECT_ID: ${{ vars.FOUNDRY_PROJECT_ID }} From 5dc928dd581160d0844e7ec8ce5ddd58444c6868 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Mon, 20 Jul 2026 15:33:32 -0700 Subject: [PATCH 11/28] Add more pre configured resources --- .github/workflows/python-sample-validation.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 1448aed6382..b14ffc2313e 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -564,12 +564,16 @@ jobs: env: FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} - TOOLBOX_ENDPOINT: ${{ vars.TOOLBOX_ENDPOINT }} # Foundry hosted agent configuration AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL }} FOUNDRY_PROJECT_ID: ${{ vars.FOUNDRY_PROJECT_ID }} AZURE_CONTAINER_REGISTRY_ENDPOINT: ${{ vars.AZURE_CONTAINER_REGISTRY_ENDPOINT }} GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + TOOLBOX_ENDPOINT: ${{ vars.TOOLBOX_ENDPOINT }} + FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_NAME }} + MEMORY_STORE_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_MEMORY_STORE }} + AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }} + AZURE_SEARCH_INDEX_NAME: ${{ secrets.FOUNDRY_HOSTED_AGENT_SEARCH_INDEX_NAME }} defaults: run: working-directory: python From d303b892ecc7033938b888e4e8272d3d77e71a0e Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 21 Jul 2026 11:12:21 -0700 Subject: [PATCH 12/28] Fix using deployed agent sample --- .../responses/09_foundry_skills/README.md | 2 + .../responses/using_deployed_agent.py | 38 +++++-------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/09_foundry_skills/README.md b/python/samples/04-hosting/foundry-hosted-agents/responses/09_foundry_skills/README.md index d11ea8096db..5932d070085 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/09_foundry_skills/README.md +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/09_foundry_skills/README.md @@ -1,5 +1,7 @@ # What this sample demonstrates +> IMPORTANT: We recommend Foundry Hosted Agents to consume skills via Foundry Toolbox. Please see the [Foundry Toolbox MCP Skills](../12_foundry_toolbox_mcp_skills/README.md) sample for a more robust and production-ready approach to consuming skills in your hosted agent. + An [Agent Framework](https://github.com/microsoft/agent-framework) agent that loads its behavioral guidelines from [**Foundry Skills**](https://learn.microsoft.com/en-us/azure/foundry/agents/how-to/tools/skills?view=foundry&pivots=python) at startup, hosted using the **Responses protocol**. Skills are authored once as `SKILL.md` files, uploaded to your Foundry project through `AIProjectClient.beta.skills`, and downloaded by the agent on boot so updates ship without code changes. ## How It Works diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py index 678979bef0e..369f549add8 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py @@ -4,8 +4,6 @@ import asyncio import os -from collections.abc import Mapping -from typing import Any, cast from agent_framework import AgentSession from agent_framework.foundry import FoundryAgent @@ -41,41 +39,25 @@ async def create_hosted_agent_session( project_client: AIProjectClient, agent_name: str, agent_version: str | None, - isolation_key: str, ) -> AgentSession: """Create a hosted-agent service session and wrap it in an AgentSession.""" - create_session_kwargs: dict[str, Any] = { - "agent_name": agent_name, - "isolation_key": isolation_key, - } resolved_agent_version = agent_version if resolved_agent_version is None: - agent_details = await cast(Any, project_client.beta.agents).get( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] - agent_name=agent_name - ) - versions = getattr(agent_details, "versions", None) - if not isinstance(versions, Mapping): - raise ValueError("Hosted agent details did not include a versions mapping.") - latest_version = getattr(cast(Any, versions.get("latest")), "version", None) - if not isinstance(latest_version, str) or not latest_version: - raise ValueError("Hosted agent details did not include a latest version string.") - resolved_agent_version = latest_version - - create_session_kwargs["version_indicator"] = VersionRefIndicator(agent_version=resolved_agent_version) - service_session = await project_client.beta.agents.create_session(**create_session_kwargs) - agent_session_id = getattr(service_session, "agent_session_id", None) - if not isinstance(agent_session_id, str) or not agent_session_id: - raise ValueError("Hosted agent session creation did not return a non-empty agent_session_id.") + agent_details = await project_client.agents.get(agent_name) + resolved_agent_version = agent_details.versions.latest.version - return agent.get_session(agent_session_id) + service_session = await project_client.beta.agents.create_session( + agent_name, + version_indicator=VersionRefIndicator(agent_version=resolved_agent_version), + ) + return agent.get_session(service_session.agent_session_id) async def main() -> None: credential = AzureCliCredential() - project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - agent_name = os.environ["FOUNDRY_AGENT_NAME"] + project_endpoint = "https://ai-foundry-sk-interation-test.services.ai.azure.com/api/projects/ai-proj-ga-sk-interation-test" # os.environ["FOUNDRY_PROJECT_ENDPOINT"] + agent_name = "agent-framework-agent-int-test-do-not-delete" # os.environ["FOUNDRY_AGENT_NAME"] agent_version = os.getenv("FOUNDRY_AGENT_VERSION") - isolation_key = "my-isolation-key" project_client = AIProjectClient( endpoint=project_endpoint, @@ -96,7 +78,6 @@ async def main() -> None: project_client=project_client, agent_name=agent_name, agent_version=agent_version, - isolation_key=isolation_key, ) try: @@ -128,7 +109,6 @@ async def main() -> None: await project_client.beta.agents.delete_session( agent_name=agent_name, session_id=session.service_session_id, - isolation_key=isolation_key, ) From 3bbcd9af6b8ac22e4b02e42d25bd1a370d68a39a Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 21 Jul 2026 11:26:31 -0700 Subject: [PATCH 13/28] Add sample status --- .../foundry-hosted-agents/README.md | 45 +++++++++++-------- .../foundry-hosted-agent-validation/SKILL.md | 6 +++ 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/python/samples/04-hosting/foundry-hosted-agents/README.md b/python/samples/04-hosting/foundry-hosted-agents/README.md index a9b9af65910..730bf2754c9 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/README.md +++ b/python/samples/04-hosting/foundry-hosted-agents/README.md @@ -7,30 +7,37 @@ This directory contains samples that demonstrate how to use hosted [Agent Framew > The `agent-framework-foundry-hosting` Python API surface is intended to remain stable, but protocol 1.0.0 and 2.0.0 are incompatible. ## Samples +**Status legend:** + +| Icon | Status | Description | +|------|--------|-------------| +| ✅ | Most up to date | Demonstrates the current, recommended usage pattern. | +| ⚠️ | Legacy | Demonstrates an older pattern that has since been superseded and is no longer recommended. These samples are kept for reference and no longer maintained. | + ### Responses API -| # | Sample | Description | -|---|--------|-------------| -| 1 | [Basic](responses/01_basic/) | A minimal agent demonstrating basic request/response interaction and multi-turn conversations using `previous_response_id`. | -| 2 | [Tools](responses/02_tools/) | An agent with local tools (e.g., weather lookup), demonstrating how to register and invoke custom tool functions alongside the LLM. | -| 3 | [MCP](responses/03_mcp/) | An agent connected to a remote MCP server (GitHub), demonstrating external MCP tool provider integration. | -| 4 | [Foundry Toolbox](responses/04_foundry_toolbox/) | An agent using Azure Foundry Toolbox, demonstrating toolbox provisioning and querying available tools at runtime. | -| 5 | [Workflows](responses/05_workflows/) | An agent with a multi-step orchestrated workflow, demonstrating chaining prompts through an orchestrated flow. | -| 6 | [Files](responses/06_files/) | An agent demonstrating how to work with files in a hosted agent session, including uploading files to a hosted agent session and having the agent read and manipulate those files at runtime. | -| 7 | [Observability](responses/07_observability/) | A sample demonstrating how to enable observability for the agent deployed to Foundry. | -| 8 | [Azure AI Search RAG](responses/08_azure_search_rag/) | An agent with Retrieval Augmented Generation (RAG) capabilities backed by Azure AI Search, grounding answers in documents indexed in a pre-provisioned search index. | -| 9 | [Foundry Skills](responses/09_foundry_skills/) | An agent that uploads `SKILL.md` files to the Foundry Skills REST API and downloads them at startup, decoupling tone/policy guidelines from agent code. | -| 10 | [Foundry Memory](responses/10_foundry_memory/) | An agent with persistent semantic memory backed by a Microsoft Foundry Memory Store, using `FoundryMemoryProvider` to remember user facts across sessions. | -| 11 | [Monty CodeAct](responses/11_monty_codeact/) | An agent with a Monty-backed CodeAct context provider, exposing a single `execute_code` tool that runs Python in a [pydantic-monty](https://github.com/pydantic/monty) interpreter and invokes typed host tools (`compute`, `fetch_data`) from inside the sandbox. Uses the alpha `agent-framework-monty` package. | -| 12 | [Foundry Toolbox MCP Skills](responses/12_foundry_toolbox_mcp_skills/) | An agent that discovers MCP-based skills attached to a Foundry Toolbox and serves them via `SkillsProvider(MCPSkillsSource(...))`, fetching `SKILL.md` bodies and supplementary resources on demand. | -| 13 | [Using deployed agent](responses/using_deployed_agent.py) | A sample demonstrating how to invoke an agent that has already been deployed to Foundry, showing how to interact with a hosted agent in code. | +| Status | # | Sample | Description | +|--------|---|--------|-------------| +| ✅ | 1 | [Basic](responses/01_basic/) | A minimal agent demonstrating basic request/response interaction and multi-turn conversations using `previous_response_id`. | +| ✅ | 2 | [Tools](responses/02_tools/) | An agent with local tools (e.g., weather lookup), demonstrating how to register and invoke custom tool functions alongside the LLM. | +| ✅ | 3 | [MCP](responses/03_mcp/) | An agent connected to a remote MCP server (GitHub), demonstrating external MCP tool provider integration. | +| ✅ | 4 | [Foundry Toolbox](responses/04_foundry_toolbox/) | An agent using Azure Foundry Toolbox, demonstrating toolbox provisioning and querying available tools at runtime. | +| ✅ | 5 | [Workflows](responses/05_workflows/) | An agent with a multi-step orchestrated workflow, demonstrating chaining prompts through an orchestrated flow. | +| ✅ | 6 | [Files](responses/06_files/) | An agent demonstrating how to work with files in a hosted agent session, including uploading files to a hosted agent session and having the agent read and manipulate those files at runtime. | +| ✅ | 7 | [Observability](responses/07_observability/) | A sample demonstrating how to enable observability for the agent deployed to Foundry. | +| ✅ | 8 | [Azure AI Search RAG](responses/08_azure_search_rag/) | An agent with Retrieval Augmented Generation (RAG) capabilities backed by Azure AI Search, grounding answers in documents indexed in a pre-provisioned search index. | +| ⚠️ | 9 | [Foundry Skills](responses/09_foundry_skills/) | An agent that uploads `SKILL.md` files to the Foundry Skills REST API and downloads them at startup, decoupling tone/policy guidelines from agent code. | +| ✅ | 10 | [Foundry Memory](responses/10_foundry_memory/) | An agent with persistent semantic memory backed by a Microsoft Foundry Memory Store, using `FoundryMemoryProvider` to remember user facts across sessions. | +| ✅ | 11 | [Monty CodeAct](responses/11_monty_codeact/) | An agent with a Monty-backed CodeAct context provider, exposing a single `execute_code` tool that runs Python in a [pydantic-monty](https://github.com/pydantic/monty) interpreter and invokes typed host tools (`compute`, `fetch_data`) from inside the sandbox. Uses the alpha `agent-framework-monty` package. | +| ✅ | 12 | [Foundry Toolbox MCP Skills](responses/12_foundry_toolbox_mcp_skills/) | An agent that discovers MCP-based skills attached to a Foundry Toolbox and serves them via `SkillsProvider(MCPSkillsSource(...))`, fetching `SKILL.md` bodies and supplementary resources on demand. | +| ✅ | 13 | [Using deployed agent](responses/using_deployed_agent.py) | A sample demonstrating how to invoke an agent that has already been deployed to Foundry, showing how to interact with a hosted agent in code. | ### Invocations API -| # | Sample | Description | -|---|--------|-------------| -| 1 | [Basic](invocations/01_basic/) | A minimal agent demonstrating session state management via `agent_session_id` in URL params/response headers. | -| 2 | [Break Glass](invocations/02_break_glass/) | An agent demonstrating a "break glass" scenario where customizations of the API behaviors are needed, allowing for more direct control over how requests and responses are handled by the hosting layer. | +| Status | # | Sample | Description | +|--------|---|--------|-------------| +| ✅ | 1 | [Basic](invocations/01_basic/) | A minimal agent demonstrating session state management via `agent_session_id` in URL params/response headers. | +| ✅ | 2 | [Break Glass](invocations/02_break_glass/) | An agent demonstrating a "break glass" scenario where customizations of the API behaviors are needed, allowing for more direct control over how requests and responses are handled by the hosting layer. | ## Running the Agent Host Locally diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md index e57c05cf384..dd69d3ff7ba 100644 --- a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -105,6 +105,12 @@ agent definition), `requirements.txt`, `Dockerfile`, `.env.example`. script as an individual sample — validate the sample via its `main.py` host, and run a helper only when the sample's `README.md` calls for it as a setup. +**Skip legacy samples — only validate "most up to date" samples.** The parent +`.../foundry-hosted-agents/README.md` sample tables have a **Status** column +(✅ Most up to date, ⚠️ Legacy). Samples marked **⚠️ Legacy** demonstrate an +older, superseded pattern and do **not** need validation — skip them and mark +them as success. Only validate samples marked **✅ Most up to date**. + Note the **protocol** (`responses` or `invocations`) from `agent.yaml` / manifest — it changes the invoke command (`--protocol invocations`) and the HTTP path (`/responses` vs the invocations route). From 24b3eca791d56f753f7f6078d710b8b37fa123c0 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 21 Jul 2026 15:20:36 -0700 Subject: [PATCH 14/28] Add playbook --- .../workflows/python-sample-validation.yml | 295 ++++++++++++++++- python/scripts/sample_validation/__main__.py | 36 ++ .../create_dynamic_workflow_executor.py | 181 +++++++++- python/scripts/sample_validation/models.py | 17 + python/scripts/sample_validation/playbook.py | 310 ++++++++++++++++++ .../sample_validation/replay_executor.py | 83 +++++ ...un_dynamic_validation_workflow_executor.py | 13 +- python/scripts/sample_validation/workflow.py | 5 +- 8 files changed, 924 insertions(+), 16 deletions(-) create mode 100644 python/scripts/sample_validation/playbook.py create mode 100644 python/scripts/sample_validation/replay_executor.py diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index b14ffc2313e..5be84fd0034 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -44,10 +44,25 @@ jobs: echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -57,6 +72,7 @@ jobs: validate-02-agents: name: Validate 02-agents + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -106,10 +122,25 @@ jobs: echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env echo "GITHUB_PAT=$GITHUB_PAT" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers harness tools --save-report --report-name 02-agents + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -119,6 +150,7 @@ jobs: validate-02-agents-harness: name: Validate 02-agents/harness + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -148,10 +180,25 @@ jobs: echo "FOUNDRY_EMBEDDING_MODEL=$FOUNDRY_EMBEDDING_MODEL" >> .env echo "FOUNDRY_MEMORY_STORE=$FOUNDRY_MEMORY_STORE" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/harness --save-report --report-name 02-agents-harness + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -161,6 +208,7 @@ jobs: validate-02-agents-tools: name: Validate 02-agents/tools + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -185,10 +233,25 @@ jobs: echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/tools --save-report --report-name 02-agents-tools + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -198,6 +261,7 @@ jobs: validate-02-agents-openai: name: Validate 02-agents/providers/openai + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -226,10 +290,25 @@ jobs: echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -239,6 +318,7 @@ jobs: validate-02-agents-azure: name: Validate 02-agents/providers/azure + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -265,10 +345,25 @@ jobs: echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env echo "AZURE_OPENAI_API_VERSION=$AZURE_OPENAI_API_VERSION" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure --save-report --report-name 02-agents-azure + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -303,10 +398,25 @@ jobs: echo "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" >> .env echo "ANTHROPIC_CHAT_MODEL=$ANTHROPIC_CHAT_MODEL" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -333,10 +443,25 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -365,10 +490,25 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -397,10 +537,25 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -410,6 +565,7 @@ jobs: validate-02-agents-foundry: name: Validate 02-agents/providers/foundry + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -438,10 +594,25 @@ jobs: echo "FOUNDRY_AGENT_NAME=$FOUNDRY_AGENT_NAME" >> .env echo "FOUNDRY_AGENT_VERSION=$FOUNDRY_AGENT_VERSION" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry --save-report --report-name 02-agents-foundry + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -480,10 +651,25 @@ jobs: echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -509,10 +695,25 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -522,6 +723,7 @@ jobs: validate-03-workflows: name: Validate 03-workflows + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -546,10 +748,25 @@ jobs: echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -588,10 +805,25 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation # Maximum parallel workers is set to 1 because all samples use the same port run: | - cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 + cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 --agent-timeout 1800 + + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -624,10 +856,25 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 04-hosting --exclude foundry-hosted-agents --save-report --report-name 04-hosting-other + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -667,10 +914,25 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -717,10 +979,25 @@ jobs: echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -730,6 +1007,7 @@ jobs: validate-semantic-kernel-migration: name: Validate semantic-kernel-migration + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -781,10 +1059,25 @@ jobs: echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env + - name: Restore sample playbooks + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration + - name: Save sample playbooks + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() + with: + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() diff --git a/python/scripts/sample_validation/__main__.py b/python/scripts/sample_validation/__main__.py index 948fed3a307..c5e1858e159 100644 --- a/python/scripts/sample_validation/__main__.py +++ b/python/scripts/sample_validation/__main__.py @@ -82,6 +82,33 @@ def parse_arguments() -> argparse.Namespace: help="Subdirectory paths to exclude (relative to the search directory set by --subdir)", ) + parser.add_argument( + "--playbooks-dir", + type=str, + default="./sample_validation/playbooks", + help=( + "Directory (relative to samples/) where cached per-sample playbooks are stored and " + "reused across runs (default: ./sample_validation/playbooks)" + ), + ) + + parser.add_argument( + "--no-cache", + action="store_true", + help="Ignore cached playbooks and always validate every sample with the agent", + ) + + parser.add_argument( + "--agent-timeout", + type=int, + default=120, + help=( + "Per-turn timeout in seconds for the GitHub Copilot agent while validating a sample. " + "Increase for long-running samples such as hosted agents that start a server and make " + "multiple calls (default: 120)" + ), + ) + return parser.parse_args() @@ -107,14 +134,23 @@ async def main() -> int: ) # Create validation config + playbooks_dir = (samples_dir / args.playbooks_dir).resolve() config = ValidationConfig( samples_dir=samples_dir, python_root=python_root, subdir=args.subdir, exclude=args.exclude, max_parallel_workers=max(1, args.max_parallel_workers), + playbooks_dir=playbooks_dir, + use_cache=not args.no_cache, + agent_timeout=max(1, args.agent_timeout), ) + if config.use_cache: + print(f"Playbook cache: {playbooks_dir}") + else: + print("Playbook cache: disabled (--no-cache)") + # Create and run the workflow workflow = create_validation_workflow(config) diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index ce245b8e7df..d98e6734302 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -20,15 +20,24 @@ from copilot.session_events import PermissionRequest from pydantic import BaseModel from sample_validation.const import WORKER_COMPLETED -from sample_validation.discovery import DiscoveryResult from sample_validation.models import ( ExecutionResult, + ReplayResult, RunResult, RunStatus, SampleInfo, ValidationConfig, WorkflowCreationResult, ) +from sample_validation.playbook import ( + FileEdit, + Playbook, + PlaybookStore, + RunSpec, + compute_sample_hash, + normalize_newlines, + sample_files, +) from typing_extensions import Never logger = logging.getLogger(__name__) @@ -37,10 +46,28 @@ SKILLS_DIR = Path(__file__).parent / "skills" +class PlaybookEdit(BaseModel): + file: str + find: str + replace: str + + +class PlaybookSpec(BaseModel): + """Reproducible recipe the agent returns so future runs can skip the agent.""" + + command: list[str] + cwd: str | None = None + stdin: list[str] = [] + timeout: int = 120 + env: dict[str, str] = {} + edits: list[PlaybookEdit] = [] + + class AgentResponseFormat(BaseModel): status: str output: str error: str + playbook: PlaybookSpec | None = None @dataclass @@ -67,13 +94,34 @@ class BatchCompletion: "follow its guidance to resolve the setup and then re-run the sample.\n" "Feel free to install any required dependencies if needed.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " - "based on your analysis of the code. You do not need to consult human on what to respond.\n" \ + "based on your analysis of the code. You do not need to consult human on what to respond.\n" "Fail fast and do not attempt to fix the sample unless instructed by a skill.\n" + "When (and only when) the status is `success`, also return a `playbook`: an exact, " + "non-interactive recipe that reproduces this successful run WITHOUT any AI assistance, so " + "future runs can replay it directly. The playbook must contain:\n" + " - `command`: the argv list to run, e.g. [\"python\", \"samples/01-get-started/foo.py\"]. " + "Paths must be RELATIVE TO THE python/ DIRECTORY (the Python repo root). Prefer `python` as the " + "first element (it is mapped to the active interpreter at replay time).\n" + " - `cwd`: the working directory to run from, relative to the python/ directory. Use \".\" for " + "the python/ directory itself (this is where the .env file lives).\n" + " - `stdin`: the exact list of input lines to feed for an interactive sample, in order " + "(empty list if non-interactive).\n" + " - `timeout`: a reasonable per-run timeout in seconds.\n" + " - `edits`: any in-place text replacements you had to apply to make the sample run (for " + "example replacing a hardcoded placeholder with an environment lookup). Each edit has `file` " + "(relative to the python/ directory), `find` (exact text), and `replace` (exact text). Use an " + "empty list if you changed nothing.\n" + "IMPORTANT: If you edited any sample file to make it run, do NOT revert or undo those edits. " + "Leave the file in its modified, working state after the successful run so the exact changes " + "can be recorded; the harness restores the file afterwards.\n" + "The playbook must be self-contained and deterministic: replaying `command` from `cwd` with the " + "given `stdin` after applying `edits` must reproduce the successful run.\n" "Return ONLY valid JSON with this schema:\n" "{\n" ' "status": "success|failure|missing_setup",\n' ' "output": "short summary of the result and what you did if the sample was interactive",\n' ' "error": "error details or empty string",\n' + ' "playbook": {"command": ["..."], "cwd": ".", "stdin": [], "timeout": 120, "edits": []}\n' "}\n\n" ) @@ -121,9 +169,16 @@ class CustomAgentExecutor(Executor): # Retry in case GitHub Copilot agent encounters transient errors unrelated to the sample execution. RETRY_COUNT = 1 - def __init__(self, agent: GitHubCopilotAgent): + def __init__( + self, + agent: GitHubCopilotAgent, + store: PlaybookStore | None = None, + python_root: Path | None = None, + ): super().__init__(id=agent.id) self.agent = agent + self._store = store + self._python_root = python_root self._session = agent.create_session() @handler @@ -131,6 +186,9 @@ async def handle_task( self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult] ) -> None: """Execute one sample task and notify collector + coordinator.""" + # Snapshot the sample's pristine content so we can (a) capture whatever edits the + # agent applies to make it run and (b) restore the working tree afterwards. + snapshot = self._snapshot_sample(sample) current_retry = 0 while True: try: @@ -150,6 +208,12 @@ async def handle_task( output=result_payload.output, error=result_payload.error, ) + if result.status == RunStatus.SUCCESS: + # Capture the agent's edits from disk (deterministic) before restoring, + # then compute the hash against the restored (committed) content. + captured_edits = self._capture_edits(snapshot) + self._restore_snapshot(snapshot) + self._save_playbook(sample, result_payload, captured_edits) break except Exception as ex: if current_retry < self.RETRY_COUNT: @@ -183,11 +247,91 @@ async def handle_task( ) break + # Always restore the working tree so a run never leaves the repository dirty + # (on success this is a no-op because we already restored above). + self._restore_snapshot(snapshot) + await ctx.send_message(result, target_id="collector") await ctx.send_message(WorkerFreed(worker_id=self.id), target_id="coordinator") await ctx.add_event(WorkflowEvent(WORKER_COMPLETED, sample)) # type: ignore + def _snapshot_sample(self, sample: SampleInfo) -> dict[Path, bytes]: + """Capture the current on-disk bytes of every file that makes up a sample.""" + snapshot: dict[Path, bytes] = {} + for file in sample_files(sample): + try: + snapshot[file] = file.read_bytes() + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not snapshot {file}: {ex}") + return snapshot + + def _capture_edits(self, snapshot: dict[Path, bytes]) -> list[FileEdit]: + """Return whole-file edits for any snapshotted file the agent modified on disk. + + Edit text is newline-normalized (LF) so the stored ``find``/``replace`` match + the checked-out sample regardless of the platform that replays it. + """ + if self._python_root is None: + return [] + edits: list[FileEdit] = [] + for file, original in snapshot.items(): + try: + current = file.read_bytes() + except OSError: # pragma: no cover - defensive + continue + if current == original: + continue + find = normalize_newlines(original.decode("utf-8", errors="replace")) + replace = normalize_newlines(current.decode("utf-8", errors="replace")) + if find == replace: + continue + rel = file.relative_to(self._python_root).as_posix() + edits.append(FileEdit(file=rel, find=find, replace=replace)) + return edits + + def _restore_snapshot(self, snapshot: dict[Path, bytes]) -> None: + """Restore snapshotted files to their pristine bytes if the agent changed them.""" + for file, original in snapshot.items(): + try: + if file.read_bytes() != original: + file.write_bytes(original) + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not restore {file}: {ex}") + + def _save_playbook( + self, sample: SampleInfo, payload: AgentResponseFormat, captured_edits: list[FileEdit] + ) -> None: + """Persist a cached playbook for a sample the agent validated successfully.""" + if self._store is None or payload.playbook is None: + return + spec = payload.playbook + if not spec.command: + logger.warning(f"Agent returned no replay command for {sample.relative_path}; skipping playbook.") + return + # Prefer edits captured deterministically from disk; fall back to the agent's + # self-reported edits only if we observed no on-disk change. + edits = captured_edits or [ + FileEdit(file=e.file, find=e.find, replace=e.replace) for e in spec.edits + ] + try: + playbook = Playbook( + sample=sample.relative_path, + sample_hash=compute_sample_hash(sample), + run=RunSpec( + command=list(spec.command), + cwd=spec.cwd, + stdin=list(spec.stdin), + timeout=int(spec.timeout), + env=dict(spec.env), + ), + edits=edits, + ) + path = self._store.save(playbook) + logger.info(f"Saved playbook for {sample.relative_path} -> {path}") + except Exception as ex: # pragma: no cover - defensive; never fail validation over caching + logger.warning(f"Could not save playbook for {sample.relative_path}: {ex}") + class BatchCoordinatorExecutor(Executor): """Dispatch sample tasks to worker executors in bounded batches.""" @@ -265,27 +409,39 @@ class CreateConcurrentValidationWorkflowExecutor(Executor): def __init__(self, config: ValidationConfig): super().__init__(id="create_dynamic_workflow") self.config = config + self._store = ( + PlaybookStore(config.playbooks_dir) + if config.use_cache and config.playbooks_dir is not None + else None + ) @handler async def create( self, - discovery: DiscoveryResult, + replay: ReplayResult, ctx: WorkflowContext[WorkflowCreationResult], ) -> None: """Create a nested workflow with a coordinator + worker fan-out/fan-in.""" - sample_count = len(discovery.samples) - print(f"\nCreating nested batched workflow for {sample_count} samples...") + samples = replay.remaining_samples + cached_results = replay.cached_results + sample_count = len(samples) + print( + f"\nCreating nested batched workflow for {sample_count} samples " + f"({len(cached_results)} served from cache)..." + ) if sample_count == 0: await ctx.send_message( - WorkflowCreationResult(samples=[], workflow=None, agents=[]) + WorkflowCreationResult( + samples=[], workflow=None, agents=[], cached_results=cached_results + ) ) return agents: list[GitHubCopilotAgent] = [] workers: list[CustomAgentExecutor] = [] - for index, sample in enumerate(discovery.samples, start=1): + for index, sample in enumerate(samples, start=1): agent_id = f"sample_validator_{index}({sample.relative_path})" agent = GitHubCopilotAgent( id=agent_id, @@ -294,12 +450,14 @@ async def create( context_providers=[SkillsProvider.from_paths(skill_paths=str(SKILLS_DIR))], default_options={ "on_permission_request": prompt_permission, - "timeout": 120, + "timeout": self.config.agent_timeout, }, # type: ignore ) agents.append(agent) - workers.append(CustomAgentExecutor(agent)) + workers.append( + CustomAgentExecutor(agent, store=self._store, python_root=self.config.python_root) + ) coordinator = BatchCoordinatorExecutor( worker_ids=[worker.id for worker in workers], @@ -317,8 +475,9 @@ async def create( await ctx.send_message( WorkflowCreationResult( - samples=discovery.samples, + samples=samples, workflow=nested_workflow, agents=agents, + cached_results=cached_results, ) ) diff --git a/python/scripts/sample_validation/models.py b/python/scripts/sample_validation/models.py index f5e6c40066d..3740e4d491b 100644 --- a/python/scripts/sample_validation/models.py +++ b/python/scripts/sample_validation/models.py @@ -20,6 +20,9 @@ class ValidationConfig: subdir: str | None = None exclude: list[str] | None = None max_parallel_workers: int = 10 + playbooks_dir: Path | None = None + use_cache: bool = True + agent_timeout: int = 120 @dataclass @@ -45,6 +48,19 @@ class DiscoveryResult: samples: list[SampleInfo] +@dataclass +class ReplayResult: + """Outcome of attempting to replay cached playbooks for discovered samples. + + Samples whose cached playbook replayed successfully are captured in + ``cached_results`` and skip the agent entirely. Everything else flows to the + agent-driven validation as ``remaining_samples``. + """ + + remaining_samples: list[SampleInfo] + cached_results: list["RunResult"] = field(default_factory=list) # type: ignore[assignment] + + @dataclass class WorkflowCreationResult: """Result of creating a nested per-sample concurrent workflow.""" @@ -52,6 +68,7 @@ class WorkflowCreationResult: samples: list[SampleInfo] workflow: Workflow | None agents: list[GitHubCopilotAgent] + cached_results: list["RunResult"] = field(default_factory=list) # type: ignore[assignment] class RunStatus(Enum): diff --git a/python/scripts/sample_validation/playbook.py b/python/scripts/sample_validation/playbook.py new file mode 100644 index 00000000000..6a6ab9a0722 --- /dev/null +++ b/python/scripts/sample_validation/playbook.py @@ -0,0 +1,310 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Cached validation playbooks. + +A *playbook* is a small JSON recipe that captures exactly how a sample was +successfully validated so future runs can replay it deterministically without +invoking an LLM agent. The agent produces a playbook the first time it +validates a sample; subsequent runs replay it directly and only fall back to +the agent when the playbook is missing, stale (the sample changed), or the +replay fails. + +All paths in a playbook are relative to the Python root (the ``python/`` +directory) so playbooks are portable across machines and CI runners. +""" + +import asyncio +import hashlib +import json +import logging +import os +import sys +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime +from pathlib import Path + +from sample_validation.models import RunResult, RunStatus, SampleInfo + +logger = logging.getLogger(__name__) + +# Bump when the on-disk playbook format changes in an incompatible way. +SCHEMA_VERSION = 1 + +# Hard cap on how long a replayed sample may run, regardless of the value the +# agent recorded, to protect the CI job from a runaway process. +MAX_REPLAY_TIMEOUT = 600 + + +@dataclass +class FileEdit: + """An in-place text replacement applied to a file before running.""" + + file: str + find: str + replace: str + + +@dataclass +class RunSpec: + """How to execute the sample.""" + + command: list[str] + cwd: str | None = None + stdin: list[str] = field(default_factory=list) + timeout: int = 120 + env: dict[str, str] = field(default_factory=dict) + + +@dataclass +class Playbook: + """A cached, replayable recipe for validating a single sample.""" + + sample: str + sample_hash: str + run: RunSpec + edits: list[FileEdit] = field(default_factory=list) + expected_status: str = RunStatus.SUCCESS.value + schema_version: int = SCHEMA_VERSION + generated_at: str = field(default_factory=lambda: datetime.now().isoformat()) + + def to_dict(self) -> dict[str, object]: + """Serialize to a JSON-friendly dict.""" + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, object]) -> "Playbook": + """Deserialize from a dict, tolerating unknown/missing optional keys.""" + run_data = dict(data.get("run") or {}) # type: ignore[arg-type] + run = RunSpec( + command=list(run_data.get("command") or []), + cwd=run_data.get("cwd"), + stdin=list(run_data.get("stdin") or []), + timeout=int(run_data.get("timeout") or 120), + env={str(k): str(v) for k, v in (run_data.get("env") or {}).items()}, + ) + edits = [ + FileEdit(file=str(e["file"]), find=str(e["find"]), replace=str(e["replace"])) + for e in (data.get("edits") or []) # type: ignore[union-attr] + ] + return cls( + sample=str(data["sample"]), + sample_hash=str(data["sample_hash"]), + run=run, + edits=edits, + expected_status=str(data.get("expected_status") or RunStatus.SUCCESS.value), + schema_version=int(data.get("schema_version") or SCHEMA_VERSION), + generated_at=str(data.get("generated_at") or datetime.now().isoformat()), + ) + + +def sample_files(sample: SampleInfo) -> list[Path]: + """Return the files that make up a sample. + + For a single-file sample this is just that file. For a directory sample + (``main.py``/``app.py`` entrypoint) it is every ``.py`` file in the tree. + """ + path = sample.path + if path.is_dir(): + return sorted(p for p in path.rglob("*.py") if "__pycache__" not in p.parts) + return [path] + + +def normalize_newlines(text: str) -> str: + """Normalize CRLF/CR line endings to LF so edits are portable across platforms.""" + return text.replace("\r\n", "\n").replace("\r", "\n") + + +def compute_sample_hash(sample: SampleInfo) -> str: + """Compute a stable content hash for a sample. + + For a single-file sample the hash covers that file. For a directory sample + (``main.py``/``app.py`` entrypoint) it covers every ``.py`` file in the + directory tree so any change to the sample invalidates the cached playbook. + """ + hasher = hashlib.sha256() + path = sample.path + for file in sample_files(sample): + try: + hasher.update(file.relative_to(path.parent).as_posix().encode("utf-8")) + hasher.update(b"\0") + hasher.update(file.read_bytes()) + hasher.update(b"\0") + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not hash {file}: {ex}") + + return f"sha256:{hasher.hexdigest()}" + + +class PlaybookStore: + """Loads and saves per-sample playbooks under a directory. + + Each sample gets one JSON file whose name is derived from the sample's + relative path (path separators replaced so it stays flat and filesystem + safe). + """ + + def __init__(self, directory: Path) -> None: + self.directory = directory + + @staticmethod + def _slug(relative_path: str) -> str: + return relative_path.replace("\\", "/").replace("/", "__") + ".json" + + def _path_for(self, relative_path: str) -> Path: + return self.directory / self._slug(relative_path) + + def load(self, sample: SampleInfo) -> Playbook | None: + """Return the stored playbook for a sample, or ``None`` if absent/invalid.""" + path = self._path_for(sample.relative_path) + if not path.exists(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + playbook = Playbook.from_dict(data) + except (OSError, ValueError, KeyError) as ex: + logger.warning(f"Ignoring unreadable playbook {path}: {ex}") + return None + if playbook.schema_version != SCHEMA_VERSION: + logger.info(f"Ignoring playbook with schema {playbook.schema_version} for {sample.relative_path}") + return None + return playbook + + def save(self, playbook: Playbook) -> Path: + """Persist a playbook to disk and return its path.""" + self.directory.mkdir(parents=True, exist_ok=True) + path = self._path_for(playbook.sample) + path.write_text(json.dumps(playbook.to_dict(), indent=2), encoding="utf-8") + return path + + def is_valid_for(self, sample: SampleInfo) -> Playbook | None: + """Return the stored playbook only if it matches the sample's current hash.""" + playbook = self.load(sample) + if playbook is None: + return None + current_hash = compute_sample_hash(sample) + if playbook.sample_hash != current_hash: + logger.info(f"Playbook for {sample.relative_path} is stale (sample changed); will re-validate.") + return None + return playbook + + +async def replay_playbook(playbook: Playbook, python_root: Path) -> RunResult: + """Deterministically replay a playbook and return a ``RunResult``. + + Applies any recorded file edits in place, runs the recorded command feeding + the recorded stdin, and maps a zero exit code to ``SUCCESS``. Any other + outcome yields a ``FAILURE`` so the caller falls back to the agent. The + working tree is always restored afterwards so a replay never leaves the + repository dirty (and a failed replay does not poison a later agent run). + """ + sample_path = playbook.sample + sample_info = SampleInfo(path=python_root / sample_path, relative_path=sample_path) + + # Snapshot every file we may edit so we can restore it no matter how we exit. + # Snapshot raw bytes so the restore is byte-exact (no CRLF/LF translation). + snapshot: dict[Path, bytes] = {} + for edit in playbook.edits: + target = python_root / edit.file + if target in snapshot: + continue + try: + snapshot[target] = target.read_bytes() + except OSError: + pass + + try: + # Apply edits in place. Mirrors what the agent does during validation. + for edit in playbook.edits: + target = python_root / edit.file + try: + text = target.read_text(encoding="utf-8") + except OSError as ex: + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay could not read edit target {edit.file}: {ex}", + ) + if edit.find not in text: + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay edit no longer applies to {edit.file}; playbook is stale.", + ) + target.write_text(text.replace(edit.find, edit.replace, 1), encoding="utf-8") + + command = list(playbook.run.command) + if not command: + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error="Playbook has an empty run command.", + ) + # Use the current interpreter so replay targets the same virtual environment. + if command[0] in ("python", "python3"): + command[0] = sys.executable + + cwd = python_root / (playbook.run.cwd or ".") + env = {**os.environ, **playbook.run.env} + stdin_bytes = "".join(line + "\n" for line in playbook.run.stdin).encode("utf-8") + timeout = min(max(1, playbook.run.timeout), MAX_REPLAY_TIMEOUT) + + start = time.perf_counter() + try: + proc = await asyncio.create_subprocess_exec( + *command, + cwd=str(cwd), + env=env, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + except OSError as ex: + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay could not start command {command!r}: {ex}", + ) + + try: + stdout, _ = await asyncio.wait_for(proc.communicate(stdin_bytes), timeout=timeout) + except (TimeoutError, asyncio.TimeoutError): + proc.kill() + await proc.wait() + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay timed out after {timeout}s.", + ) + + duration = time.perf_counter() - start + output_text = stdout.decode("utf-8", errors="replace") if stdout else "" + + if proc.returncode == 0: + return RunResult( + sample=sample_info, + status=RunStatus.SUCCESS, + output=f"Replayed cached playbook successfully in {duration:.1f}s.", + error="", + ) + + # Non-zero exit: surface a truncated tail to aid debugging; caller retries with the agent. + tail = output_text[-500:] + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay exited with code {proc.returncode}. Output tail: {tail}", + ) + finally: + for target, original in snapshot.items(): + try: + if target.read_bytes() != original: + target.write_bytes(original) + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not restore {target} after replay: {ex}") diff --git a/python/scripts/sample_validation/replay_executor.py b/python/scripts/sample_validation/replay_executor.py new file mode 100644 index 00000000000..6806243d6d3 --- /dev/null +++ b/python/scripts/sample_validation/replay_executor.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Executor that replays cached playbooks before falling back to the agent.""" + +import asyncio +import logging + +from agent_framework import Executor, WorkflowContext, handler + +from sample_validation.models import ( + DiscoveryResult, + ReplayResult, + RunResult, + RunStatus, + SampleInfo, + ValidationConfig, +) +from sample_validation.playbook import PlaybookStore, replay_playbook + +logger = logging.getLogger(__name__) + + +class ReplayCachedPlaybooksExecutor(Executor): + """Replay cached playbooks and split samples into cached vs. agent-bound. + + For every discovered sample we look for a playbook whose recorded hash still + matches the sample's current content. Matching playbooks are replayed + deterministically (no LLM). Samples that pass are returned as cached + results; samples without a valid playbook, or whose replay fails, are passed + on for agent-driven validation (which will refresh the playbook). + """ + + def __init__(self, config: ValidationConfig) -> None: + super().__init__(id="replay_cached_playbooks") + self.config = config + self._store = PlaybookStore(config.playbooks_dir) if config.playbooks_dir else None + + @handler + async def replay(self, discovery: DiscoveryResult, ctx: WorkflowContext[ReplayResult]) -> None: + """Attempt cached replay for each sample, then forward the remainder.""" + samples = discovery.samples + + if not self.config.use_cache or self._store is None or not samples: + await ctx.send_message(ReplayResult(remaining_samples=samples, cached_results=[])) + return + + store = self._store + candidates: list[tuple[SampleInfo, object]] = [] + remaining: list[SampleInfo] = [] + + for sample in samples: + playbook = store.is_valid_for(sample) + if playbook is None: + remaining.append(sample) + else: + candidates.append((sample, playbook)) + + print( + f"\nCache check: {len(candidates)} sample(s) have a valid playbook, " + f"{len(remaining)} need the agent." + ) + + cached_results: list[RunResult] = [] + if candidates: + semaphore = asyncio.Semaphore(max(1, self.config.max_parallel_workers)) + + async def _run(sample: SampleInfo, playbook: object) -> tuple[SampleInfo, RunResult]: + async with semaphore: + result = await replay_playbook(playbook, self.config.python_root) # type: ignore[arg-type] + return sample, result + + for coro in asyncio.as_completed([_run(s, pb) for s, pb in candidates]): + sample, result = await coro + if result.status == RunStatus.SUCCESS: + cached_results.append(result) + print(f"[cache hit] {sample.relative_path}") + else: + # Replay failed: hand the sample to the agent to re-validate and refresh the playbook. + logger.info(f"Cached replay failed for {sample.relative_path}: {result.error}") + print(f"[cache miss] {sample.relative_path} (replay failed, will use agent)") + remaining.append(sample) + + await ctx.send_message(ReplayResult(remaining_samples=remaining, cached_results=cached_results)) diff --git a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py index a895e5ddc83..143a8214caf 100644 --- a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py +++ b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py @@ -36,8 +36,11 @@ async def run( self, creation: WorkflowCreationResult, ctx: WorkflowContext[ExecutionResult] ) -> None: """Run the nested workflow and emit execution results.""" + cached_results = list(creation.cached_results) + if creation.workflow is None: - await ctx.send_message(ExecutionResult(results=[])) + # Nothing left for the agent; emit whatever the cache produced. + await ctx.send_message(ExecutionResult(results=cached_results)) return print("\nRunning nested batched workflow...") @@ -61,7 +64,9 @@ async def run( ) if result is not None: - await ctx.send_message(result) + await ctx.send_message( + ExecutionResult(results=cached_results + result.results) + ) else: fallback_results = [ RunResult( @@ -72,6 +77,8 @@ async def run( ) for sample in creation.samples ] - await ctx.send_message(ExecutionResult(results=fallback_results)) + await ctx.send_message( + ExecutionResult(results=cached_results + fallback_results) + ) finally: await stop_agents(creation.agents) diff --git a/python/scripts/sample_validation/workflow.py b/python/scripts/sample_validation/workflow.py index 10187c069be..49ded23aa4a 100644 --- a/python/scripts/sample_validation/workflow.py +++ b/python/scripts/sample_validation/workflow.py @@ -12,6 +12,7 @@ CreateConcurrentValidationWorkflowExecutor, ) from sample_validation.discovery import DiscoverSamplesExecutor, ValidationConfig +from sample_validation.replay_executor import ReplayCachedPlaybooksExecutor from sample_validation.report import GenerateReportExecutor from sample_validation.run_dynamic_validation_workflow_executor import ( RunDynamicValidationWorkflowExecutor, @@ -31,13 +32,15 @@ def create_validation_workflow( Configured Workflow instance """ discover = DiscoverSamplesExecutor(config) + replay = ReplayCachedPlaybooksExecutor(config) create_dynamic_workflow = CreateConcurrentValidationWorkflowExecutor(config) run_dynamic_workflow = RunDynamicValidationWorkflowExecutor() generate = GenerateReportExecutor() return ( WorkflowBuilder(start_executor=discover) - .add_edge(discover, create_dynamic_workflow) + .add_edge(discover, replay) + .add_edge(replay, create_dynamic_workflow) .add_edge(create_dynamic_workflow, run_dynamic_workflow) .add_edge(run_dynamic_workflow, generate) .build() From 5324213fe9983c2ffc507472b188a189d2742ecd Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 21 Jul 2026 17:12:15 -0700 Subject: [PATCH 15/28] Exclude hidden folder in sample discovery --- .github/workflows/python-sample-validation.yml | 4 ++-- .../orchestrations/04_magentic_one.py | 4 ++-- .../single_agent/02_agent_with_tool.py | 12 ++++++++++++ python/scripts/sample_validation/discovery.py | 11 ++++++++--- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 5be84fd0034..2a502013cb2 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -789,8 +789,8 @@ jobs: TOOLBOX_ENDPOINT: ${{ vars.TOOLBOX_ENDPOINT }} FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_NAME }} MEMORY_STORE_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_MEMORY_STORE }} - AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }} - AZURE_SEARCH_INDEX_NAME: ${{ secrets.FOUNDRY_HOSTED_AGENT_SEARCH_INDEX_NAME }} + AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }} + AZURE_SEARCH_INDEX_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_SEARCH_INDEX_NAME }} defaults: run: working-directory: python diff --git a/python/samples/autogen-migration/orchestrations/04_magentic_one.py b/python/samples/autogen-migration/orchestrations/04_magentic_one.py index 9a7dec30eee..c71a2a8196a 100644 --- a/python/samples/autogen-migration/orchestrations/04_magentic_one.py +++ b/python/samples/autogen-migration/orchestrations/04_magentic_one.py @@ -62,7 +62,7 @@ async def run_autogen() -> None: team = MagenticOneGroupChat( participants=[researcher, coder, reviewer], model_client=client, # Coordinator uses this client - max_turns=20, + max_turns=10, max_stalls=3, ) @@ -109,7 +109,7 @@ async def run_agent_framework() -> None: instructions="You coordinate a team to complete complex tasks efficiently.", description="Orchestrator for team coordination", ), - max_round_count=20, + max_round_count=10, max_stall_count=3, max_reset_count=1, ).build() diff --git a/python/samples/autogen-migration/single_agent/02_agent_with_tool.py b/python/samples/autogen-migration/single_agent/02_agent_with_tool.py index 75cebac2f45..8cae2710e2f 100644 --- a/python/samples/autogen-migration/single_agent/02_agent_with_tool.py +++ b/python/samples/autogen-migration/single_agent/02_agent_with_tool.py @@ -1,3 +1,15 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "agent-framework-openai", +# "autogen-agentchat", +# "autogen-ext[openai]", +# "python-dotenv", +# ] +# /// +# Run with any PEP 723 compatible runner, e.g.: +# uv run samples/autogen-migration/single_agent/02_agent_with_tool.py + # Copyright (c) Microsoft. All rights reserved. import asyncio diff --git a/python/scripts/sample_validation/discovery.py b/python/scripts/sample_validation/discovery.py index 1b613c1f5b5..492843ada56 100644 --- a/python/scripts/sample_validation/discovery.py +++ b/python/scripts/sample_validation/discovery.py @@ -84,18 +84,23 @@ def discover_samples( # Walk through all subdirectories and find .py files for root, dirs, files in os.walk(search_dir): - # Skip directories that start with _, __pycache__, or excluded paths + # Skip directories that start with _ or ., __pycache__, virtual envs, or excluded paths. + # Dot-directories (e.g. .venv) may be created in a sample folder during validation and + # must never be treated as samples. dirs[:] = [ d for d in dirs if not d.startswith("_") - and d != "__pycache__" + and not d.startswith(".") + and d not in ("__pycache__", "venv", "node_modules") and (Path(root) / d).resolve() not in exclude_paths ] - # If the whole directory is a sample, add the directory itself and skip its subdirectories + # If the whole directory is a sample, add the directory itself and do NOT descend into + # it: everything under a main.py/app.py entry point belongs to that one sample. if any(file in ("main.py", "app.py") for file in files): samples.append(Path(root)) + dirs[:] = [] continue for file in files: From 9ab10bbdd63d0f2060d09fb26e864d3f836a0c71 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 21 Jul 2026 17:47:31 -0700 Subject: [PATCH 16/28] Install autogen dependencies --- .../sample-validation-setup/action.yml | 13 + .../workflows/python-sample-validation.yml | 286 +----------------- 2 files changed, 15 insertions(+), 284 deletions(-) diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml index c9d2d2d6ac6..a040cb1cba0 100644 --- a/.github/actions/sample-validation-setup/action.yml +++ b/.github/actions/sample-validation-setup/action.yml @@ -48,3 +48,16 @@ runs: with: python-version: ${{ inputs.python-version }} os: ${{ inputs.os }} + + - name: Cache sample playbooks + # Combined cache action: restores here and registers a post-job hook to save + # automatically. Keyed per job so each validate-* job keeps its own playbooks. + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + # Must match the sample_validation --playbooks-dir default, which resolves to + # samples/sample_validation/playbooks (see python/scripts/sample_validation/__main__.py). + # If a job overrides --playbooks-dir, update this path to match. + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 2a502013cb2..a3da194b72a 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -44,25 +44,10 @@ jobs: echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -122,25 +107,10 @@ jobs: echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env echo "GITHUB_PAT=$GITHUB_PAT" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers harness tools --save-report --report-name 02-agents - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -180,25 +150,10 @@ jobs: echo "FOUNDRY_EMBEDDING_MODEL=$FOUNDRY_EMBEDDING_MODEL" >> .env echo "FOUNDRY_MEMORY_STORE=$FOUNDRY_MEMORY_STORE" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/harness --save-report --report-name 02-agents-harness - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -233,25 +188,10 @@ jobs: echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/tools --save-report --report-name 02-agents-tools - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -290,25 +230,10 @@ jobs: echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -345,25 +270,10 @@ jobs: echo "AZURE_OPENAI_MODEL=$AZURE_OPENAI_MODEL" >> .env echo "AZURE_OPENAI_API_VERSION=$AZURE_OPENAI_API_VERSION" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure --save-report --report-name 02-agents-azure - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -398,25 +308,10 @@ jobs: echo "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" >> .env echo "ANTHROPIC_CHAT_MODEL=$ANTHROPIC_CHAT_MODEL" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -443,25 +338,10 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -490,25 +370,10 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -537,25 +402,10 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -594,25 +444,10 @@ jobs: echo "FOUNDRY_AGENT_NAME=$FOUNDRY_AGENT_NAME" >> .env echo "FOUNDRY_AGENT_VERSION=$FOUNDRY_AGENT_VERSION" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry --save-report --report-name 02-agents-foundry - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -651,25 +486,10 @@ jobs: echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -695,25 +515,10 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -748,25 +553,10 @@ jobs: echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -805,26 +595,11 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation # Maximum parallel workers is set to 1 because all samples use the same port run: | cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 --agent-timeout 1800 - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -856,25 +631,10 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 04-hosting --exclude foundry-hosted-agents --save-report --report-name 04-hosting-other - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -914,25 +674,10 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -979,25 +724,13 @@ jobs: echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- + - name: Pre-install AutoGen dependencies for migration samples + run: uv pip install "autogen-agentchat" "autogen-ext[openai]" - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -1059,25 +792,10 @@ jobs: echo "COPILOTSTUDIOAGENT__TENANTID=$COPILOTSTUDIOAGENT__TENANTID" >> .env echo "COPILOTSTUDIOAGENT__AGENTAPPID=$COPILOTSTUDIOAGENT__AGENTAPPID" >> .env - - name: Restore sample playbooks - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - restore-keys: | - sample-playbooks-${{ github.job }}- - - name: Run sample validation run: | cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration - - name: Save sample playbooks - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - if: always() - with: - path: python/samples/sample_validation/playbooks/ - key: sample-playbooks-${{ github.job }}-${{ github.run_id }} - - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() From 0087a9f005e88accaf9a8ca994fceca404809270 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Wed, 22 Jul 2026 11:34:12 -0700 Subject: [PATCH 17/28] Grant azure search RBAC role --- .../workflows/python-sample-validation.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index a3da194b72a..f075d55a784 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -595,6 +595,26 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Grant Azure AI Search access to the CI identity + shell: bash + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + run: | + set -euo pipefail + # Derive the search service name from the endpoint (https://.search.windows.net) + service_name="$(echo "$AZURE_SEARCH_ENDPOINT" | sed -E 's#^https?://([^.]+)\..*#\1#')" + # Look up the full resource ID so we don't need a hardcoded resource group + scope="$(az resource list --name "$service_name" --resource-type Microsoft.Search/searchServices --query "[0].id" -o tsv)" + if [ -z "$scope" ]; then + echo "Could not resolve resource ID for search service '$service_name'" >&2 + exit 1 + fi + # Idempotent: az returns the existing assignment if it already exists + az role assignment create \ + --assignee "$AZURE_CLIENT_ID" \ + --role "Search Index Data Reader" \ + --scope "$scope" + - name: Run sample validation # Maximum parallel workers is set to 1 because all samples use the same port run: | From 98f5262a584628626146a5c7719d2cdf50981299 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Wed, 22 Jul 2026 11:37:18 -0700 Subject: [PATCH 18/28] Increase timeout for magentic --- .github/workflows/python-sample-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index f075d55a784..ee1e0e7d2ce 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -749,7 +749,7 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration + cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration --agent-timeout 600 - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 From 5b1f742d5fbf9e24135ce1eef221bdc9ac3c6417 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Wed, 22 Jul 2026 14:12:27 -0700 Subject: [PATCH 19/28] Build search resouce id deterministically --- .github/workflows/python-sample-validation.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index ee1e0e7d2ce..16cc90e3000 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -581,6 +581,7 @@ jobs: MEMORY_STORE_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_MEMORY_STORE }} AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }} AZURE_SEARCH_INDEX_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_SEARCH_INDEX_NAME }} + AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} defaults: run: working-directory: python @@ -603,12 +604,15 @@ jobs: set -euo pipefail # Derive the search service name from the endpoint (https://.search.windows.net) service_name="$(echo "$AZURE_SEARCH_ENDPOINT" | sed -E 's#^https?://([^.]+)\..*#\1#')" - # Look up the full resource ID so we don't need a hardcoded resource group - scope="$(az resource list --name "$service_name" --resource-type Microsoft.Search/searchServices --query "[0].id" -o tsv)" - if [ -z "$scope" ]; then - echo "Could not resolve resource ID for search service '$service_name'" >&2 + if [ -z "$AZURE_RESOURCE_GROUP" ]; then + echo "AZURE_RESOURCE_GROUP variable is not set" >&2 exit 1 fi + # Build the resource ID deterministically. The CI identity can create role + # assignments but may not have resource-read permission to enumerate resources, + # so avoid 'az resource list' and construct the scope directly. + subscription_id="$(az account show --query id -o tsv)" + scope="/subscriptions/${subscription_id}/resourceGroups/${AZURE_RESOURCE_GROUP}/providers/Microsoft.Search/searchServices/${service_name}" # Idempotent: az returns the existing assignment if it already exists az role assignment create \ --assignee "$AZURE_CLIENT_ID" \ From a6fb9b457b64ab27ff1d7a5f5060f8e4af87fee0 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Wed, 22 Jul 2026 14:24:54 -0700 Subject: [PATCH 20/28] Remove grant in the workflow --- .../workflows/python-sample-validation.yml | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 16cc90e3000..0f9f153b313 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -581,7 +581,6 @@ jobs: MEMORY_STORE_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_MEMORY_STORE }} AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }} AZURE_SEARCH_INDEX_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_SEARCH_INDEX_NAME }} - AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} defaults: run: working-directory: python @@ -596,29 +595,6 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} - - name: Grant Azure AI Search access to the CI identity - shell: bash - env: - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - run: | - set -euo pipefail - # Derive the search service name from the endpoint (https://.search.windows.net) - service_name="$(echo "$AZURE_SEARCH_ENDPOINT" | sed -E 's#^https?://([^.]+)\..*#\1#')" - if [ -z "$AZURE_RESOURCE_GROUP" ]; then - echo "AZURE_RESOURCE_GROUP variable is not set" >&2 - exit 1 - fi - # Build the resource ID deterministically. The CI identity can create role - # assignments but may not have resource-read permission to enumerate resources, - # so avoid 'az resource list' and construct the scope directly. - subscription_id="$(az account show --query id -o tsv)" - scope="/subscriptions/${subscription_id}/resourceGroups/${AZURE_RESOURCE_GROUP}/providers/Microsoft.Search/searchServices/${service_name}" - # Idempotent: az returns the existing assignment if it already exists - az role assignment create \ - --assignee "$AZURE_CLIENT_ID" \ - --role "Search Index Data Reader" \ - --scope "$scope" - - name: Run sample validation # Maximum parallel workers is set to 1 because all samples use the same port run: | From eba964cb9931ea3c9fc75b981224d0f0dc5782fd Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Wed, 22 Jul 2026 15:44:00 -0700 Subject: [PATCH 21/28] Move azure cli login closer to when the sample actually runs --- .github/actions/sample-validation-setup/action.yml | 14 +++++++------- .github/workflows/python-sample-validation.yml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml index a040cb1cba0..03ed52316f5 100644 --- a/.github/actions/sample-validation-setup/action.yml +++ b/.github/actions/sample-validation-setup/action.yml @@ -36,13 +36,6 @@ runs: shell: bash run: copilot --version && copilot -p "What can you do in one sentence?" - - name: Azure CLI Login - uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2 - with: - client-id: ${{ inputs.azure-client-id }} - tenant-id: ${{ inputs.azure-tenant-id }} - subscription-id: ${{ inputs.azure-subscription-id }} - - name: Set up python and install the project uses: ./.github/actions/python-setup with: @@ -61,3 +54,10 @@ runs: key: sample-playbooks-${{ github.job }}-${{ github.run_id }} restore-keys: | sample-playbooks-${{ github.job }}- + + - name: Azure CLI Login + uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2 + with: + client-id: ${{ inputs.azure-client-id }} + tenant-id: ${{ inputs.azure-tenant-id }} + subscription-id: ${{ inputs.azure-subscription-id }} diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 0f9f153b313..2b2551f02e3 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -598,7 +598,7 @@ jobs: - name: Run sample validation # Maximum parallel workers is set to 1 because all samples use the same port run: | - cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 --agent-timeout 1800 + cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 From a6bad9d1557b18fe5883bdc1aac7bae36ddec9c7 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 23 Jul 2026 13:22:01 -0700 Subject: [PATCH 22/28] Refactor playbook --- .../create_dynamic_workflow_executor.py | 132 ++++------ python/scripts/sample_validation/playbook.py | 242 ++++++++++-------- .../foundry-hosted-agent-validation/SKILL.md | 31 +++ 3 files changed, 218 insertions(+), 187 deletions(-) diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index d98e6734302..249ae1aac69 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -30,12 +30,9 @@ WorkflowCreationResult, ) from sample_validation.playbook import ( - FileEdit, Playbook, PlaybookStore, - RunSpec, compute_sample_hash, - normalize_newlines, sample_files, ) from typing_extensions import Never @@ -46,28 +43,24 @@ SKILLS_DIR = Path(__file__).parent / "skills" -class PlaybookEdit(BaseModel): - file: str - find: str - replace: str +class PlaybookScript(BaseModel): + """Reproducible Python validation script the agent returns so future runs can skip the agent. + ``script`` is a complete, self-contained Python program that reproduces a successful + validation without any AI assistance. The harness runs it with the active interpreter from + the Python root and treats a zero exit code as success. + """ -class PlaybookSpec(BaseModel): - """Reproducible recipe the agent returns so future runs can skip the agent.""" - - command: list[str] - cwd: str | None = None - stdin: list[str] = [] + script: str timeout: int = 120 env: dict[str, str] = {} - edits: list[PlaybookEdit] = [] class AgentResponseFormat(BaseModel): status: str output: str error: str - playbook: PlaybookSpec | None = None + playbook: PlaybookScript | None = None @dataclass @@ -96,32 +89,40 @@ class BatchCompletion: "The sample can be interactive. If it is interactive, respond to the sample when prompted " "based on your analysis of the code. You do not need to consult human on what to respond.\n" "Fail fast and do not attempt to fix the sample unless instructed by a skill.\n" - "When (and only when) the status is `success`, also return a `playbook`: an exact, " - "non-interactive recipe that reproduces this successful run WITHOUT any AI assistance, so " - "future runs can replay it directly. The playbook must contain:\n" - " - `command`: the argv list to run, e.g. [\"python\", \"samples/01-get-started/foo.py\"]. " - "Paths must be RELATIVE TO THE python/ DIRECTORY (the Python repo root). Prefer `python` as the " - "first element (it is mapped to the active interpreter at replay time).\n" - " - `cwd`: the working directory to run from, relative to the python/ directory. Use \".\" for " - "the python/ directory itself (this is where the .env file lives).\n" - " - `stdin`: the exact list of input lines to feed for an interactive sample, in order " - "(empty list if non-interactive).\n" - " - `timeout`: a reasonable per-run timeout in seconds.\n" - " - `edits`: any in-place text replacements you had to apply to make the sample run (for " - "example replacing a hardcoded placeholder with an environment lookup). Each edit has `file` " - "(relative to the python/ directory), `find` (exact text), and `replace` (exact text). Use an " - "empty list if you changed nothing.\n" - "IMPORTANT: If you edited any sample file to make it run, do NOT revert or undo those edits. " - "Leave the file in its modified, working state after the successful run so the exact changes " - "can be recorded; the harness restores the file afterwards.\n" - "The playbook must be self-contained and deterministic: replaying `command` from `cwd` with the " - "given `stdin` after applying `edits` must reproduce the successful run.\n" + "When (and only when) the status is `success`, also return a `playbook`: a self-contained " + "Python script that reproduces this successful validation WITHOUT any AI assistance, so future " + "runs can replay it deterministically. `playbook.script` is a COMPLETE Python program with " + "these rules:\n" + " - It is run with the SAME Python interpreter, and its WORKING DIRECTORY is the python/ " + "directory (the repo's Python root, where the shared .env lives). Reference the sample with a " + 'path relative to python/, e.g. "samples/04-hosting/foundry-hosted-agents/responses/01_basic".\n' + " - It MUST exit 0 when validation passes and exit non-zero (raise or sys.exit(1)) when it " + "fails. The harness maps exit code 0 to success and anything else to failure.\n" + " - It may import ONLY: the sample's own already-installed dependencies, the Python standard " + "library, and `httpx`. Do not require any other third-party package.\n" + " - It must be non-interactive and deterministic (no prompts, no reliance on you).\n" + " - For a normal run-to-completion sample: launch it (e.g. subprocess.run([sys.executable, " + '"/main.py"], ...) feeding any needed stdin via input=), then assert on the exit code ' + "and/or expected output.\n" + " - For a LONG-LIVED SERVER sample (an HTTP server that never returns on its own, e.g. a " + "hosted agent listening on a port): start it in the BACKGROUND (subprocess.Popen), POLL until " + "the port accepts connections (bounded readiness loop), send the real HTTP request(s) with " + "httpx, and assert HTTP 200 plus a non-empty/expected response. When the sample keeps " + "conversational state, exercise MULTIPLE TURNS (e.g. turn 1 provides a fact like a name, turn 2 " + "asks it to be recalled) and assert the recall. ALWAYS terminate the server in a finally block " + "so no process or port is leaked (the harness also force-kills the process group as a backstop).\n" + " - If you had to edit any sample file to make it run, BAKE that edit into the script (apply it " + "in code before running); the harness restores sample files after replay.\n" + " - On failure, print the captured server/sample output before exiting non-zero to aid debugging.\n" + "Also set `playbook.timeout` (whole-run seconds; the harness caps it) and optional " + "`playbook.env` (extra environment overrides; the CI environment already provides the real " + "credentials, so usually leave this empty).\n" "Return ONLY valid JSON with this schema:\n" "{\n" ' "status": "success|failure|missing_setup",\n' ' "output": "short summary of the result and what you did if the sample was interactive",\n' ' "error": "error details or empty string",\n' - ' "playbook": {"command": ["..."], "cwd": ".", "stdin": [], "timeout": 120, "edits": []}\n' + ' "playbook": {"script": "", "timeout": 120, "env": {}}\n' "}\n\n" ) @@ -186,8 +187,8 @@ async def handle_task( self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult] ) -> None: """Execute one sample task and notify collector + coordinator.""" - # Snapshot the sample's pristine content so we can (a) capture whatever edits the - # agent applies to make it run and (b) restore the working tree afterwards. + # Snapshot the sample's pristine content so we can restore the working tree afterwards + # (the agent, or a baked-in playbook edit, may modify sample files while validating). snapshot = self._snapshot_sample(sample) current_retry = 0 while True: @@ -209,11 +210,10 @@ async def handle_task( error=result_payload.error, ) if result.status == RunStatus.SUCCESS: - # Capture the agent's edits from disk (deterministic) before restoring, - # then compute the hash against the restored (committed) content. - captured_edits = self._capture_edits(snapshot) + # Restore the sample to its committed content before hashing so the playbook's + # sample_hash matches what a fresh checkout (and future staleness check) sees. self._restore_snapshot(snapshot) - self._save_playbook(sample, result_payload, captured_edits) + self._save_playbook(sample, result_payload) break except Exception as ex: if current_retry < self.RETRY_COUNT: @@ -266,30 +266,6 @@ def _snapshot_sample(self, sample: SampleInfo) -> dict[Path, bytes]: logger.warning(f"Could not snapshot {file}: {ex}") return snapshot - def _capture_edits(self, snapshot: dict[Path, bytes]) -> list[FileEdit]: - """Return whole-file edits for any snapshotted file the agent modified on disk. - - Edit text is newline-normalized (LF) so the stored ``find``/``replace`` match - the checked-out sample regardless of the platform that replays it. - """ - if self._python_root is None: - return [] - edits: list[FileEdit] = [] - for file, original in snapshot.items(): - try: - current = file.read_bytes() - except OSError: # pragma: no cover - defensive - continue - if current == original: - continue - find = normalize_newlines(original.decode("utf-8", errors="replace")) - replace = normalize_newlines(current.decode("utf-8", errors="replace")) - if find == replace: - continue - rel = file.relative_to(self._python_root).as_posix() - edits.append(FileEdit(file=rel, find=find, replace=replace)) - return edits - def _restore_snapshot(self, snapshot: dict[Path, bytes]) -> None: """Restore snapshotted files to their pristine bytes if the agent changed them.""" for file, original in snapshot.items(): @@ -299,33 +275,21 @@ def _restore_snapshot(self, snapshot: dict[Path, bytes]) -> None: except OSError as ex: # pragma: no cover - defensive logger.warning(f"Could not restore {file}: {ex}") - def _save_playbook( - self, sample: SampleInfo, payload: AgentResponseFormat, captured_edits: list[FileEdit] - ) -> None: + def _save_playbook(self, sample: SampleInfo, payload: AgentResponseFormat) -> None: """Persist a cached playbook for a sample the agent validated successfully.""" if self._store is None or payload.playbook is None: return spec = payload.playbook - if not spec.command: - logger.warning(f"Agent returned no replay command for {sample.relative_path}; skipping playbook.") + if not spec.script.strip(): + logger.warning(f"Agent returned no replay script for {sample.relative_path}; skipping playbook.") return - # Prefer edits captured deterministically from disk; fall back to the agent's - # self-reported edits only if we observed no on-disk change. - edits = captured_edits or [ - FileEdit(file=e.file, find=e.find, replace=e.replace) for e in spec.edits - ] try: playbook = Playbook( sample=sample.relative_path, sample_hash=compute_sample_hash(sample), - run=RunSpec( - command=list(spec.command), - cwd=spec.cwd, - stdin=list(spec.stdin), - timeout=int(spec.timeout), - env=dict(spec.env), - ), - edits=edits, + script=spec.script, + timeout=int(spec.timeout), + env=dict(spec.env), ) path = self._store.save(playbook) logger.info(f"Saved playbook for {sample.relative_path} -> {path}") diff --git a/python/scripts/sample_validation/playbook.py b/python/scripts/sample_validation/playbook.py index 6a6ab9a0722..7a8a48bae57 100644 --- a/python/scripts/sample_validation/playbook.py +++ b/python/scripts/sample_validation/playbook.py @@ -9,8 +9,19 @@ the agent when the playbook is missing, stale (the sample changed), or the replay fails. -All paths in a playbook are relative to the Python root (the ``python/`` -directory) so playbooks are portable across machines and CI runners. +The playbook payload is an agent-authored **Python validation script**. Replaying +a playbook writes that script to a temporary file and runs it with the active +interpreter, treating a zero exit code as success. A single script can validate +any sample shape: a plain sample that runs to completion, or a long-lived server +sample (e.g. a hosted agent) that must be started in the background, exercised +over HTTP, asserted, and then torn down. + +The script is a cache-only artifact (it is never committed to the repository), so +it is exempt from the repository's Python style/type rules; the harness treats it +as an opaque, self-contained program. Any paths the script references should be +relative to the Python root (the ``python/`` directory), which is the working +directory used at replay time, so playbooks stay portable across machines and CI +runners. """ import asyncio @@ -18,7 +29,10 @@ import json import logging import os +import signal +import subprocess import sys +import tempfile import time from dataclasses import asdict, dataclass, field from datetime import datetime @@ -29,41 +43,31 @@ logger = logging.getLogger(__name__) # Bump when the on-disk playbook format changes in an incompatible way. -SCHEMA_VERSION = 1 +# v2: the payload is an agent-authored Python script (replaces the v1 RunSpec). +SCHEMA_VERSION = 2 # Hard cap on how long a replayed sample may run, regardless of the value the # agent recorded, to protect the CI job from a runaway process. MAX_REPLAY_TIMEOUT = 600 -@dataclass -class FileEdit: - """An in-place text replacement applied to a file before running.""" - - file: str - find: str - replace: str - - -@dataclass -class RunSpec: - """How to execute the sample.""" - - command: list[str] - cwd: str | None = None - stdin: list[str] = field(default_factory=list) - timeout: int = 120 - env: dict[str, str] = field(default_factory=dict) - - @dataclass class Playbook: - """A cached, replayable recipe for validating a single sample.""" + """A cached, replayable recipe for validating a single sample. + + The recipe is an agent-authored Python ``script`` that reproduces a + successful validation without any AI assistance. Replaying runs the script + with the active interpreter from the Python root and maps a zero exit code to + success. The script is self-contained: it launches/exercises the sample, + performs its own assertions, bakes in any edits it needs, and (for a server + sample) starts and tears down the server itself. + """ sample: str sample_hash: str - run: RunSpec - edits: list[FileEdit] = field(default_factory=list) + script: str + timeout: int = 120 + env: dict[str, str] = field(default_factory=dict) # type: ignore expected_status: str = RunStatus.SUCCESS.value schema_version: int = SCHEMA_VERSION generated_at: str = field(default_factory=lambda: datetime.now().isoformat()) @@ -74,26 +78,20 @@ def to_dict(self) -> dict[str, object]: @classmethod def from_dict(cls, data: dict[str, object]) -> "Playbook": - """Deserialize from a dict, tolerating unknown/missing optional keys.""" - run_data = dict(data.get("run") or {}) # type: ignore[arg-type] - run = RunSpec( - command=list(run_data.get("command") or []), - cwd=run_data.get("cwd"), - stdin=list(run_data.get("stdin") or []), - timeout=int(run_data.get("timeout") or 120), - env={str(k): str(v) for k, v in (run_data.get("env") or {}).items()}, - ) - edits = [ - FileEdit(file=str(e["file"]), find=str(e["find"]), replace=str(e["replace"])) - for e in (data.get("edits") or []) # type: ignore[union-attr] - ] + """Deserialize from a dict, tolerating unknown/missing optional keys. + + Tolerant of older schema versions (which lack ``script``) so + ``PlaybookStore.load`` can read them and then reject them on the schema + version check rather than raising. + """ return cls( sample=str(data["sample"]), sample_hash=str(data["sample_hash"]), - run=run, - edits=edits, + script=str(data.get("script") or ""), + timeout=int(data.get("timeout") or 120), # type: ignore[arg-type] + env={str(k): str(v) for k, v in (data.get("env") or {}).items()}, # type: ignore[union-attr] expected_status=str(data.get("expected_status") or RunStatus.SUCCESS.value), - schema_version=int(data.get("schema_version") or SCHEMA_VERSION), + schema_version=int(data.get("schema_version") or SCHEMA_VERSION), # type: ignore[arg-type] generated_at=str(data.get("generated_at") or datetime.now().isoformat()), ) @@ -110,11 +108,6 @@ def sample_files(sample: SampleInfo) -> list[Path]: return [path] -def normalize_newlines(text: str) -> str: - """Normalize CRLF/CR line endings to LF so edits are portable across platforms.""" - return text.replace("\r\n", "\n").replace("\r", "\n") - - def compute_sample_hash(sample: SampleInfo) -> str: """Compute a stable content hash for a sample. @@ -189,92 +182,123 @@ def is_valid_for(self, sample: SampleInfo) -> Playbook | None: return playbook +def _new_process_group_kwargs() -> dict[str, object]: + """Return subprocess kwargs that launch the child in its own process group. + + Isolating the replay in a new group/session lets us tear down the whole tree + (for example a background server the script spawned) even if the script exits + without cleaning up after itself. + """ + if os.name == "posix": + return {"start_new_session": True} + # Windows: a new process group so the whole tree can be terminated together. + return {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} + + +def _terminate_process_tree(proc: "asyncio.subprocess.Process", pgid: int | None) -> None: + """Best-effort kill of a replay process and any descendants it leaked. + + This always runs, even when the script itself already exited cleanly, because + the whole point of the backstop is to reap a background server the script may + have started but failed to tear down. On POSIX we signal the saved process + group id (captured at spawn time, since the leader may already be reaped); + on Windows we kill the process tree by PID. + """ + try: + if os.name == "posix": + if pgid is not None: + os.killpg(pgid, signal.SIGKILL) + elif proc.returncode is None: + proc.kill() + else: + # Kill by PID (not image name) including the whole child tree. + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], + capture_output=True, + check=False, + ) + except (ProcessLookupError, OSError) as ex: # pragma: no cover - defensive + logger.debug(f"Could not terminate replay process tree {proc.pid}: {ex}") + + async def replay_playbook(playbook: Playbook, python_root: Path) -> RunResult: """Deterministically replay a playbook and return a ``RunResult``. - Applies any recorded file edits in place, runs the recorded command feeding - the recorded stdin, and maps a zero exit code to ``SUCCESS``. Any other - outcome yields a ``FAILURE`` so the caller falls back to the agent. The - working tree is always restored afterwards so a replay never leaves the - repository dirty (and a failed replay does not poison a later agent run). + Writes the recorded Python script to a temporary file and runs it with the + active interpreter, using the Python root as the working directory. A zero + exit code maps to ``SUCCESS``; anything else yields a ``FAILURE`` so the + caller falls back to the agent. + + Sample files are snapshotted and restored so a replay never leaves the + repository dirty (the script may edit the sample to make it run), and the + replay process is launched in its own process group so a background server it + starts is always torn down afterwards, even if the script fails to clean up. """ sample_path = playbook.sample sample_info = SampleInfo(path=python_root / sample_path, relative_path=sample_path) - # Snapshot every file we may edit so we can restore it no matter how we exit. - # Snapshot raw bytes so the restore is byte-exact (no CRLF/LF translation). + if not playbook.script.strip(): + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error="Playbook has an empty validation script.", + ) + + # Snapshot every file that makes up the sample so we can restore it byte-exact + # no matter how we exit (the script is allowed to edit the sample in place). snapshot: dict[Path, bytes] = {} - for edit in playbook.edits: - target = python_root / edit.file - if target in snapshot: - continue + for file in sample_files(sample_info): try: - snapshot[target] = target.read_bytes() + snapshot[file] = file.read_bytes() except OSError: pass + script_file = tempfile.NamedTemporaryFile( # noqa: SIM115 - closed explicitly below + prefix="playbook_", suffix=".py", delete=False + ) + script_path = Path(script_file.name) + proc: asyncio.subprocess.Process | None = None + pgid: int | None = None try: - # Apply edits in place. Mirrors what the agent does during validation. - for edit in playbook.edits: - target = python_root / edit.file - try: - text = target.read_text(encoding="utf-8") - except OSError as ex: - return RunResult( - sample=sample_info, - status=RunStatus.FAILURE, - output="", - error=f"Replay could not read edit target {edit.file}: {ex}", - ) - if edit.find not in text: - return RunResult( - sample=sample_info, - status=RunStatus.FAILURE, - output="", - error=f"Replay edit no longer applies to {edit.file}; playbook is stale.", - ) - target.write_text(text.replace(edit.find, edit.replace, 1), encoding="utf-8") - - command = list(playbook.run.command) - if not command: - return RunResult( - sample=sample_info, - status=RunStatus.FAILURE, - output="", - error="Playbook has an empty run command.", - ) - # Use the current interpreter so replay targets the same virtual environment. - if command[0] in ("python", "python3"): - command[0] = sys.executable + script_file.write(playbook.script.encode("utf-8")) + script_file.close() - cwd = python_root / (playbook.run.cwd or ".") - env = {**os.environ, **playbook.run.env} - stdin_bytes = "".join(line + "\n" for line in playbook.run.stdin).encode("utf-8") - timeout = min(max(1, playbook.run.timeout), MAX_REPLAY_TIMEOUT) + env = {**os.environ, **playbook.env} + timeout = min(max(1, playbook.timeout), MAX_REPLAY_TIMEOUT) start = time.perf_counter() try: proc = await asyncio.create_subprocess_exec( - *command, - cwd=str(cwd), + sys.executable, + str(script_path), + cwd=str(python_root), env=env, - stdin=asyncio.subprocess.PIPE, + stdin=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, + **_new_process_group_kwargs(), # type: ignore[arg-type] ) except OSError as ex: return RunResult( sample=sample_info, status=RunStatus.FAILURE, output="", - error=f"Replay could not start command {command!r}: {ex}", + error=f"Replay could not start the script: {ex}", ) + # Capture the process-group id now (== proc.pid, since we start a new + # session/group) so teardown can kill leaked descendants even after the + # leader process has exited and been reaped. + if os.name == "posix": + try: + pgid = os.getpgid(proc.pid) + except OSError: # pragma: no cover - defensive + pgid = None + try: - stdout, _ = await asyncio.wait_for(proc.communicate(stdin_bytes), timeout=timeout) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout) except (TimeoutError, asyncio.TimeoutError): - proc.kill() - await proc.wait() return RunResult( sample=sample_info, status=RunStatus.FAILURE, @@ -294,7 +318,7 @@ async def replay_playbook(playbook: Playbook, python_root: Path) -> RunResult: ) # Non-zero exit: surface a truncated tail to aid debugging; caller retries with the agent. - tail = output_text[-500:] + tail = output_text[-1000:] return RunResult( sample=sample_info, status=RunStatus.FAILURE, @@ -302,9 +326,21 @@ async def replay_playbook(playbook: Playbook, python_root: Path) -> RunResult: error=f"Replay exited with code {proc.returncode}. Output tail: {tail}", ) finally: + # Always tear down the process group (even if the script exited cleanly) + # so a background server it leaked is killed, then reap the leader. + if proc is not None: + _terminate_process_tree(proc, pgid) + try: + await proc.wait() + except Exception: # pragma: no cover - defensive + pass for target, original in snapshot.items(): try: if target.read_bytes() != original: target.write_bytes(original) except OSError as ex: # pragma: no cover - defensive logger.warning(f"Could not restore {target} after replay: {ex}") + try: + script_path.unlink() + except OSError: # pragma: no cover - defensive + pass diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md index dd69d3ff7ba..018953b75ca 100644 --- a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -152,6 +152,37 @@ PowerShell: use `Invoke-WebRequest -Uri http://localhost:8088/responses -Method **Pass:** HTTP 200, non-empty `output[].content[].text`, and the second turn recalls the name. Stop the server afterward. +### Recording the playbook (cached replay) + +The sample-validation harness caches a **playbook** so future runs replay your +result without an agent. For a hosted-agent sample the playbook is an +agent-authored **Python script** that reproduces **only this Phase 1 +native-local smoke test** — never the `azd`/deploy phases (those are +credential- and deployment-bound, non-deterministic, and must not enter the +replay cache). + +Emit a self-contained script that the harness runs with the active interpreter +from the `python/` directory (exit code 0 = success). It may import only the +sample's own installed deps, the Python stdlib, and `httpx`. It must: + +1. Start the sample server in the **background** with + `subprocess.Popen([sys.executable, "/main.py"])` (path relative to + `python/`; the shared `python/.env` and the process env supply credentials). +2. **Poll** `http://localhost:8088` until the port accepts a connection (bounded + readiness loop with a timeout), failing if it never comes up. +3. POST turn 1 to the protocol's route (`/responses` for `responses` samples; + the invocations route for `invocations` samples), capture the `response_id`, + then POST turn 2 with `previous_response_id` to confirm recall. +4. **Assert** HTTP 200, non-empty `output[].content[].text`, and that turn 2 + recalls the fact from turn 1. +5. **Always** terminate the server in a `finally` block so port 8088 is freed + (the harness force-kills the process group as a backstop, but the script must + still clean up). On any failure, print the captured server output before + exiting non-zero. + +Keep the deep three-phase validation (below) for a full manual/`azd` pass; it is +out of scope for the cached playbook. + --- ## Phase 2 — Local validation via `azd ai agent run` From 342131ce7906b28d6c00621ff16f4a1297c5bfba Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 23 Jul 2026 14:07:14 -0700 Subject: [PATCH 23/28] Fix using deployed agent sample --- .../foundry-hosted-agents/responses/using_deployed_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py index 7b8e906fda4..6512af4bf58 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py @@ -46,7 +46,7 @@ async def create_hosted_agent_session( agent_details = await project_client.agents.get(agent_name) resolved_agent_version = agent_details.versions.latest.version - service_session = await project_client.beta.agents.create_session( + service_session = await project_client.agents.create_session( agent_name, version_indicator=VersionRefIndicator(agent_version=resolved_agent_version), ) @@ -60,7 +60,7 @@ async def delete_hosted_agent_session( session: AgentSession, ) -> None: """Delete a hosted-agent service session.""" - await project_client.beta.agents.delete_session( + await project_client.agents.delete_session( agent_name, session.session_id, ) From 2ca8eb2338d1a900e13ba9d44c89a50880176ef0 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 23 Jul 2026 16:33:08 -0700 Subject: [PATCH 24/28] Actually save the playbooks --- .../action.yml | 19 +++++ .../sample-validation-setup/action.yml | 11 ++- .../workflows/python-sample-validation.yml | 76 +++++++++++++++++++ 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 .github/actions/sample-validation-save-playbooks/action.yml diff --git a/.github/actions/sample-validation-save-playbooks/action.yml b/.github/actions/sample-validation-save-playbooks/action.yml new file mode 100644 index 00000000000..22df27cd3ff --- /dev/null +++ b/.github/actions/sample-validation-save-playbooks/action.yml @@ -0,0 +1,19 @@ +name: Save Sample Playbooks +description: > + Save the cached sample-validation playbooks. Split out from + sample-validation-setup (which only restores) so the save runs even when the + validation step fails. Combining restore+save via actions/cache would skip the + save on a failing job (post-if: success()), so freshly authored playbooks for + samples that failed validation would never persist. Invoke this with + `if: ${{ !cancelled() }}` after the validation step in each job. + +runs: + using: "composite" + steps: + - name: Save sample playbooks cache + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + # Must match the restore path/key in sample-validation-setup/action.yml and the + # sample_validation --playbooks-dir default (samples/sample_validation/playbooks). + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml index 03ed52316f5..9bb1b90f555 100644 --- a/.github/actions/sample-validation-setup/action.yml +++ b/.github/actions/sample-validation-setup/action.yml @@ -42,10 +42,13 @@ runs: python-version: ${{ inputs.python-version }} os: ${{ inputs.os }} - - name: Cache sample playbooks - # Combined cache action: restores here and registers a post-job hook to save - # automatically. Keyed per job so each validate-* job keeps its own playbooks. - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Restore sample playbooks + # Restore-only. The matching save is a separate step in each job that runs with + # `if: ${{ !cancelled() }}` (see .github/actions/sample-validation-save-playbooks). + # A combined actions/cache would skip its post-job save on a failing job + # (post-if: success()), so playbooks authored for samples that failed validation + # would never persist. Keyed per job so each validate-* job keeps its own playbooks. + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: # Must match the sample_validation --playbooks-dir default, which resolves to # samples/sample_validation/playbooks (see python/scripts/sample_validation/__main__.py). diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 2b2551f02e3..8c6a7db34e8 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -48,6 +48,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -111,6 +115,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers harness tools --save-report --report-name 02-agents + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -154,6 +162,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/harness --save-report --report-name 02-agents-harness + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -192,6 +204,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/tools --save-report --report-name 02-agents-tools + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -234,6 +250,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -274,6 +294,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure --save-report --report-name 02-agents-azure + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -312,6 +336,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -342,6 +370,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -374,6 +406,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -406,6 +442,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -448,6 +488,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry --save-report --report-name 02-agents-foundry + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -490,6 +534,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -519,6 +567,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -557,6 +609,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -600,6 +656,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -635,6 +695,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 04-hosting --exclude foundry-hosted-agents --save-report --report-name 04-hosting-other + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -678,6 +742,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -731,6 +799,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration --agent-timeout 600 + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -796,6 +868,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() From 92045ef4c902bcf909a015c2d96786f220bb1076 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 23 Jul 2026 16:43:31 -0700 Subject: [PATCH 25/28] Fix action syntax error --- .github/actions/sample-validation-save-playbooks/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/sample-validation-save-playbooks/action.yml b/.github/actions/sample-validation-save-playbooks/action.yml index 22df27cd3ff..5d1ffb15afb 100644 --- a/.github/actions/sample-validation-save-playbooks/action.yml +++ b/.github/actions/sample-validation-save-playbooks/action.yml @@ -5,7 +5,7 @@ description: > validation step fails. Combining restore+save via actions/cache would skip the save on a failing job (post-if: success()), so freshly authored playbooks for samples that failed validation would never persist. Invoke this with - `if: ${{ !cancelled() }}` after the validation step in each job. + 'if: not-cancelled' after the validation step in each job. runs: using: "composite" From 5849ebc63cd946b5f4fbb5cb562911283c356ca7 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Fri, 24 Jul 2026 11:11:45 -0700 Subject: [PATCH 26/28] Fix magentic sample --- .../_magentic.py | 4 +- .../03-workflows/orchestrations/magentic.py | 42 ++++++++++--------- .../orchestrations/04_magentic_one.py | 37 ++++++++-------- python/scripts/sample_validation/__main__.py | 3 ++ .../create_dynamic_workflow_executor.py | 7 +++- 5 files changed, 53 insertions(+), 40 deletions(-) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py index 14ae940002e..d3a62997c90 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py @@ -878,6 +878,8 @@ class MagenticOrchestrator(BaseGroupChatOrchestrator): 5. The outer loop handles replanning and reenters the inner loop. """ + MANAGER_NAME: ClassVar[str] = "magentic_orchestrator" + def __init__( self, manager: MagenticManagerBase, @@ -894,7 +896,7 @@ def __init__( Keyword Args: require_plan_signoff: If True, requires human approval of the initial plan before proceeding. """ - super().__init__("magentic_orchestrator", participant_registry) + super().__init__(self.MANAGER_NAME, participant_registry) self._manager = manager self._require_plan_signoff = require_plan_signoff diff --git a/python/samples/03-workflows/orchestrations/magentic.py b/python/samples/03-workflows/orchestrations/magentic.py index 263408aa5b0..cc16eb96a05 100644 --- a/python/samples/03-workflows/orchestrations/magentic.py +++ b/python/samples/03-workflows/orchestrations/magentic.py @@ -14,6 +14,7 @@ ) from agent_framework.foundry import FoundryChatClient from agent_framework.orchestrations import GroupChatRequestSentEvent, MagenticBuilder, MagenticProgressLedger +from agent_framework_orchestrations import MagenticOrchestrator from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -113,17 +114,21 @@ async def main() -> None: print("\nStarting workflow execution...") # Keep track of the last executor to format output nicely in streaming mode - last_response_id: str | None = None + last_message_id: str | None = None output_event: WorkflowEvent | None = None async for event in workflow.run(task, stream=True): - if event.type in ("intermediate", "output") and isinstance(event.data, AgentResponseUpdate): - response_id = event.data.response_id - if response_id != last_response_id: - if last_response_id is not None: - print("\n") - print(f"- {event.executor_id}:", end=" ", flush=True) - last_response_id = response_id - print(event.data, end="", flush=True) + if event.type in ("intermediate", "output"): + if event.executor_id == MagenticOrchestrator.MANAGER_NAME: + output_event = event + else: + event_data = cast(AgentResponseUpdate, event.data) + message_id = event_data.message_id + if message_id != last_message_id: + if last_message_id is not None: + print("\n") + print(f"- {event.executor_id}:", end=" ", flush=True) + last_message_id = message_id + print(event_data, end="", flush=True) elif event.type == "magentic_orchestrator": print(f"\n[Magentic Orchestrator Event] Type: {event.data.event_type.name}") @@ -137,21 +142,20 @@ async def main() -> None: # Block to allow user to read the plan/progress before continuing # Note: this is for demonstration only and is not the recommended way to handle human interaction. # Please refer to `with_plan_review` for proper human interaction during planning phases. - await asyncio.get_event_loop().run_in_executor(None, input, "Press Enter to continue...") + # await asyncio.get_event_loop().run_in_executor(None, input, "Press Enter to continue...") elif event.type == "group_chat" and isinstance(event.data, GroupChatRequestSentEvent): print(f"\n[REQUEST SENT ({event.data.round_index})] to agent: {event.data.participant_name}") - elif event.type == "output": - output_event = event + if not output_event: + raise RuntimeError("Workflow did not produce a final output event.") - if output_event: - # The output of the magentic workflow is a collection of chat messages from all participants - outputs = cast(list[Message], output_event.data) - print("\n" + "=" * 80) - print("\nFinal Conversation Transcript:\n") - for message in outputs: - print(f"{message.author_name or message.role}: {message.text}\n") + print("\n\nWorkflow completed!") + print("Final Output:") + # The output of the Magentic workflow is an AgentResponse or AgentResponseUpdate, + # which contains the final message text. + output_message = cast(AgentResponseUpdate, output_event.data) + print(output_message.text) if __name__ == "__main__": diff --git a/python/samples/autogen-migration/orchestrations/04_magentic_one.py b/python/samples/autogen-migration/orchestrations/04_magentic_one.py index c71a2a8196a..736d07408ce 100644 --- a/python/samples/autogen-migration/orchestrations/04_magentic_one.py +++ b/python/samples/autogen-migration/orchestrations/04_magentic_one.py @@ -10,7 +10,7 @@ Message, WorkflowEvent, ) -from agent_framework.orchestrations import MagenticProgressLedger +from agent_framework.orchestrations import MagenticOrchestrator, MagenticProgressLedger from dotenv import load_dotenv """AutoGen MagenticOneGroupChat vs Agent Framework MagenticBuilder. @@ -25,7 +25,6 @@ async def run_autogen() -> None: """AutoGen's MagenticOneGroupChat for orchestrated collaboration.""" - from autogen_agentchat.agents import AssistantAgent from autogen_agentchat.teams import MagenticOneGroupChat from autogen_agentchat.ui import Console @@ -119,14 +118,18 @@ async def run_agent_framework() -> None: output_event: WorkflowEvent | None = None print("[Agent Framework] Magentic conversation:") async for event in workflow.run("Research Python async patterns and write a simple example", stream=True): - if event.type == "output" and isinstance(event.data, AgentResponseUpdate): - message_id = event.data.message_id - if message_id != last_message_id: - if last_message_id is not None: - print("\n") - print(f"- {event.executor_id}:", end=" ", flush=True) - last_message_id = message_id - print(event.data, end="", flush=True) + if event.type == "output": + if event.executor_id == MagenticOrchestrator.MANAGER_NAME: + output_event = event + else: + event_data = cast(AgentResponseUpdate, event.data) + message_id = event_data.message_id + if message_id != last_message_id: + if last_message_id is not None: + print("\n") + print(f"- {event.executor_id}:", end=" ", flush=True) + last_message_id = message_id + print(event_data, end="", flush=True) elif event.type == "magentic_orchestrator": print(f"\n[Magentic Orchestrator Event] Type: {event.data.event_type.name}") @@ -142,19 +145,15 @@ async def run_agent_framework() -> None: # Please refer to `with_plan_review` for proper human interaction during planning phases. await asyncio.get_event_loop().run_in_executor(None, input, "Press Enter to continue...") - elif event.type == "output": - output_event = event - if not output_event: raise RuntimeError("Workflow did not produce a final output event.") + print("\n\nWorkflow completed!") print("Final Output:") - # The output of the Magentic workflow is a list of ChatMessages with only one final message - # generated by the orchestrator. - output_messages = cast(list[Message], output_event.data) - if output_messages: - output = output_messages[-1].text - print(output) + # The output of the Magentic workflow is an AgentResponse or AgentResponseUpdate, + # which contains the final message text. + output_message = cast(AgentResponseUpdate, output_event.data) + print(output_message.text) async def main() -> None: diff --git a/python/scripts/sample_validation/__main__.py b/python/scripts/sample_validation/__main__.py index c5e1858e159..f0b93b2b67c 100644 --- a/python/scripts/sample_validation/__main__.py +++ b/python/scripts/sample_validation/__main__.py @@ -17,6 +17,7 @@ import argparse import asyncio +import logging import os import sys import time @@ -29,6 +30,8 @@ from sample_validation.report import save_report from sample_validation.workflow import ValidationConfig, create_validation_workflow +logging.basicConfig(level=logging.INFO) + def parse_arguments() -> argparse.Namespace: """Parse command line arguments.""" diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 249ae1aac69..e7436cf64ad 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -411,7 +411,12 @@ async def create( id=agent_id, name=agent_id, instructions=AgentInstruction, - context_providers=[SkillsProvider.from_paths(skill_paths=str(SKILLS_DIR))], + context_providers=[SkillsProvider.from_paths( + skill_paths=str(SKILLS_DIR), + disable_load_skill_approval=True, + disable_read_skill_resource_approval=True, + disable_run_skill_script_approval=True, + )], default_options={ "on_permission_request": prompt_permission, "timeout": self.config.agent_timeout, From 8b895b867c0fc2fb5ed399f945fb23ec5f329dd3 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Mon, 27 Jul 2026 15:43:47 -0700 Subject: [PATCH 27/28] Address copilot comments --- .github/.linkspector.yml | 3 ++- .../foundry-hosted-agents/responses/using_deployed_agent.py | 3 ++- python/scripts/sample_validation/playbook.py | 3 +++ .../scripts/validate_hosted_agent.sh | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/.linkspector.yml b/.github/.linkspector.yml index 15ca806f41a..3bdb4d9ff21 100644 --- a/.github/.linkspector.yml +++ b/.github/.linkspector.yml @@ -25,8 +25,9 @@ ignorePatterns: # whole domain rather than just /download. - pattern: "https:\/\/dotnet.microsoft.com" - pattern: "https://github.com/Rel1cx/eslint-react" + - pattern: "SKILL.md" # excludedDirs: - # Folders which include links to localhost, since it's not ignored with regular expressions +# Folders which include links to localhost, since it's not ignored with regular expressions baseUrl: https://github.com/microsoft/agent-framework/ aliveStatusCodes: - 200 diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py index 6512af4bf58..6f574b41252 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py @@ -4,6 +4,7 @@ import asyncio import os +from typing import cast from agent_framework import AgentSession from agent_framework.foundry import FoundryAgent @@ -62,7 +63,7 @@ async def delete_hosted_agent_session( """Delete a hosted-agent service session.""" await project_client.agents.delete_session( agent_name, - session.session_id, + cast(str, session.service_session_id), ) diff --git a/python/scripts/sample_validation/playbook.py b/python/scripts/sample_validation/playbook.py index 7a8a48bae57..c35fbb0eae6 100644 --- a/python/scripts/sample_validation/playbook.py +++ b/python/scripts/sample_validation/playbook.py @@ -101,6 +101,9 @@ def sample_files(sample: SampleInfo) -> list[Path]: For a single-file sample this is just that file. For a directory sample (``main.py``/``app.py`` entrypoint) it is every ``.py`` file in the tree. + + Note: we only consider source code files, not data files, nor deployment + artifacts (Dockerfile, requirements.txt, etc.) for now. """ path = sample.path if path.is_dir(): diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh index 23e385ffc9a..1eab498da85 100755 --- a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/scripts/validate_hosted_agent.sh @@ -178,6 +178,7 @@ command -v lsof >/dev/null 2>&1 || command -v fuser >/dev/null 2>&1 || \ [ -d "$SAMPLE_DIR" ] || die "sample dir not found: $SAMPLE_DIR" [ -f "$SAMPLE_DIR/main.py" ] || die "no main.py in sample dir: $SAMPLE_DIR" [ -f "$SAMPLE_DIR/agent.yaml" ] || die "no agent.yaml in sample dir: $SAMPLE_DIR" +[ -f "$SAMPLE_DIR/agent.manifest.yaml" ] || die "no agent.manifest.yaml in sample dir: $SAMPLE_DIR" [ -n "$MODEL" ] || die "model deployment name required (--model or AZURE_AI_MODEL_DEPLOYMENT_NAME)" # derive agent name + protocol from agent.yaml From c67d84239ab71bcde2dae00dc1633c29fcfdd3b3 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Mon, 27 Jul 2026 16:18:25 -0700 Subject: [PATCH 28/28] Fix link inspection --- .github/.linkspector.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/.linkspector.yml b/.github/.linkspector.yml index 3bdb4d9ff21..84ba43c9666 100644 --- a/.github/.linkspector.yml +++ b/.github/.linkspector.yml @@ -2,6 +2,7 @@ dirs: - . excludedFiles: - ./python/CHANGELOG.md + - "**/SKILL.md" ignorePatterns: - pattern: "/github/" - pattern: "./actions" @@ -25,7 +26,6 @@ ignorePatterns: # whole domain rather than just /download. - pattern: "https:\/\/dotnet.microsoft.com" - pattern: "https://github.com/Rel1cx/eslint-react" - - pattern: "SKILL.md" # excludedDirs: # Folders which include links to localhost, since it's not ignored with regular expressions baseUrl: https://github.com/microsoft/agent-framework/