From 8c143edc211063b6364ca19742e25e72efd7f7a8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Jul 2026 06:16:33 +0000 Subject: [PATCH 1/2] Initial plan From b42ad7da8a339621c537c3db990d4a1fb65107b5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Jul 2026 07:08:32 +0000 Subject: [PATCH 2/2] refactor: extract helper functions to reduce function-length lint findings in pkg/cli and pkg/workflow Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .refactor-work/bak/mcp_config_custom.go | 629 +++++++++ .refactor-work/bak/mcp_property_validation.go | 270 ++++ .refactor-work/bak/mcp_renderer_builtin.go | 395 ++++++ .refactor-work/bak/mcp_renderer_github.go | 315 +++++ .refactor-work/orig_github.go | 328 +++++ .scratch_refactor/INSTRUCTIONS.md | 42 + .scratch_refactor/PARAM_INSTRUCTIONS.md | 34 + .scratch_refactor/batch_01 | 19 + .scratch_refactor/batch_02 | 14 + .scratch_refactor/batch_03 | 14 + .scratch_refactor/batch_04 | 15 + .scratch_refactor/batch_05 | 19 + .scratch_refactor/batch_06 | 20 + .scratch_refactor/batch_07 | 21 + .scratch_refactor/batch_08 | 18 + .scratch_refactor/batch_09 | 20 + .scratch_refactor/batch_10 | 19 + .scratch_refactor/batch_11 | 22 + .scratch_refactor/batch_12 | 20 + .scratch_worktree | 1 + pkg/cli/access_log.go | 101 +- pkg/cli/actions_build_command.go | 38 +- pkg/cli/add_interactive_engine.go | 242 ++-- pkg/cli/add_interactive_git.go | 289 +++-- pkg/cli/add_interactive_orchestrator.go | 156 ++- pkg/cli/add_interactive_schedule.go | 298 +++-- pkg/cli/add_interactive_workflow.go | 233 ++-- pkg/cli/add_package_manifest.go | 245 ++-- pkg/cli/add_wizard_command.go | 135 +- pkg/cli/add_workflow_pr.go | 169 +-- pkg/cli/add_workflow_resolution.go | 281 ++-- pkg/cli/audit_agentic_analysis.go | 192 +-- pkg/cli/audit_comparison.go | 107 +- pkg/cli/audit_diff.go | 801 ++++++------ pkg/cli/audit_diff_command.go | 291 +++-- pkg/cli/audit_diff_render.go | 368 +++--- pkg/cli/audit_expanded.go | 146 ++- pkg/cli/audit_report.go | 536 ++++---- pkg/cli/audit_report_analysis.go | 332 ++--- pkg/cli/audit_report_render.go | 25 + pkg/cli/bootstrap_profile_manifest.go | 110 +- pkg/cli/bootstrap_shared.go | 80 +- pkg/cli/codemod_agent_session.go | 124 +- pkg/cli/codemod_assign_to_agent.go | 167 ++- pkg/cli/codemod_bash_single_quoted_args.go | 193 +-- ...emod_checkout_persist_credentials_false.go | 238 ++-- pkg/cli/codemod_cli_proxy_mode.go | 53 +- pkg/cli/codemod_dependabot_permissions.go | 105 +- pkg/cli/codemod_difc_proxy.go | 24 +- pkg/cli/codemod_discussion_flag.go | 230 ++-- .../codemod_discussion_trigger_categories.go | 129 +- .../codemod_effective_tokens_to_ai_credits.go | 130 +- pkg/cli/codemod_engine_env_secrets.go | 119 +- pkg/cli/codemod_engine_steps.go | 470 +++---- .../codemod_engine_to_top_level_helpers.go | 130 +- pkg/cli/codemod_factory.go | 327 +++-- pkg/cli/codemod_github_app.go | 167 +-- pkg/cli/codemod_github_repos.go | 75 +- pkg/cli/codemod_mcp_network.go | 404 +++--- .../codemod_mentions_allow_team_members.go | 180 ++- ...tive_tokens_suffix_to_ai_credits_suffix.go | 179 ++- pkg/cli/codemod_network_firewall.go | 72 +- pkg/cli/codemod_permissions_read.go | 68 +- pkg/cli/codemod_permissions_write.go | 212 ++-- pkg/cli/codemod_playwright_domains.go | 304 +++-- pkg/cli/codemod_run_install_scripts.go | 107 +- ...mod_safe_output_add_reviewer_allowlists.go | 161 ++- ...odemod_safe_output_merge_pr_constraints.go | 138 +- ...odemod_safe_output_require_title_prefix.go | 165 ++- pkg/cli/codemod_schedule.go | 238 ++-- pkg/cli/codemod_serena_import.go | 33 +- pkg/cli/codemod_slash_command.go | 119 +- pkg/cli/codemod_steps_run_secrets_env.go | 350 ++--- pkg/cli/codemod_upload_assets.go | 119 +- pkg/cli/codemod_user_rate_limit.go | 99 +- pkg/cli/codemod_workflow_dispatch_required.go | 199 +-- pkg/cli/codemod_workflow_run_branches.go | 85 +- pkg/cli/commands.go | 11 +- pkg/cli/compile_compiler_setup.go | 16 + pkg/cli/compile_file_operations.go | 50 +- pkg/cli/compile_orchestrator.go | 143 ++- pkg/cli/compile_pipeline.go | 770 ++++++----- pkg/cli/compile_repository_manifest.go | 52 +- pkg/cli/compile_schedule_calendar.go | 16 +- pkg/cli/compile_stats.go | 221 ++-- pkg/cli/compile_validation.go | 99 +- pkg/cli/compile_watch.go | 316 +++-- pkg/cli/compile_workflow_processor.go | 79 +- pkg/cli/completion_command.go | 61 +- pkg/cli/copilot_agent_logs.go | 132 +- pkg/cli/copilot_events_jsonl.go | 236 ++-- pkg/cli/copilot_setup.go | 155 ++- pkg/cli/dependency_graph.go | 82 +- pkg/cli/deps_report.go | 244 ++-- pkg/cli/deps_security.go | 16 +- pkg/cli/devcontainer.go | 283 +++-- pkg/cli/dispatch.go | 716 +++++++---- pkg/cli/docker_images.go | 205 ++- pkg/cli/download_workflow.go | 105 +- pkg/cli/drain3_integration.go | 49 +- pkg/cli/drain3_train.go | 43 +- pkg/cli/enable.go | 475 +++---- pkg/cli/engine_secrets.go | 243 ++-- pkg/cli/experiments_analyze_statistics.go | 201 +-- pkg/cli/experiments_command.go | 67 +- pkg/cli/fetch.go | 192 +-- pkg/cli/firewall_log.go | 173 +-- pkg/cli/firewall_policy.go | 217 ++-- pkg/cli/fix_codemods.go | 70 +- pkg/cli/fix_command.go | 332 ++--- pkg/cli/forecast.go | 222 ++-- pkg/cli/forecast_command.go | 106 +- pkg/cli/forecast_compute.go | 266 ++-- pkg/cli/forecast_montecarlo.go | 18 +- pkg/cli/forecast_render.go | 105 +- pkg/cli/frontmatter_editor.go | 530 ++++---- pkg/cli/gateway_logs_mcp.go | 257 ++-- pkg/cli/gateway_logs_parsing.go | 263 ++-- pkg/cli/gateway_logs_render.go | 308 +++-- pkg/cli/gateway_logs_rpc.go | 440 ++++--- pkg/cli/gateway_logs_timeline_render.go | 443 ++++--- pkg/cli/generate_action_metadata_command.go | 232 ++-- pkg/cli/git.go | 161 ++- pkg/cli/health_command.go | 103 +- pkg/cli/health_metrics.go | 78 +- pkg/cli/import_url_fetcher.go | 64 +- pkg/cli/importpath.go | 76 +- pkg/cli/imports.go | 412 +++--- pkg/cli/includes.go | 640 +++++----- pkg/cli/init.go | 408 +++--- pkg/cli/init_command.go | 190 +-- pkg/cli/interactive.go | 272 ++-- pkg/cli/jsonworkflow_to_markdown.go | 309 ++--- pkg/cli/list_workflows_command.go | 180 ++- pkg/cli/log_aggregation.go | 68 +- pkg/cli/logs_cache.go | 68 +- pkg/cli/logs_command.go | 556 ++++---- pkg/cli/logs_download.go | 741 ++++++----- pkg/cli/logs_episode.go | 334 +++-- pkg/cli/logs_format_compact.go | 413 +++--- pkg/cli/logs_format_tsv.go | 176 +-- pkg/cli/logs_github_api.go | 80 +- pkg/cli/logs_github_rate_limit_usage.go | 211 ++-- pkg/cli/logs_metrics.go | 537 ++++---- pkg/cli/logs_orchestrator_download.go | 27 +- pkg/cli/logs_orchestrator_filters.go | 197 +-- pkg/cli/logs_orchestrator_render.go | 170 +-- pkg/cli/logs_orchestrator_stdin.go | 217 ++-- pkg/cli/logs_parsing_firewall.go | 99 +- pkg/cli/logs_parsing_javascript.go | 115 +- pkg/cli/logs_report.go | 526 ++++---- pkg/cli/logs_report_mcp.go | 185 +-- pkg/cli/logs_report_tools.go | 50 +- pkg/cli/logs_safe_output_chains.go | 28 +- pkg/cli/logs_usage_activity.go | 115 +- pkg/cli/logs_utils.go | 341 ++--- pkg/cli/mcp_add.go | 286 ++--- pkg/cli/mcp_inspect.go | 267 ++-- pkg/cli/mcp_inspect_inspector.go | 347 ++--- pkg/cli/mcp_inspect_mcp.go | 461 ++++--- pkg/cli/mcp_inspect_mcp_scripts_files.go | 74 +- pkg/cli/mcp_inspect_mcp_scripts_server.go | 102 +- pkg/cli/mcp_list.go | 281 ++-- pkg/cli/mcp_list_tools.go | 108 +- pkg/cli/mcp_logs_guardrail.go | 76 +- pkg/cli/mcp_permissions.go | 90 +- pkg/cli/mcp_registry.go | 278 ++-- pkg/cli/mcp_registry_list.go | 50 +- pkg/cli/mcp_secrets.go | 67 +- pkg/cli/mcp_server.go | 60 +- pkg/cli/mcp_server_command.go | 90 +- pkg/cli/mcp_tool_table.go | 115 +- pkg/cli/mcp_tools_management.go | 102 +- pkg/cli/mcp_tools_privileged.go | 724 +++++------ pkg/cli/mcp_tools_readonly.go | 328 +++-- pkg/cli/mcp_validation.go | 225 ++-- pkg/cli/model_costs.go | 11 + pkg/cli/observability_insights.go | 213 ++-- pkg/cli/org_runner.go | 365 +++--- pkg/cli/outcome_domain_breakdown.go | 20 +- pkg/cli/outcome_eval.go | 165 ++- pkg/cli/outcome_eval_agent.go | 131 +- pkg/cli/outcome_eval_comment.go | 60 +- pkg/cli/outcome_eval_enrich.go | 30 +- pkg/cli/outcome_eval_issue.go | 24 +- pkg/cli/outcome_eval_label.go | 32 +- pkg/cli/outcome_eval_pr.go | 73 +- pkg/cli/outcome_eval_review.go | 188 +-- pkg/cli/outcome_eval_update.go | 41 +- pkg/cli/outcome_eval_workflow.go | 45 +- pkg/cli/outcome_evaluation.go | 47 +- pkg/cli/outcomes_command.go | 223 ++-- pkg/cli/outcomes_history.go | 142 ++- pkg/cli/packages.go | 233 ++-- pkg/cli/poutine.go | 577 ++++----- pkg/cli/pr_automerge.go | 75 +- pkg/cli/pr_command.go | 609 +++++---- pkg/cli/preconditions.go | 9 +- pkg/cli/project_command.go | 331 ++--- pkg/cli/redacted_domains.go | 73 +- pkg/cli/remove_command.go | 396 +++--- pkg/cli/resources.go | 270 ++-- pkg/cli/retry.go | 112 +- pkg/cli/run_interactive.go | 228 ++-- pkg/cli/run_push.go | 501 ++++---- pkg/cli/run_workflow_execution.go | 800 ++++++------ pkg/cli/run_workflow_tracking.go | 249 ++-- pkg/cli/run_workflow_validation.go | 215 ++-- pkg/cli/runner_guard.go | 315 +++-- pkg/cli/secret_set_command.go | 90 +- pkg/cli/setup_repository.go | 71 +- pkg/cli/shell_completion.go | 196 ++- pkg/cli/signal_aware_poll.go | 103 +- pkg/cli/spec.go | 187 +-- pkg/cli/status_command.go | 342 +++-- pkg/cli/token_usage.go | 419 +++--- pkg/cli/tokens_bootstrap.go | 76 +- pkg/cli/tool_graph.go | 128 +- pkg/cli/trial_command.go | 134 +- pkg/cli/trial_confirmation.go | 234 ++-- pkg/cli/trial_helpers.go | 386 ++++-- pkg/cli/trial_repository.go | 392 +++--- pkg/cli/trial_runner.go | 376 +++--- pkg/cli/trial_support.go | 106 +- pkg/cli/update_actions.go | 992 ++++++++------- pkg/cli/update_command.go | 327 +++-- pkg/cli/update_container_pins.go | 177 +-- pkg/cli/update_extension_check.go | 353 +++--- pkg/cli/update_manifest.go | 288 +++-- pkg/cli/update_merge.go | 231 ++-- pkg/cli/update_org.go | 107 +- pkg/cli/update_redirects.go | 149 ++- pkg/cli/update_workflows.go | 658 +++++----- pkg/cli/upgrade_command.go | 510 ++++---- pkg/cli/upgrade_org.go | 88 +- pkg/cli/validate_command.go | 95 +- pkg/cli/view_command.go | 126 +- pkg/cli/workflows.go | 181 +-- pkg/cli/yaml_frontmatter_utils.go | 116 +- pkg/cli/zizmor.go | 244 ++-- pkg/workflow/antigravity_engine.go | 178 +-- pkg/workflow/awf_config.go | 273 ++-- pkg/workflow/awf_helpers.go | 653 +++++----- pkg/workflow/behavior_defined_engine.go | 168 +-- pkg/workflow/cache.go | 583 ++++----- .../central_slash_command_workflow.go | 320 ++--- pkg/workflow/checkout_config_parser.go | 375 +++--- pkg/workflow/checkout_manager.go | 136 +- pkg/workflow/checkout_step_generator.go | 411 +++--- pkg/workflow/claude_engine.go | 424 +++---- pkg/workflow/claude_logs.go | 437 ++++--- pkg/workflow/claude_tools.go | 78 +- pkg/workflow/codex_engine.go | 566 ++++----- pkg/workflow/codex_logs.go | 254 ++-- pkg/workflow/codex_mcp.go | 141 +-- pkg/workflow/command.go | 217 ++-- pkg/workflow/compiler.go | 436 ++++--- pkg/workflow/compiler_activation_daily_aic.go | 125 +- pkg/workflow/compiler_activation_job.go | 117 +- .../compiler_activation_permissions.go | 70 +- pkg/workflow/compiler_activation_steps.go | 116 +- pkg/workflow/compiler_aw_context.go | 81 +- pkg/workflow/compiler_custom_jobs.go | 227 ++-- pkg/workflow/compiler_difc_proxy.go | 69 +- pkg/workflow/compiler_experiments.go | 240 ++-- pkg/workflow/compiler_github_actions_steps.go | 49 +- pkg/workflow/compiler_github_mcp_steps.go | 112 +- pkg/workflow/compiler_jobs.go | 77 +- pkg/workflow/compiler_orchestrator_engine.go | 38 +- .../compiler_orchestrator_frontmatter.go | 225 ++-- .../compiler_orchestrator_workflow.go | 216 ++-- pkg/workflow/compiler_safe_output_jobs.go | 435 ++++--- pkg/workflow/compiler_safe_outputs_job.go | 788 +++++------- pkg/workflow/compiler_safe_outputs_steps.go | 258 ++-- pkg/workflow/compiler_string_api.go | 138 +- pkg/workflow/compiler_unlock_job.go | 124 +- pkg/workflow/compiler_workflow_call.go | 148 ++- pkg/workflow/compiler_yaml.go | 157 +-- pkg/workflow/compiler_yaml_ai_execution.go | 210 +-- pkg/workflow/compiler_yaml_checkout.go | 169 ++- pkg/workflow/compiler_yaml_header.go | 200 ++- pkg/workflow/compiler_yaml_normalize.go | 170 +-- pkg/workflow/compiler_yaml_post_agent.go | 225 ++-- pkg/workflow/compiler_yaml_prompt.go | 419 +++--- pkg/workflow/compiler_yaml_runtime_setup.go | 109 +- pkg/workflow/compiler_yaml_step_conversion.go | 103 +- pkg/workflow/compiler_yaml_step_generation.go | 166 +-- pkg/workflow/compiler_yaml_step_lifecycle.go | 302 ++--- pkg/workflow/concurrency.go | 129 +- pkg/workflow/copilot_engine_tools.go | 301 +++-- pkg/workflow/copilot_logs.go | 664 +++++----- pkg/workflow/create_agent_session.go | 73 +- pkg/workflow/create_code_scanning_alert.go | 111 +- pkg/workflow/create_discussion.go | 8 +- pkg/workflow/create_pull_request.go | 92 +- pkg/workflow/dependabot.go | 613 ++++----- pkg/workflow/dispatch_repository.go | 153 +-- .../dispatch_repository_validation.go | 104 +- pkg/workflow/dispatch_workflow_validation.go | 169 ++- pkg/workflow/docker.go | 292 ++--- pkg/workflow/docker_validation.go | 149 +-- pkg/workflow/engine.go | 833 ++++++------ pkg/workflow/engine_definition_loader.go | 97 +- pkg/workflow/engine_validation.go | 86 +- pkg/workflow/error_recovery.go | 159 +-- pkg/workflow/evals_job.go | 164 +-- pkg/workflow/evals_steps.go | 108 +- pkg/workflow/event_validation.go | 86 +- pkg/workflow/expression_optimizer.go | 356 +++--- pkg/workflow/expression_parser.go | 211 ++-- pkg/workflow/expression_safety_validation.go | 285 ++--- pkg/workflow/expression_syntax_validation.go | 54 +- pkg/workflow/filters.go | 530 +++----- pkg/workflow/frontmatter_error.go | 131 +- .../frontmatter_extraction_security.go | 471 +++---- pkg/workflow/frontmatter_extraction_yaml.go | 73 +- .../frontmatter_on_section_cleanup.go | 1125 +++++++---------- pkg/workflow/frontmatter_parsing.go | 449 ++++--- pkg/workflow/frontmatter_serialization.go | 282 ++--- pkg/workflow/gemini_engine.go | 249 ++-- pkg/workflow/gemini_tools.go | 114 +- pkg/workflow/gh_cli_permissions.go | 244 ++-- pkg/workflow/github_toolsets.go | 111 +- pkg/workflow/glob_validation.go | 156 ++- pkg/workflow/header.go | 81 +- pkg/workflow/imports.go | 264 ++-- pkg/workflow/jobs.go | 181 ++- pkg/workflow/js.go | 191 ++- pkg/workflow/known_needs_expressions.go | 138 +- pkg/workflow/lock_validation.go | 68 +- pkg/workflow/lsp_manager.go | 120 +- pkg/workflow/maintenance_workflow.go | 374 +++--- pkg/workflow/maintenance_workflow_yaml.go | 878 +++++-------- pkg/workflow/markdown_security_scanner.go | 351 ++--- pkg/workflow/mcp_cli_mount.go | 140 +- pkg/workflow/mcp_config_custom.go | 923 ++++++-------- .../mcp_config_playwright_renderer.go | 13 +- pkg/workflow/mcp_environment.go | 244 ++-- pkg/workflow/mcp_gateway_config.go | 75 +- pkg/workflow/mcp_property_validation.go | 245 ++-- pkg/workflow/mcp_renderer.go | 227 ++-- pkg/workflow/mcp_renderer_builtin.go | 146 +-- pkg/workflow/mcp_renderer_github.go | 239 ++-- .../mcp_scripts_dependencies_validation.go | 124 +- pkg/workflow/mcp_scripts_generator.go | 153 ++- pkg/workflow/mcp_scripts_parser.go | 163 +-- pkg/workflow/mcp_setup_generator.go | 172 ++- pkg/workflow/metrics.go | 134 +- pkg/workflow/missing_issue_reporting.go | 129 +- pkg/workflow/model_identifier.go | 70 +- pkg/workflow/network_firewall_validation.go | 233 ++-- pkg/workflow/nodejs.go | 122 +- pkg/workflow/notify_comment.go | 89 +- pkg/workflow/npm_validation.go | 57 +- pkg/workflow/observability_otlp.go | 59 +- .../permissions_compiler_validator.go | 150 +-- pkg/workflow/permissions_operations.go | 245 ++-- pkg/workflow/permissions_parser.go | 211 ++-- pkg/workflow/permissions_validation.go | 440 ++++--- pkg/workflow/pi_engine.go | 303 ++--- pkg/workflow/pi_logs.go | 87 +- pkg/workflow/pip_validation.go | 58 +- pkg/workflow/pr.go | 45 +- pkg/workflow/project_config_parsing.go | 84 +- pkg/workflow/publish_artifacts.go | 159 +-- pkg/workflow/publish_assets.go | 252 ++-- .../pull_request_target_validation.go | 107 +- pkg/workflow/push_to_pull_request_branch.go | 266 ++-- pkg/workflow/redact_secrets.go | 88 +- pkg/workflow/repo_memory.go | 863 ++++++------- pkg/workflow/repo_memory_prompt.go | 175 +-- .../repository_features_validation.go | 93 +- pkg/workflow/resolve.go | 149 ++- pkg/workflow/role_checks.go | 245 ++-- pkg/workflow/run_step_sanitizer.go | 90 +- pkg/workflow/runtime_deduplication.go | 332 ++--- pkg/workflow/runtime_detection.go | 55 +- pkg/workflow/runtime_import_validation.go | 118 +- pkg/workflow/runtime_overrides.go | 183 +-- pkg/workflow/runtime_step_generator.go | 92 +- pkg/workflow/runtime_validation.go | 240 ++-- pkg/workflow/safe_jobs.go | 509 ++++---- pkg/workflow/safe_jobs_needs_validation.go | 97 +- pkg/workflow/safe_outputs_actions.go | 302 +++-- pkg/workflow/safe_outputs_app_config.go | 480 +++---- pkg/workflow/safe_outputs_config_base.go | 23 +- .../safe_outputs_config_extraction.go | 523 +++----- .../safe_outputs_config_generation.go | 332 +++-- pkg/workflow/safe_outputs_config_global.go | 34 +- pkg/workflow/safe_outputs_config_runtime.go | 123 +- pkg/workflow/safe_outputs_handler_registry.go | 163 +-- pkg/workflow/safe_outputs_jobs.go | 143 +-- pkg/workflow/safe_outputs_max_validation.go | 55 +- pkg/workflow/safe_outputs_messages_config.go | 151 +-- ...utputs_steps_shell_expansion_validation.go | 116 +- .../safe_outputs_tools_computation.go | 245 ++-- pkg/workflow/safe_outputs_tools_generation.go | 425 +++---- .../safe_outputs_tools_repo_params.go | 203 +-- pkg/workflow/safe_outputs_validation.go | 87 +- pkg/workflow/safe_update_manifest.go | 57 +- pkg/workflow/sandbox_validation.go | 254 ++-- pkg/workflow/schedule_preprocessing.go | 501 ++++---- pkg/workflow/service_ports.go | 122 +- pkg/workflow/side_repo_maintenance.go | 467 ++++--- pkg/workflow/skills_frontmatter.go | 165 ++- pkg/workflow/step_order_validation.go | 95 +- pkg/workflow/step_types.go | 28 +- pkg/workflow/stop_after.go | 389 +++--- pkg/workflow/strict_mode_env_validation.go | 71 +- .../strict_mode_network_validation.go | 109 +- .../strict_mode_permissions_validation.go | 103 +- .../strict_mode_sandbox_validation.go | 59 +- pkg/workflow/strict_mode_steps_validation.go | 64 +- pkg/workflow/submit_pr_review.go | 162 +-- pkg/workflow/template.go | 78 +- pkg/workflow/template_injection_utils.go | 72 +- pkg/workflow/template_injection_validation.go | 128 +- pkg/workflow/threat_detection_config.go | 31 +- pkg/workflow/threat_detection_external.go | 70 +- .../threat_detection_inline_engine.go | 192 ++- pkg/workflow/threat_detection_job.go | 139 +- pkg/workflow/threat_detection_steps.go | 76 +- pkg/workflow/time_delta.go | 272 ++-- pkg/workflow/tools.go | 833 ++++++------ pkg/workflow/tools_parser.go | 484 +++---- pkg/workflow/tools_validation_github.go | 98 +- ...s_validation_github_integrity_reactions.go | 62 +- .../tools_validation_github_toolsets.go | 90 +- pkg/workflow/trigger_parser.go | 683 +++++----- pkg/workflow/unified_prompt_step.go | 565 ++++----- pkg/workflow/universal_llm_consumer_engine.go | 149 +-- pkg/workflow/workflow_builder.go | 266 ++-- pkg/workflow/workflow_github_app.go | 68 +- pkg/workflow/xml_comments.go | 105 +- 434 files changed, 48195 insertions(+), 44767 deletions(-) create mode 100644 .refactor-work/bak/mcp_config_custom.go create mode 100644 .refactor-work/bak/mcp_property_validation.go create mode 100644 .refactor-work/bak/mcp_renderer_builtin.go create mode 100644 .refactor-work/bak/mcp_renderer_github.go create mode 100644 .refactor-work/orig_github.go create mode 100644 .scratch_refactor/INSTRUCTIONS.md create mode 100644 .scratch_refactor/PARAM_INSTRUCTIONS.md create mode 100644 .scratch_refactor/batch_01 create mode 100644 .scratch_refactor/batch_02 create mode 100644 .scratch_refactor/batch_03 create mode 100644 .scratch_refactor/batch_04 create mode 100644 .scratch_refactor/batch_05 create mode 100644 .scratch_refactor/batch_06 create mode 100644 .scratch_refactor/batch_07 create mode 100644 .scratch_refactor/batch_08 create mode 100644 .scratch_refactor/batch_09 create mode 100644 .scratch_refactor/batch_10 create mode 100644 .scratch_refactor/batch_11 create mode 100644 .scratch_refactor/batch_12 create mode 160000 .scratch_worktree diff --git a/.refactor-work/bak/mcp_config_custom.go b/.refactor-work/bak/mcp_config_custom.go new file mode 100644 index 00000000000..fafa5b73e47 --- /dev/null +++ b/.refactor-work/bak/mcp_config_custom.go @@ -0,0 +1,629 @@ +package workflow + +import ( + "fmt" + "maps" + "os" + "strings" + + "github.com/github/gh-aw/pkg/console" + "github.com/github/gh-aw/pkg/logger" + "github.com/github/gh-aw/pkg/parser" + "github.com/github/gh-aw/pkg/setutil" + "github.com/github/gh-aw/pkg/sliceutil" + "github.com/github/gh-aw/pkg/types" +) + +var mcpCustomLog = logger.New("workflow:mcp-config-custom") + +// renderCustomMCPConfigWrapperWithContext generates custom MCP server configuration wrapper with workflow context +// This version includes workflowData to determine if localhost URLs should be rewritten +func renderCustomMCPConfigWrapperWithContext(yaml *strings.Builder, toolName string, toolConfig map[string]any, isLast bool, workflowData *WorkflowData) error { + mcpCustomLog.Printf("Rendering custom MCP config wrapper with context for tool: %s", toolName) + fmt.Fprintf(yaml, " \"%s\": {\n", toolName) + + // Determine if localhost URLs should be rewritten to host.docker.internal + // This is needed when firewall is enabled (agent is not disabled) + rewriteLocalhost := shouldRewriteLocalhostToDocker(workflowData) + + // Use the shared MCP config renderer with JSON format + renderer := MCPConfigRenderer{ + IndentLevel: " ", + Format: "json", + RewriteLocalhostToDocker: rewriteLocalhost, + GuardPolicies: deriveWriteSinkGuardPolicyFromWorkflow(workflowData), + } + + err := renderSharedMCPConfig(yaml, toolName, toolConfig, renderer) + if err != nil { + return err + } + + if isLast { + yaml.WriteString(" }\n") + } else { + yaml.WriteString(" },\n") + } + + return nil +} + +// renderCustomMCPEnvVars normalizes custom MCP env values for the target output +// format before serialization. +// +// For TOML output, GitHub Actions template expressions are rewritten to direct +// ${VAR} references because Codex config expects shell-style environment +// expansion. For JSON output, Copilot uses escaped \${VAR} passthrough syntax, +// while non-Copilot engines use bash variable substitution to avoid embedding +// secret expressions directly in the generated run block. +func renderCustomMCPEnvVars(env map[string]string, tomlFormat, requiresCopilotFields bool) map[string]string { + renderedEnv := make(map[string]string, len(env)) + for envKey, envValue := range env { + if tomlFormat { + // Replace template expressions with environment variable references for TOML. + // For TOML, we use direct shell variable syntax without backslash. + envValue = strings.ReplaceAll(envValue, "${{ secrets.", "${") + envValue = strings.ReplaceAll(envValue, "${{ env.", "${") + envValue = strings.ReplaceAll(envValue, "${{ github.workspace }}", "${GITHUB_WORKSPACE}") + envValue = strings.ReplaceAll(envValue, " }}", "}") + } else if requiresCopilotFields { + // For Copilot, replace all template expressions with \${VAR} syntax. + envValue = ReplaceTemplateExpressionsWithEnvVars(envValue) + } else { + // For non-Copilot engines, replace secrets with ${VAR} bash expansion so + // they are never directly interpolated in the run block (RGS-008). The + // env vars are injected into the step env block by collectMCPEnvironmentVariables. + envValue = ReplaceSecretsWithBashVars(envValue) + } + renderedEnv[envKey] = envValue + } + + return renderedEnv +} + +// renderSharedMCPConfig generates MCP server configuration for a single tool using shared logic +// This function handles the common logic for rendering MCP configurations across different engines +func renderSharedMCPConfig(yaml *strings.Builder, toolName string, toolConfig map[string]any, renderer MCPConfigRenderer) error { + mcpCustomLog.Printf("Rendering MCP config for tool: %s, format: %s", toolName, renderer.Format) + + mcpConfig, err := getMCPConfig(toolConfig, toolName) + if err != nil { + mcpCustomLog.Printf("Failed to parse MCP config for tool %s: %v", toolName, err) + return fmt.Errorf("failed to parse MCP config for tool '%s': %w", toolName, err) + } + if err := validateMCPGatewayStdioCommand(toolName, mcpConfig); err != nil { + return err + } + + headerSecrets := mcpHeaderSecrets(mcpConfig) + existingProperties, err := existingMCPConfigProperties(toolName, mcpConfig, renderer, headerSecrets) + if err != nil || len(existingProperties) == 0 { + return err + } + + hasTrailingGuardPolicies := renderer.Format == "json" && len(renderer.GuardPolicies) > 0 + renderMCPConfigProperties(yaml, mcpConfig, renderer, existingProperties, headerSecrets, hasTrailingGuardPolicies) + renderMCPConfigGuardPolicies(yaml, toolName, renderer, hasTrailingGuardPolicies) + return nil +} + +func validateMCPGatewayStdioCommand(toolName string, mcpConfig *parser.RegistryMCPServerConfig) error { + if mcpConfig.Type != "stdio" || mcpConfig.Command == "" || mcpConfig.Command == "docker" { + return nil + } + return fmt.Errorf( + "tool '%s' stdio MCP server uses command %q which is not supported by MCP Gateway. "+ + "Stdio servers must be containerized (use 'container' with 'entrypoint'), "+ + "or switch to HTTP transport for servers that run directly on the runner.\n\n"+ + "Example (container):\ntools:\n %s:\n container: \"my-registry/my-tool:latest\"\n entrypoint: \"my-tool\"\n args: [\"--verbose\"]\n\n"+ + "Example (HTTP — for Python/Node servers installed on the runner):\ntools:\n %s:\n type: http\n url: \"http://localhost:8765/mcp\"", + toolName, mcpConfig.Command, toolName, toolName, + ) +} + +func mcpHeaderSecrets(mcpConfig *parser.RegistryMCPServerConfig) map[string]string { + if mcpConfig.Type == "http" { + return ExtractSecretsFromMap(mcpConfig.Headers) + } + return nil +} + +func existingMCPConfigProperties(toolName string, mcpConfig *parser.RegistryMCPServerConfig, renderer MCPConfigRenderer, headerSecrets map[string]string) ([]string, error) { + propertyOrder, err := mcpConfigPropertyOrder(toolName, mcpConfig, renderer, headerSecrets) + if err != nil { + return nil, err + } + var existingProperties []string + for _, prop := range propertyOrder { + if mcpConfigPropertyExists(prop, mcpConfig, renderer, headerSecrets) { + existingProperties = append(existingProperties, prop) + } + } + return existingProperties, nil +} + +func mcpConfigPropertyOrder(toolName string, mcpConfig *parser.RegistryMCPServerConfig, renderer MCPConfigRenderer, headerSecrets map[string]string) ([]string, error) { + switch mcpConfig.Type { + case "stdio": + if renderer.Format == "toml" { + return []string{"container", "entrypoint", "entrypointArgs", "mounts", "command", "args", "env", "proxy-args", "registry"}, nil + } + return []string{"type", "container", "entrypoint", "entrypointArgs", "mounts", "command", "args", "tools", "env", "proxy-args", "registry"}, nil + case "http": + if renderer.Format == "toml" { + return []string{"url", "http_headers"}, nil + } + if len(headerSecrets) > 0 { + return []string{"type", "url", "headers", "auth", "tools", "env"}, nil + } + return []string{"type", "url", "headers", "auth", "tools"}, nil + default: + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Custom MCP server '%s' has unsupported type '%s'. Supported types: stdio, http", toolName, mcpConfig.Type))) + return nil, nil + } +} + +func mcpConfigPropertyExists(prop string, mcpConfig *parser.RegistryMCPServerConfig, renderer MCPConfigRenderer, headerSecrets map[string]string) bool { + switch prop { + case "type": + return true + case "tools": + return renderer.RequiresCopilotFields || len(mcpConfig.Allowed) > 0 + case "container": + return mcpConfig.Container != "" + case "entrypoint": + return mcpConfig.Entrypoint != "" + case "entrypointArgs": + return len(mcpConfig.EntrypointArgs) > 0 + case "mounts": + return len(mcpConfig.Mounts) > 0 + case "command": + return mcpConfig.Command != "" + case "args": + return len(mcpConfig.Args) > 0 + case "env": + return len(mcpConfig.Env) > 0 || len(headerSecrets) > 0 + case "url": + return mcpConfig.URL != "" + case "headers", "http_headers": + return len(mcpConfig.Headers) > 0 + case "auth": + return mcpConfig.Auth != nil + case "proxy-args": + return len(mcpConfig.ProxyArgs) > 0 + case "registry": + return mcpConfig.Registry != "" + default: + return false + } +} + +func renderMCPConfigProperties(yaml *strings.Builder, mcpConfig *parser.RegistryMCPServerConfig, renderer MCPConfigRenderer, properties []string, headerSecrets map[string]string, hasTrailingGuardPolicies bool) { + for propIndex, property := range properties { + isLast := (propIndex == len(properties)-1) && !hasTrailingGuardPolicies + renderMCPConfigProperty(yaml, property, mcpConfig, renderer, headerSecrets, isLast) + } +} + +func renderMCPConfigProperty(yaml *strings.Builder, property string, mcpConfig *parser.RegistryMCPServerConfig, renderer MCPConfigRenderer, headerSecrets map[string]string, isLast bool) { + switch property { + case "type": + renderMCPJSONStringProperty(yaml, renderer.IndentLevel, "type", mcpConfig.Type, isLast) + case "tools": + renderMCPToolsProperty(yaml, renderer.IndentLevel, mcpConfig.Allowed, isLast) + case "container": + renderMCPStringProperty(yaml, renderer, "container", "container", mcpConfig.Container, isLast) + case "entrypoint": + renderMCPStringProperty(yaml, renderer, "entrypoint", "entrypoint", mcpConfig.Entrypoint, isLast) + case "entrypointArgs": + renderMCPArrayProperty(yaml, renderer, "entrypointArgs", "entrypointArgs", mcpConfig.EntrypointArgs, true, true, isLast) + case "mounts": + renderMCPArrayProperty(yaml, renderer, "mounts", "mounts", mcpConfig.Mounts, true, true, isLast) + case "command": + renderMCPStringProperty(yaml, renderer, "command", "command", mcpConfig.Command, isLast) + case "args": + renderMCPArrayProperty(yaml, renderer, "args", "args", mcpConfig.Args, false, false, isLast) + case "env": + renderMCPEnvProperty(yaml, renderer, mcpConfig.Env, headerSecrets, isLast) + case "url": + renderMCPURLProperty(yaml, renderer, mcpConfig.URL, isLast) + case "http_headers": + writeTOMLInlineStringMapSection(yaml, renderer.IndentLevel, "http_headers", mcpConfig.Headers) + case "headers": + renderMCPHeadersProperty(yaml, renderer, mcpConfig.Headers, headerSecrets, isLast) + case "auth": + renderMCPAuthProperty(yaml, renderer.IndentLevel, mcpConfig.Auth, isLast) + case "proxy-args": + renderMCPArrayProperty(yaml, renderer, "proxy_args", "proxy-args", mcpConfig.ProxyArgs, false, false, isLast) + case "registry": + renderMCPStringProperty(yaml, renderer, "registry", "registry", mcpConfig.Registry, isLast) + } +} + +func renderMCPStringProperty(yaml *strings.Builder, renderer MCPConfigRenderer, tomlName string, jsonName string, value string, isLast bool) { + if renderer.Format == "toml" { + fmt.Fprintf(yaml, "%s%s = \"%s\"\n", renderer.IndentLevel, tomlName, value) + return + } + renderMCPJSONStringProperty(yaml, renderer.IndentLevel, jsonName, value, isLast) +} + +func renderMCPJSONStringProperty(yaml *strings.Builder, indent string, name string, value string, isLast bool) { + fmt.Fprintf(yaml, "%s\"%s\": \"%s\"%s\n", indent, name, value, mcpJSONComma(isLast)) +} + +func renderMCPToolsProperty(yaml *strings.Builder, indent string, allowed []string, isLast bool) { + fmt.Fprintf(yaml, "%s\"tools\": [\n", indent) + tools := allowed + if len(tools) == 0 { + tools = []string{"*"} + } + for toolIndex, tool := range tools { + fmt.Fprintf(yaml, "%s \"%s\"%s\n", indent, tool, mcpJSONComma(toolIndex == len(tools)-1)) + } + fmt.Fprintf(yaml, "%s]%s\n", indent, mcpJSONComma(isLast)) +} + +func renderMCPArrayProperty(yaml *strings.Builder, renderer MCPConfigRenderer, tomlName string, jsonName string, values []string, tomlInline bool, replaceTemplates bool, isLast bool) { + if renderer.Format == "toml" { + renderMCPTOMLArrayProperty(yaml, renderer.IndentLevel, tomlName, values, tomlInline) + return + } + fmt.Fprintf(yaml, "%s\"%s\": [\n", renderer.IndentLevel, jsonName) + for valueIndex, value := range values { + if replaceTemplates && renderer.RequiresCopilotFields { + value = ReplaceTemplateExpressionsWithEnvVars(value) + } + fmt.Fprintf(yaml, "%s \"%s\"%s\n", renderer.IndentLevel, value, mcpJSONComma(valueIndex == len(values)-1)) + } + fmt.Fprintf(yaml, "%s]%s\n", renderer.IndentLevel, mcpJSONComma(isLast)) +} + +func renderMCPTOMLArrayProperty(yaml *strings.Builder, indent string, name string, values []string, inline bool) { + if inline { + fmt.Fprintf(yaml, "%s%s = [", indent, name) + for valueIndex, value := range values { + if valueIndex > 0 { + yaml.WriteString(", ") + } + fmt.Fprintf(yaml, "\"%s\"", value) + } + yaml.WriteString("]\n") + return + } + fmt.Fprintf(yaml, "%s%s = [\n", indent, name) + for _, value := range values { + fmt.Fprintf(yaml, "%s \"%s\",\n", indent, value) + } + fmt.Fprintf(yaml, "%s]\n", indent) +} + +func renderMCPEnvProperty(yaml *strings.Builder, renderer MCPConfigRenderer, env map[string]string, headerSecrets map[string]string, isLast bool) { + renderedEnv := renderCustomMCPEnvVars(env, renderer.Format == "toml", renderer.RequiresCopilotFields) + if renderer.Format == "toml" { + writeTOMLInlineStringMapSection(yaml, renderer.IndentLevel, "env", renderedEnv) + return + } + for varName := range headerSecrets { + if _, exists := renderedEnv[varName]; !exists { + renderedEnv[varName] = "\\${" + varName + "}" + } + } + writeJSONStringMapSectionRaw(yaml, renderer.IndentLevel, "env", renderedEnv, !isLast) +} + +func renderMCPURLProperty(yaml *strings.Builder, renderer MCPConfigRenderer, urlValue string, isLast bool) { + if renderer.RewriteLocalhostToDocker { + urlValue = rewriteLocalhostToDockerHost(urlValue) + } + renderMCPStringProperty(yaml, renderer, "url", "url", urlValue, isLast) +} + +func renderMCPHeadersProperty(yaml *strings.Builder, renderer MCPConfigRenderer, headers map[string]string, headerSecrets map[string]string, isLast bool) { + renderedHeaders := make(map[string]string, len(headers)) + for headerKey, headerValue := range headers { + if len(headerSecrets) > 0 { + headerValue = ReplaceSecretsWithEnvVars(headerValue, headerSecrets) + } + renderedHeaders[headerKey] = headerValue + } + writeJSONStringMapSectionRaw(yaml, renderer.IndentLevel, "headers", renderedHeaders, !isLast) +} + +func renderMCPAuthProperty(yaml *strings.Builder, indent string, auth *types.MCPAuthConfig, isLast bool) { + if auth == nil { + return + } + fmt.Fprintf(yaml, "%s\"auth\": {\n", indent) + if auth.Audience != "" { + fmt.Fprintf(yaml, "%s \"type\": \"%s\",\n", indent, auth.Type) + fmt.Fprintf(yaml, "%s \"audience\": \"%s\"\n", indent, auth.Audience) + } else { + fmt.Fprintf(yaml, "%s \"type\": \"%s\"\n", indent, auth.Type) + } + fmt.Fprintf(yaml, "%s}%s\n", indent, mcpJSONComma(isLast)) +} + +func renderMCPConfigGuardPolicies(yaml *strings.Builder, toolName string, renderer MCPConfigRenderer, hasTrailingGuardPolicies bool) { + if hasTrailingGuardPolicies { + renderGuardPoliciesJSON(yaml, renderer.GuardPolicies, renderer.IndentLevel) + } else if renderer.Format == "toml" && len(renderer.GuardPolicies) > 0 { + renderGuardPoliciesToml(yaml, renderer.GuardPolicies, toolName) + } +} + +func mcpJSONComma(isLast bool) string { + if isLast { + return "" + } + return "," +} + +// collectHTTPMCPHeaderSecrets collects all secrets from HTTP MCP tool headers +// Returns a map of environment variable names to their secret expressions +func collectHTTPMCPHeaderSecrets(tools map[string]any) map[string]string { + allSecrets := make(map[string]string) + + for toolName, toolValue := range tools { + // Check if this is an MCP tool configuration + if toolConfig, ok := toolValue.(map[string]any); ok { + if hasMcp, mcpType := hasMCPConfig(toolConfig); hasMcp && mcpType == "http" { + // Extract MCP config to get headers + if mcpConfig, err := getMCPConfig(toolConfig, toolName); err == nil { + secrets := ExtractSecretsFromMap(mcpConfig.Headers) + maps.Copy(allSecrets, secrets) + } + } + } + } + + return allSecrets +} + +// getMCPConfig extracts MCP configuration from a tool config and returns a structured MCPServerConfig +func getMCPConfig(toolConfig map[string]any, toolName string) (*parser.RegistryMCPServerConfig, error) { + mcpCustomLog.Printf("Extracting MCP config for tool: %s", toolName) + + config := MapToolConfig(toolConfig) + result := &parser.RegistryMCPServerConfig{ + BaseMCPServerConfig: types.BaseMCPServerConfig{ + Env: make(map[string]string), + Headers: make(map[string]string), + }, + Name: toolName, + } + + if err := validateMCPConfigProperties(toolConfig, toolName); err != nil { + return nil, err + } + if err := inferMCPConfigType(result, config, toolName); err != nil { + return nil, err + } + extractMCPCommonFields(result, config) + if err := extractMCPTypeSpecificFields(result, config, toolName); err != nil { + return nil, err + } + if allowed, hasAllowed := config.GetStringArray("allowed"); hasAllowed { + result.Allowed = allowed + } + finalizeStdioMCPConfig(result) + return result, nil +} + +func validateMCPConfigProperties(toolConfig map[string]any, toolName string) error { + knownProperties := map[string]struct{}{ + "type": {}, "mode": {}, "command": {}, "container": {}, "version": {}, "args": {}, + "entrypoint": {}, "entrypointArgs": {}, "mounts": {}, "env": {}, "proxy-args": {}, + "url": {}, "headers": {}, "auth": {}, "registry": {}, "allowed": {}, "toolsets": {}, + } + for key := range toolConfig { + if !setutil.Contains(knownProperties, key) { + mcpCustomLog.Printf("Unknown property '%s' in MCP config for tool '%s'", key, toolName) + validProps := sliceutil.SortedKeys(knownProperties) + return fmt.Errorf( + "unknown property '%s' in MCP configuration for tool '%s'. Valid properties are: %s. "+ + "Example:\n"+ + "mcp-servers:\n"+ + " %s:\n"+ + " command: \"npx @my/tool\"\n"+ + " args: [\"--port\", \"3000\"]", + key, toolName, strings.Join(validProps, ", "), toolName) + } + } + return nil +} + +func inferMCPConfigType(result *parser.RegistryMCPServerConfig, config MapToolConfig, toolName string) error { + if typeStr, hasType := config.GetString("type"); hasType { + mcpCustomLog.Printf("MCP type explicitly set to: %s", typeStr) + if typeStr == "local" { + result.Type = "stdio" + } else { + result.Type = typeStr + } + return nil + } + + mcpCustomLog.Print("No explicit MCP type, inferring from fields") + if _, hasURL := config.GetString("url"); hasURL { + result.Type = "http" + mcpCustomLog.Printf("Inferred MCP type as http (has url field)") + } else if _, hasCommand := config.GetString("command"); hasCommand { + result.Type = "stdio" + mcpCustomLog.Printf("Inferred MCP type as stdio (has command field)") + } else if _, hasContainer := config.GetString("container"); hasContainer { + result.Type = "stdio" + mcpCustomLog.Printf("Inferred MCP type as stdio (has container field)") + } else { + return missingMCPTypeError(toolName) + } + return nil +} + +func missingMCPTypeError(toolName string) error { + mcpCustomLog.Printf("Unable to determine MCP type for tool '%s': missing type, url, command, or container", toolName) + return fmt.Errorf( + "unable to determine MCP type for tool '%s': missing type, url, command, or container. "+ + "Must specify one of: 'type' (stdio/http), 'url' (for HTTP MCP), 'command' (for command-based), or 'container' (for Docker-based). "+ + "Example:\n"+ + "mcp-servers:\n"+ + " %s:\n"+ + " command: \"npx @my/tool\"\n"+ + " args: [\"--port\", \"3000\"]", + toolName, toolName, + ) +} + +func extractMCPCommonFields(result *parser.RegistryMCPServerConfig, config MapToolConfig) { + if registry, hasRegistry := config.GetString("registry"); hasRegistry { + result.Registry = registry + } +} + +func extractMCPTypeSpecificFields(result *parser.RegistryMCPServerConfig, config MapToolConfig, toolName string) error { + mcpCustomLog.Printf("Extracting fields for MCP type: %s", result.Type) + switch result.Type { + case "stdio": + extractStdioMCPFields(result, config) + return nil + case "http": + return extractHTTPMCPFields(result, config, toolName) + default: + return unsupportedMCPTypeError(result.Type, toolName) + } +} + +func extractStdioMCPFields(result *parser.RegistryMCPServerConfig, config MapToolConfig) { + if command, hasCommand := config.GetString("command"); hasCommand { + result.Command = command + } + if container, hasContainer := config.GetString("container"); hasContainer { + result.Container = container + } + if version, hasVersion := config.GetString("version"); hasVersion { + result.Version = version + } + if args, hasArgs := config.GetStringArray("args"); hasArgs { + result.Args = args + } + if entrypoint, hasEntrypoint := config.GetString("entrypoint"); hasEntrypoint { + result.Entrypoint = entrypoint + } + if entrypointArgs, hasEntrypointArgs := config.GetStringArray("entrypointArgs"); hasEntrypointArgs { + result.EntrypointArgs = entrypointArgs + } + if mounts, hasMounts := config.GetStringArray("mounts"); hasMounts { + result.Mounts = mounts + } + if env, hasEnv := config.GetStringMap("env"); hasEnv { + result.Env = env + } + if proxyArgs, hasProxyArgs := config.GetStringArray("proxy-args"); hasProxyArgs { + result.ProxyArgs = proxyArgs + } +} + +func extractHTTPMCPFields(result *parser.RegistryMCPServerConfig, config MapToolConfig, toolName string) error { + if url, hasURL := config.GetString("url"); hasURL { + result.URL = url + } else { + return missingHTTPMCPURLError(toolName) + } + if headers, hasHeaders := config.GetStringMap("headers"); hasHeaders { + result.Headers = headers + } + if authVal, hasAuth := config.GetAny("auth"); hasAuth { + result.Auth = parseMCPAuthConfig(authVal) + } + return nil +} + +func parseMCPAuthConfig(authVal any) *types.MCPAuthConfig { + if authMap, ok := authVal.(map[string]any); ok { + authConfig := &types.MCPAuthConfig{} + if authType, ok := authMap["type"].(string); ok { + authConfig.Type = authType + } + if audience, ok := authMap["audience"].(string); ok { + authConfig.Audience = audience + } + if authConfig.Type != "" { + return authConfig + } + } + if authCfg, ok := authVal.(*types.MCPAuthConfig); ok { + return authCfg + } + return nil +} + +func missingHTTPMCPURLError(toolName string) error { + mcpCustomLog.Printf("HTTP MCP tool '%s' missing required 'url' field", toolName) + return fmt.Errorf( + "http MCP tool '%s' missing required 'url' field. HTTP MCP servers must specify a URL endpoint. "+ + "Example:\n"+ + "mcp-servers:\n"+ + " %s:\n"+ + " type: http\n"+ + " url: \"https://api.example.com/mcp\"\n"+ + " headers:\n"+ + " Authorization: \"****** secrets.API_KEY }}\"", + toolName, toolName, + ) +} + +func unsupportedMCPTypeError(mcpType string, toolName string) error { + mcpCustomLog.Printf("Unsupported MCP type '%s' for tool '%s'", mcpType, toolName) + return fmt.Errorf( + "unsupported MCP type '%s' for tool '%s'. Valid types are: stdio, http. "+ + "Example:\n"+ + "mcp-servers:\n"+ + " %s:\n"+ + " type: stdio\n"+ + " command: \"npx @my/tool\"\n"+ + " args: [\"--port\", \"3000\"]", + mcpType, toolName, toolName) +} + +func finalizeStdioMCPConfig(result *parser.RegistryMCPServerConfig) { + if result.Type == "stdio" && result.Container == "" && result.Command != "" { + containerConfig := getWellKnownContainer(result.Command) + if containerConfig != nil { + mcpCustomLog.Printf("Auto-assigning container for command '%s': %s", result.Command, containerConfig.Image) + result.Container = containerConfig.Image + result.Entrypoint = containerConfig.Entrypoint + result.EntrypointArgs = result.Args + result.Args = nil + result.Command = "" + } + } + if result.Type == "stdio" && result.Container != "" && result.Version != "" { + result.Container = result.Container + ":" + result.Version + result.Version = "" + } +} + +// hasMCPConfig checks if a tool configuration has MCP configuration +func hasMCPConfig(toolConfig map[string]any) (bool, string) { + // Check for direct type field + if mcpType, hasType := toolConfig["type"]; hasType { + if typeStr, ok := mcpType.(string); ok && parser.IsMCPType(typeStr) { + // Normalize "local" to "stdio" for consistency + if typeStr == "local" { + return true, "stdio" + } + return true, typeStr + } + } + + // Infer type from presence of fields (same logic as getMCPConfig) + if _, hasURL := toolConfig["url"]; hasURL { + return true, "http" + } else if _, hasCommand := toolConfig["command"]; hasCommand { + return true, "stdio" + } else if _, hasContainer := toolConfig["container"]; hasContainer { + return true, "stdio" + } + + return false, "" +} diff --git a/.refactor-work/bak/mcp_property_validation.go b/.refactor-work/bak/mcp_property_validation.go new file mode 100644 index 00000000000..8f65c42000b --- /dev/null +++ b/.refactor-work/bak/mcp_property_validation.go @@ -0,0 +1,270 @@ +// This file provides focused MCP property and schema-shaping validation helpers. +// +// # MCP Property Validation +// +// - validateStringProperty() - Validates required MCP string properties +// - validateMCPRequirements() - Validates type-specific MCP requirements +// - buildSchemaMCPConfig() - Projects tool config to schema-compatible MCP fields +// +// This file contains validation details used by entry points in +// mcp_config_validation.go. + +package workflow + +import ( + "fmt" + "strings" + + "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/logger" + "github.com/github/gh-aw/pkg/parser" +) + +var mcpPropertyValidationLog = logger.New("workflow:mcp_property_validation") + +// validateStringProperty validates that a property is a string and returns appropriate error message +func validateStringProperty(toolName, propertyName string, value any, exists bool) error { + field := fmt.Sprintf("mcp-servers.%s.%s", toolName, propertyName) + if !exists { + return NewValidationError( + field, + "", + fmt.Sprintf("missing required property '%s'; expected this field in MCP configuration", propertyName), + fmt.Sprintf("Example:\n\ntools:\n %s:\n %s: \"value\"\n\nSee: %s", toolName, propertyName, constants.DocsToolsURL), + ) + } + if _, ok := value.(string); !ok { + return NewValidationError( + field, + fmt.Sprintf("%v", value), + fmt.Sprintf("'%s' must be a string, got %T", propertyName, value), + fmt.Sprintf("Example:\n\ntools:\n %s:\n %s: \"my-value\"\n\nSee: %s", toolName, propertyName, constants.DocsToolsURL), + ) + } + return nil +} + +// validateMCPRequirements validates the specific requirements for MCP configuration +func validateMCPRequirements(toolName string, mcpConfig map[string]any, toolConfig map[string]any) error { + mcpPropertyValidationLog.Printf("Validating MCP requirements for tool: %s", toolName) + typeStr, err := resolveMCPType(toolName, mcpConfig) + if err != nil { + return err + } + if !parser.IsMCPType(typeStr) { + return NewValidationError( + fmt.Sprintf("mcp-servers.%s.type", toolName), + typeStr, + "'type' must be one of: stdio, http. Note: 'local' is accepted for backward compatibility and treated as 'stdio'", + fmt.Sprintf("Example:\n\ntools:\n %s:\n type: stdio\n command: \"node server.js\"\n\nSee: %s", toolName, constants.DocsToolsURL), + ) + } + switch typeStr { + case "http": + return validateHTTPMCPRequirements(toolName, mcpConfig, toolConfig) + case "stdio": + return validateStdioMCPRequirements(toolName, mcpConfig, toolConfig) + } + return nil +} + +func resolveMCPType(toolName string, mcpConfig map[string]any) (string, error) { + mcpType, hasType := mcpConfig["type"] + if hasType { + typeStr, ok := mcpType.(string) + if !ok { + return "", NewValidationError( + fmt.Sprintf("mcp-servers.%s.type", toolName), + fmt.Sprintf("%v", mcpType), + fmt.Sprintf("'type' must be a string, got %T. Valid types per MCP Gateway Specification: stdio, http. Note: 'local' is accepted for backward compatibility and treated as 'stdio'", mcpType), + fmt.Sprintf("Example:\n\ntools:\n %s:\n type: \"stdio\"\n command: \"node server.js\"\n\nSee: %s", toolName, constants.DocsToolsURL), + ) + } + mcpPropertyValidationLog.Printf("Tool %s: explicit MCP type=%s", toolName, typeStr) + return normalizeMCPTypeString(typeStr), nil + } + typeStr := inferMCPType(mcpConfig) + if typeStr == "" { + return "", NewValidationError( + "mcp-servers."+toolName, + "", + "unable to determine MCP type: expected 'type' or one of 'url', 'command', 'container'", + fmt.Sprintf("Must specify one of: type, url, command, or container.\n\nExample:\n\ntools:\n %s:\n type: stdio\n command: \"node server.js\"\n args: [\"--port\", \"3000\"]\n\nSee: %s", toolName, constants.DocsToolsURL), + ) + } + mcpPropertyValidationLog.Printf("Tool %s: inferred MCP type=%s", toolName, typeStr) + return normalizeMCPTypeString(typeStr), nil +} + +func normalizeMCPTypeString(typeStr string) string { + normalized := strings.ToLower(strings.TrimSpace(typeStr)) + if normalized == "local" { + return "stdio" + } + return normalized +} + +func validateHTTPMCPRequirements(toolName string, mcpConfig map[string]any, toolConfig map[string]any) error { + url, hasURL := mcpConfig["url"] + if _, hasContainer := mcpConfig["container"]; hasContainer { + return NewValidationError( + fmt.Sprintf("mcp-servers.%s.container", toolName), + "container", + "HTTP MCP configuration cannot use 'container'; expected URL-based HTTP configuration", + fmt.Sprintf("Example:\n\ntools:\n %s:\n type: http\n url: \"https://api.example.com/mcp\"\n headers:\n Authorization: \"****** secrets.API_KEY }}\"\n\nSee: %s", toolName, constants.DocsToolsURL), + ) + } + if _, hasMounts := toolConfig["mounts"]; hasMounts { + return NewValidationError( + fmt.Sprintf("mcp-servers.%s.mounts", toolName), + "mounts", + "expected HTTP MCP configuration without mounts", + fmt.Sprintf("Example:\n\ntools:\n %s:\n type: http\n url: \"https://api.example.com/mcp\"\n\nSee: %s", toolName, constants.DocsToolsURL), + ) + } + if err := validateHTTPMCPAuth(toolName, toolConfig); err != nil { + return err + } + return validateStringProperty(toolName, "url", url, hasURL) +} + +func validateHTTPMCPAuth(toolName string, toolConfig map[string]any) error { + authRaw, hasAuth := toolConfig["auth"] + if !hasAuth { + return nil + } + authMap, ok := authRaw.(map[string]any) + if !ok { + return mcpAuthValidationError(toolName, fmt.Sprintf("%v", authRaw), "expected 'auth' to be an object") + } + authType, hasAuthType := authMap["type"] + if !hasAuthType { + return mcpAuthTypeValidationError(toolName, "", "'auth.type' is required for HTTP auth configuration") + } + authTypeStr, ok := authType.(string) + if !ok || authTypeStr == "" { + return mcpAuthTypeValidationError(toolName, fmt.Sprintf("%v", authType), "'auth.type' must be a non-empty string") + } + if authTypeStr != "github-oidc" { + return mcpAuthTypeValidationError(toolName, authTypeStr, fmt.Sprintf("'auth.type' value %q is not supported; expected 'github-oidc'", authTypeStr)) + } + return nil +} + +func mcpAuthValidationError(toolName, value, message string) error { + return NewValidationError( + fmt.Sprintf("mcp-servers.%s.auth", toolName), + value, + message, + fmt.Sprintf("Example:\n\ntools:\n %s:\n type: http\n url: \"https://api.example.com/mcp\"\n auth:\n type: github-oidc\n\nSee: %s", toolName, constants.DocsToolsURL), + ) +} + +func mcpAuthTypeValidationError(toolName, value, message string) error { + return NewValidationError( + fmt.Sprintf("mcp-servers.%s.auth.type", toolName), + value, + message, + fmt.Sprintf("Example:\n\ntools:\n %s:\n type: http\n url: \"https://api.example.com/mcp\"\n auth:\n type: github-oidc\n\nSee: %s", toolName, constants.DocsToolsURL), + ) +} + +func validateStdioMCPRequirements(toolName string, mcpConfig map[string]any, toolConfig map[string]any) error { + if _, hasAuth := toolConfig["auth"]; hasAuth { + return mcpAuthValidationError(toolName, "auth", "'auth' is only supported for HTTP servers (type: 'http')") + } + command, hasCommand := mcpConfig["command"] + container, hasContainer := mcpConfig["container"] + if hasCommand && hasContainer { + return NewValidationError( + "mcp-servers."+toolName, + "command + container", + "cannot specify both 'container' and 'command'; expected exactly one of them for stdio MCP servers", + fmt.Sprintf("Choose one.\n\nExample with command:\n\ntools:\n %s:\n command: \"node server.js\"\n\nExample with container:\n\ntools:\n %s:\n container: \"my-registry/my-tool\"\n version: \"latest\"\n\nSee: %s", toolName, toolName, constants.DocsToolsURL), + ) + } + if err := validateStdioMCPCommandOrContainer(toolName, command, container, hasCommand, hasContainer); err != nil { + return err + } + if mountsRaw, hasMounts := toolConfig["mounts"]; hasMounts { + return validateMCPMountsSyntax(toolName, mountsRaw) + } + return nil +} + +func validateStdioMCPCommandOrContainer(toolName string, command, container any, hasCommand, hasContainer bool) error { + if hasCommand { + return validateStringProperty(toolName, "command", command, true) + } + if hasContainer { + return validateStringProperty(toolName, "container", container, true) + } + return NewValidationError( + "mcp-servers."+toolName, + "", + "must specify either 'command' or 'container' for stdio MCP servers", + fmt.Sprintf("Example with command:\n\ntools:\n %s:\n command: \"node server.js\"\n args: [\"--port\", \"3000\"]\n\nExample with container:\n\ntools:\n %s:\n container: \"my-registry/my-tool\"\n version: \"latest\"\n\nSee: %s", toolName, toolName, constants.DocsToolsURL), + ) +} + +// mcpSchemaTopLevelFields is the set of properties defined at the top level of +// mcp_config_schema.json. Only these fields should be passed to +// parser.ValidateMCPConfigWithSchema; the schema uses additionalProperties: false +// so any extra field would cause a spurious validation failure. +// +// WARNING: This map must be kept in sync with the properties defined in +// pkg/parser/schemas/mcp_config_schema.json. If you add or remove a property +// from that schema, update this map accordingly. +var mcpSchemaTopLevelFields = map[string]bool{ + "type": true, + "registry": true, + "url": true, + "command": true, + "container": true, + "args": true, + "entrypoint": true, + "entrypointArgs": true, + "mounts": true, + "env": true, + "headers": true, + "network": true, + "allowed": true, + "version": true, +} + +// buildSchemaMCPConfig extracts only the fields defined in mcp_config_schema.json +// from a full tool config map. Tool-specific fields that are not part of the MCP +// schema (e.g. auth, proxy-args, mode, github-token) are excluded so that schema +// validation does not fail on fields unknown to the schema. +// +// If the 'type' field is absent but can be inferred from other fields (url → http, +// command/container → stdio), the inferred type is injected. This is necessary because +// the schema's if/then conditions use properties-based matching which is vacuously true +// when 'type' is absent, causing contradictory constraints to fire for valid configs +// that rely on type inference. +func buildSchemaMCPConfig(toolConfig map[string]any) map[string]any { + result := make(map[string]any, len(mcpSchemaTopLevelFields)) + for field := range mcpSchemaTopLevelFields { + if value, exists := toolConfig[field]; exists { + result[field] = value + } + } + // If 'type' is not present, infer it from other fields so the schema's + // if/then conditions do not fire vacuously and reject valid inferred-type configs. + // + // Why this is necessary: the JSON Schema draft-07 `properties` keyword is + // vacuously satisfied when the checked property is absent. This means the + // `if {"properties": {"type": {"enum": ["stdio"]}}}` condition evaluates to + // true even when 'type' is not in the config, causing the stdio `then` clause + // (requiring command/container) to apply unexpectedly for HTTP-only configs. + // Injecting the inferred type before schema validation ensures the correct + // if/then branch fires. When inference is not possible (empty string returned), + // the map is left without a 'type'; the schema's anyOf constraint will then + // report a clear "missing required property" error on its own. + if _, hasType := result["type"]; !hasType { + if inferred := inferMCPType(result); inferred != "" { + result["type"] = inferred + } + } + return result +} diff --git a/.refactor-work/bak/mcp_renderer_builtin.go b/.refactor-work/bak/mcp_renderer_builtin.go new file mode 100644 index 00000000000..f73e009cd64 --- /dev/null +++ b/.refactor-work/bak/mcp_renderer_builtin.go @@ -0,0 +1,395 @@ +package workflow + +import ( + "strings" + + "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/logger" +) + +type mcpEnvVar struct { + name string + value string + isLiteral bool +} + +var mcpRendererBuiltinLog = logger.New("workflow:mcp_renderer_builtin") + +// RenderPlaywrightMCP generates the Playwright MCP server configuration +func (r *MCPConfigRendererUnified) RenderPlaywrightMCP(yaml *strings.Builder, playwrightTool any) { + mcpRendererLog.Printf("Rendering Playwright MCP: format=%s, inline_args=%t", r.options.Format, r.options.InlineArgs) + + // Parse playwright tool configuration to strongly-typed struct + playwrightConfig := parsePlaywrightTool(playwrightTool) + + if r.options.Format == "toml" { + r.renderPlaywrightTOML(yaml, playwrightConfig) + // Add guard policies for TOML format as a separate section + if len(r.options.WriteSinkGuardPolicies) > 0 { + mcpRendererLog.Print("Adding guard-policies to playwright TOML (derived from GitHub guard-policy)") + renderGuardPoliciesToml(yaml, r.options.WriteSinkGuardPolicies, "playwright") + } + return + } + + // JSON format + renderPlaywrightMCPConfigWithOptions(yaml, playwrightConfig, r.options.IsLast, r.options.IncludeCopilotFields, r.options.InlineArgs, r.options.WriteSinkGuardPolicies) +} + +// renderPlaywrightTOML generates Playwright MCP configuration in TOML format +// Per MCP Gateway Specification v1.0.0 section 3.2.1, stdio-based MCP servers MUST be containerized. +// Uses MCP Gateway spec format: container, entrypointArgs, mounts, and args fields. +func (r *MCPConfigRendererUnified) renderPlaywrightTOML(yaml *strings.Builder, playwrightConfig *PlaywrightToolConfig) { + mcpRendererBuiltinLog.Print("Rendering Playwright MCP in TOML format") + customArgs := getPlaywrightCustomArgs(playwrightConfig) + + // Use official Playwright MCP Docker image (no version tag - only one image) + playwrightImage := "mcr.microsoft.com/playwright/mcp" + + yaml.WriteString(" \n") + yaml.WriteString(" [mcp_servers.playwright]\n") + yaml.WriteString(" container = \"" + playwrightImage + "\"\n") + + // Docker runtime args (goes before container image in docker run command) + // Add security-opt and ipc flags for Chromium browser compatibility in GitHub Actions + // --security-opt seccomp=unconfined: Required for Chromium sandbox to function properly + // --ipc=host: Provides shared memory access required by Chromium + yaml.WriteString(" args = [\n") + yaml.WriteString(" \"--init\",\n") + yaml.WriteString(" \"--network\",\n") + yaml.WriteString(" \"host\",\n") + yaml.WriteString(" \"--security-opt\",\n") + yaml.WriteString(" \"seccomp=unconfined\",\n") + yaml.WriteString(" \"--ipc=host\",\n") + yaml.WriteString(" ]\n") + + // Entrypoint args for Playwright MCP server (goes after container image) + yaml.WriteString(" entrypointArgs = [\n") + yaml.WriteString(" \"--output-dir\",\n") + yaml.WriteString(" \"/tmp/gh-aw/mcp-logs/playwright\"") + + // Append custom args if present + writeArgsToYAML(yaml, customArgs, " ") + + yaml.WriteString("\n") + yaml.WriteString(" ]\n") + + // Add volume mounts + yaml.WriteString(" mounts = [\"/tmp/gh-aw/mcp-logs:/tmp/gh-aw/mcp-logs:rw\"]\n") +} + +// RenderSafeOutputsMCP generates the Safe Outputs MCP server configuration +func (r *MCPConfigRendererUnified) RenderSafeOutputsMCP(yaml *strings.Builder, workflowData *WorkflowData) { + mcpRendererLog.Printf("Rendering Safe Outputs MCP: format=%s", r.options.Format) + + if r.options.Format == "toml" { + r.renderSafeOutputsTOML(yaml, workflowData) + return + } + + // JSON format + renderSafeOutputsMCPConfigWithOptions(yaml, r.options.IsLast, r.options.IncludeCopilotFields, workflowData) +} + +// renderSafeOutputsTOML generates Safe Outputs MCP configuration in TOML format +// Uses containerized stdio transport in the gh-aw-node image, overriding the container's +// default entrypoint to run the stdio MCP server script. +func (r *MCPConfigRendererUnified) renderSafeOutputsTOML(yaml *strings.Builder, workflowData *WorkflowData) { + containerImage := resolveMCPGatewayContainerImage(constants.DefaultGhAwNodeImage, workflowData) + yaml.WriteString(" \n") + yaml.WriteString(" [mcp_servers." + constants.SafeOutputsMCPServerID.String() + "]\n") + yaml.WriteString(" container = \"" + containerImage + "\"\n") + yaml.WriteString(" mounts = [\"" + constants.DefaultWorkspaceMount + "\", \"" + constants.DefaultSafeOutputsMount + "\", \"" + constants.DefaultTmpGhAwMount + "\"]\n") + yaml.WriteString(" args = [\"-w\", \"$GITHUB_WORKSPACE\"]\n") + yaml.WriteString(" entrypoint = \"sh\"\n") + yaml.WriteString(" entrypointArgs = [\"-c\", \"sh ${RUNNER_TEMP}/gh-aw/safeoutputs/start_safe_outputs_mcp.sh\"]\n") + yaml.WriteString(" env_vars = [\"DEBUG\", \"DEFAULT_BRANCH\", \"GH_AW_ASSETS_ALLOWED_EXTS\", \"GH_AW_ASSETS_BRANCH\", \"GH_AW_ASSETS_MAX_SIZE_KB\", \"GH_AW_MCP_LOG_DIR\", \"GH_AW_SAFE_OUTPUTS\", \"GH_AW_SAFE_OUTPUTS_CONFIG_PATH\", \"GH_AW_SAFE_OUTPUTS_TOOLS_PATH\", \"GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST\", \"GITHUB_REPOSITORY\", \"GITHUB_SHA\", \"GITHUB_TOKEN\", \"GITHUB_WORKSPACE\", \"RUNNER_TEMP\"]\n") + + // Check if GitHub tool has guard-policies configured (or auto-lockdown will run) + // If so, generate a linked write-sink guard-policy for safeoutputs + guardPolicies := deriveWriteSinkGuardPolicyFromWorkflow(workflowData) + if len(guardPolicies) > 0 { + mcpRendererLog.Print("Adding guard-policies to safeoutputs TOML (derived from GitHub guard-policy or auto-lockdown detection)") + // Render guard-policies in TOML format + renderGuardPoliciesToml(yaml, guardPolicies, constants.SafeOutputsMCPServerID.String()) + } +} + +// RenderMCPScriptsMCP generates the MCP Scripts server configuration +func (r *MCPConfigRendererUnified) RenderMCPScriptsMCP(yaml *strings.Builder, mcpScripts *MCPScriptsConfig, workflowData *WorkflowData) { + mcpRendererLog.Printf("Rendering MCP Scripts: format=%s", r.options.Format) + + if r.options.Format == "toml" { + r.renderMCPScriptsTOML(yaml, mcpScripts, workflowData) + // Add guard policies for TOML format as a separate section + if len(r.options.WriteSinkGuardPolicies) > 0 { + mcpRendererLog.Print("Adding guard-policies to mcp-scripts TOML (derived from GitHub guard-policy)") + renderGuardPoliciesToml(yaml, r.options.WriteSinkGuardPolicies, constants.MCPScriptsMCPServerID.String()) + } + return + } + + // JSON format + renderMCPScriptsMCPConfigWithOptions(yaml, mcpScripts, r.options.IsLast, r.options.IncludeCopilotFields, workflowData, r.options.WriteSinkGuardPolicies) +} + +// renderMCPScriptsTOML generates MCP Scripts configuration in TOML format +// Uses HTTP transport exclusively +func (r *MCPConfigRendererUnified) renderMCPScriptsTOML(yaml *strings.Builder, mcpScripts *MCPScriptsConfig, workflowData *WorkflowData) { + yaml.WriteString(" \n") + yaml.WriteString(" [mcp_servers." + constants.MCPScriptsMCPServerID.String() + "]\n") + yaml.WriteString(" type = \"http\"\n") + + // Determine host based on whether agent is disabled + host := "host.docker.internal" + if workflowData != nil && workflowData.SandboxConfig != nil && workflowData.SandboxConfig.Agent != nil && workflowData.SandboxConfig.Agent.Disabled { + // When agent is disabled (no firewall), use localhost instead of host.docker.internal + host = "localhost" + mcpRendererLog.Print("Using localhost for mcp-scripts (agent disabled)") + } else { + mcpRendererLog.Print("Using host.docker.internal for mcp-scripts (agent enabled)") + } + + yaml.WriteString(" url = \"http://" + host + ":$GH_AW_MCP_SCRIPTS_PORT\"\n") + yaml.WriteString(" headers = { Authorization = \"$GH_AW_MCP_SCRIPTS_API_KEY\" }\n") + // Note: env_vars is not supported for HTTP transport in MCP configuration + // Environment variables are passed via the workflow job's env: section instead +} + +// RenderAgenticWorkflowsMCP generates the Agentic Workflows MCP server configuration +func (r *MCPConfigRendererUnified) RenderAgenticWorkflowsMCP(yaml *strings.Builder) { + mcpRendererLog.Printf("Rendering Agentic Workflows MCP: format=%s, action_mode=%s", r.options.Format, r.options.ActionMode) + + if r.options.Format == "toml" { + r.renderAgenticWorkflowsTOML(yaml) + // Add guard policies for TOML format as a separate section + if len(r.options.WriteSinkGuardPolicies) > 0 { + mcpRendererLog.Print("Adding guard-policies to agentic-workflows TOML (derived from GitHub guard-policy)") + renderGuardPoliciesToml(yaml, r.options.WriteSinkGuardPolicies, constants.AgenticWorkflowsMCPServerID.String()) + } + return + } + + // JSON format + renderAgenticWorkflowsMCPConfigWithOptions(yaml, r.options.IsLast, r.options.IncludeCopilotFields, r.options.ActionMode, r.options.WriteSinkGuardPolicies) +} + +// renderAgenticWorkflowsTOML generates Agentic Workflows MCP configuration in TOML format +// Per MCP Gateway Specification v1.0.0 section 3.2.1, stdio-based MCP servers MUST be containerized. +func (r *MCPConfigRendererUnified) renderAgenticWorkflowsTOML(yaml *strings.Builder) { + mcpRendererBuiltinLog.Printf("Rendering Agentic Workflows MCP in TOML format: action_mode=%s", r.options.ActionMode) + yaml.WriteString(" \n") + yaml.WriteString(" [mcp_servers." + constants.AgenticWorkflowsMCPServerID.String() + "]\n") + + containerImage := constants.DefaultAlpineImage + var entrypoint string + var entrypointArgs []string + var mounts []string + + if r.options.ActionMode.IsDev() { + // Dev mode: Use locally built Docker image which includes gh-aw binary and gh CLI + // The Dockerfile sets ENTRYPOINT ["gh-aw"] and CMD ["mcp-server", "--validate-actor"] + // So we don't need to specify entrypoint or entrypointArgs + containerImage = constants.DevModeGhAwImage + entrypoint = "" // Use container's default ENTRYPOINT + entrypointArgs = nil // Use container's default CMD + // Only mount workspace and temp directory - binary and gh CLI are in the image + mounts = []string{constants.DefaultWorkspaceMount, constants.DefaultTmpGhAwMount} + } else { + // Release mode: Use minimal Alpine image with mounted binaries + entrypoint = GhAwBinaryPath + entrypointArgs = []string{"mcp-server", "--validate-actor"} + // Mount gh-aw binary, gh CLI binary, workspace, and temp directory + mounts = []string{constants.DefaultGhAwMount, constants.DefaultGhBinaryMount, constants.DefaultWorkspaceMount, constants.DefaultTmpGhAwMount} + } + + yaml.WriteString(" container = \"" + containerImage + "\"\n") + + // Only write entrypoint if it's specified (release mode) + // In dev mode, use the container's default ENTRYPOINT + if entrypoint != "" { + yaml.WriteString(" entrypoint = \"" + entrypoint + "\"\n") + } + + // Only write entrypointArgs if specified (release mode) + // In dev mode, use the container's default CMD + if entrypointArgs != nil { + yaml.WriteString(" entrypointArgs = [") + for i, arg := range entrypointArgs { + if i > 0 { + yaml.WriteString(", ") + } + yaml.WriteString("\"" + arg + "\"") + } + yaml.WriteString("]\n") + } + + // Write mounts + yaml.WriteString(" mounts = [") + for i, mount := range mounts { + if i > 0 { + yaml.WriteString(", ") + } + yaml.WriteString("\"" + mount + "\"") + } + yaml.WriteString("]\n") + + yaml.WriteString(" env_vars = [\"DEBUG\", \"GH_TOKEN\", \"GITHUB_TOKEN\", \"GITHUB_ACTOR\", \"GITHUB_REPOSITORY\"]\n") +} + +// renderSafeOutputsMCPConfigWithOptions generates the Safe Outputs MCP server configuration with engine-specific options. +// The server runs as a containerized stdio MCP server in the published gh-aw node image. +func renderSafeOutputsMCPConfigWithOptions(yaml *strings.Builder, isLast bool, includeCopilotFields bool, workflowData *WorkflowData) { + mcpRendererBuiltinLog.Printf("Rendering Safe Outputs MCP config with options: isLast=%v, includeCopilotFields=%v", isLast, includeCopilotFields) + containerImage := resolveMCPGatewayContainerImage(constants.DefaultGhAwNodeImage, workflowData) + yaml.WriteString(" \"" + constants.SafeOutputsMCPServerID.String() + "\": {\n") + + if includeCopilotFields { + yaml.WriteString(" \"type\": \"stdio\",\n") + } + yaml.WriteString(" \"container\": \"" + containerImage + "\",\n") + yaml.WriteString(" \"mounts\": [\"" + constants.DefaultWorkspaceMount + "\", \"" + constants.DefaultSafeOutputsMount + "\", \"" + constants.DefaultTmpGhAwMount + "\"],\n") + yaml.WriteString(" \"args\": [\"-w\", \"\\${GITHUB_WORKSPACE}\"],\n") + yaml.WriteString(" \"entrypoint\": \"sh\",\n") + yaml.WriteString(" \"entrypointArgs\": [\"-c\", \"sh ${RUNNER_TEMP}/gh-aw/safeoutputs/start_safe_outputs_mcp.sh\"],\n") + yaml.WriteString(" \"env\": {\n") + + envVars := []mcpEnvVar{ + {"DEBUG", "*", true}, + {"DEFAULT_BRANCH", "DEFAULT_BRANCH", false}, + {"GH_AW_ASSETS_ALLOWED_EXTS", "GH_AW_ASSETS_ALLOWED_EXTS", false}, + {"GH_AW_ASSETS_BRANCH", "GH_AW_ASSETS_BRANCH", false}, + {"GH_AW_ASSETS_MAX_SIZE_KB", "GH_AW_ASSETS_MAX_SIZE_KB", false}, + {"GH_AW_MCP_LOG_DIR", "GH_AW_MCP_LOG_DIR", false}, + {"GH_AW_SAFE_OUTPUTS", "GH_AW_SAFE_OUTPUTS", false}, + {"GH_AW_SAFE_OUTPUTS_CONFIG_PATH", "GH_AW_SAFE_OUTPUTS_CONFIG_PATH", false}, + {"GH_AW_SAFE_OUTPUTS_TOOLS_PATH", "GH_AW_SAFE_OUTPUTS_TOOLS_PATH", false}, + {"GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST", "GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST", false}, + {"GITHUB_REPOSITORY", "GITHUB_REPOSITORY", false}, + {"GITHUB_SHA", "GITHUB_SHA", false}, + {"GITHUB_TOKEN", "GITHUB_TOKEN", false}, + {"GITHUB_WORKSPACE", "GITHUB_WORKSPACE", false}, + {"RUNNER_TEMP", "RUNNER_TEMP", false}, + } + + renderMCPEnvVars(yaml, envVars, " ") + yaml.WriteString(" }") + + // Check if GitHub tool has guard-policies configured (or auto-lockdown will run) + // If so, generate a linked write-sink guard-policy for safeoutputs + guardPolicies := deriveWriteSinkGuardPolicyFromWorkflow(workflowData) + + // Add guard-policies if configured + if len(guardPolicies) > 0 { + mcpRendererBuiltinLog.Print("Adding guard-policies to safeoutputs (derived from GitHub guard-policy or auto-lockdown detection)") + yaml.WriteString(",\n") + renderGuardPoliciesJSON(yaml, guardPolicies, " ") + } else { + yaml.WriteString("\n") + } + + if isLast { + yaml.WriteString(" }\n") + } else { + yaml.WriteString(" },\n") + } +} + +// renderAgenticWorkflowsMCPConfigWithOptions generates the Agentic Workflows MCP server configuration with engine-specific options +// Per MCP Gateway Specification v1.0.0 section 3.2.1, stdio-based MCP servers MUST be containerized. +// Uses MCP Gateway spec format: container, entrypoint, entrypointArgs, and mounts fields. +func renderAgenticWorkflowsMCPConfigWithOptions(yaml *strings.Builder, isLast bool, includeCopilotFields bool, actionMode ActionMode, guardPolicies map[string]any) { + mcpRendererBuiltinLog.Printf("Rendering Agentic Workflows MCP config: isLast=%v, includeCopilotFields=%v, actionMode=%v", isLast, includeCopilotFields, actionMode) + + envVars := []mcpEnvVar{ + {"DEBUG", "*", true}, // Literal value "*" + {"GITHUB_TOKEN", "GITHUB_TOKEN", false}, // Variable reference (gh CLI auto-sets GH_TOKEN from GITHUB_TOKEN if needed) + {"GITHUB_ACTOR", "GITHUB_ACTOR", false}, // Variable reference for actor-based access control + {"GITHUB_REPOSITORY", "GITHUB_REPOSITORY", false}, // Variable reference for repository context + } + + // Use MCP Gateway spec format with container, entrypoint, entrypointArgs, and mounts + yaml.WriteString(" \"" + constants.AgenticWorkflowsMCPServerID.String() + "\": {\n") + + // Add type field for Copilot (per MCP Gateway Specification v1.0.0, use "stdio" for containerized servers) + if includeCopilotFields { + yaml.WriteString(" \"type\": \"stdio\",\n") + } + + renderAgenticWorkflowsMCPRuntimeFields(yaml, actionMode) + yaml.WriteString(" \"env\": {\n") + renderMCPEnvVars(yaml, envVars, " ") + renderMCPEnvCloseAndGuardPolicies(yaml, guardPolicies) + + if isLast { + yaml.WriteString(" }\n") + } else { + yaml.WriteString(" },\n") + } +} + +func renderAgenticWorkflowsMCPRuntimeFields(yaml *strings.Builder, actionMode ActionMode) { + containerImage, entrypoint, entrypointArgs, mounts := agenticWorkflowsMCPRuntime(actionMode) + yaml.WriteString(" \"container\": \"" + containerImage + "\",\n") + if entrypoint != "" { + yaml.WriteString(" \"entrypoint\": \"" + entrypoint + "\",\n") + } + if entrypointArgs != nil { + yaml.WriteString(" \"entrypointArgs\": [") + for i, arg := range entrypointArgs { + if i > 0 { + yaml.WriteString(", ") + } + yaml.WriteString("\"" + arg + "\"") + } + yaml.WriteString("],\n") + } + yaml.WriteString(" \"mounts\": [") + for i, mount := range mounts { + if i > 0 { + yaml.WriteString(", ") + } + yaml.WriteString("\"" + mount + "\"") + } + yaml.WriteString("],\n") + yaml.WriteString(" \"args\": [\"--network\", \"host\", \"-w\", \"\\${GITHUB_WORKSPACE}\"],\n") +} + +func renderMCPEnvCloseAndGuardPolicies(yaml *strings.Builder, guardPolicies map[string]any) { + if len(guardPolicies) > 0 { + yaml.WriteString(" },\n") + renderGuardPoliciesJSON(yaml, guardPolicies, " ") + return + } + yaml.WriteString(" }\n") +} + +func renderMCPEnvVars(yaml *strings.Builder, envVars []mcpEnvVar, indent string) { + for i, envVar := range envVars { + comma := "," + if i == len(envVars)-1 { + comma = "" + } + yaml.WriteString(indent + "\"" + envVar.name + "\": \"" + renderMCPEnvValue(envVar) + "\"" + comma + "\n") + } +} + +func renderMCPEnvValue(envVar mcpEnvVar) string { + if envVar.isLiteral { + return envVar.value + } + return "\\${" + envVar.value + "}" +} + +func agenticWorkflowsMCPRuntime(actionMode ActionMode) (string, string, []string, []string) { + if actionMode.IsDev() { + mcpRendererBuiltinLog.Print("Using dev mode configuration with locally built Docker image") + return constants.DevModeGhAwImage, "", nil, []string{constants.DefaultWorkspaceMount, constants.DefaultTmpGhAwMount} + } + return constants.DefaultAlpineImage, GhAwBinaryPath, []string{"mcp-server", "--validate-actor"}, []string{ + constants.DefaultGhAwMount, + constants.DefaultGhBinaryMount, + constants.DefaultWorkspaceMount, + constants.DefaultTmpGhAwMount, + } +} diff --git a/.refactor-work/bak/mcp_renderer_github.go b/.refactor-work/bak/mcp_renderer_github.go new file mode 100644 index 00000000000..66f04b7d28d --- /dev/null +++ b/.refactor-work/bak/mcp_renderer_github.go @@ -0,0 +1,315 @@ +package workflow + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/github/gh-aw/pkg/console" + "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/sliceutil" +) + +// RenderGitHubMCP generates the GitHub MCP server configuration +// Supports both local (Docker) and remote (hosted) modes +func (r *MCPConfigRendererUnified) RenderGitHubMCP(yaml *strings.Builder, githubTool map[string]any, workflowData *WorkflowData) { + githubType := getGitHubType(githubTool) + readOnly := getGitHubReadOnly() + lockdown := getGitHubLockdown(githubTool) + explicitGuardPolicies := getGitHubGuardPolicies(githubTool) + warnIfIntegrityReactionsIgnored(githubTool, workflowData) + shouldUseStepOutputForGuardPolicy := len(explicitGuardPolicies) == 0 && !hasGitHubApp(githubTool) + toolsets := getGitHubToolsets(githubTool) + features := getGitHubFeatures(githubTool) + + mcpRendererLog.Printf("Rendering GitHub MCP: type=%s, read_only=%t, lockdown=%t (explicit=%t), guard_from_step=%t, toolsets=%v, features=%s, format=%s", + githubType, readOnly, lockdown, hasGitHubLockdownExplicitlySet(githubTool), shouldUseStepOutputForGuardPolicy, toolsets, features, r.options.Format) + + if r.options.Format == "toml" { + mcpRendererLog.Print("GitHub MCP format=toml, dispatching to renderGitHubTOML") + r.renderGitHubTOML(yaml, githubTool, workflowData) + return + } + + yaml.WriteString(" \"github\": {\n") + r.renderGitHubMCPJSONBody(yaml, githubTool, githubType, readOnly, lockdown, shouldUseStepOutputForGuardPolicy, toolsets, features, explicitGuardPolicies) + if r.options.IsLast { + yaml.WriteString(" }\n") + } else { + yaml.WriteString(" },\n") + } +} + +func warnIfIntegrityReactionsIgnored(githubTool map[string]any, workflowData *WorkflowData) { + if !isFeatureEnabled(constants.IntegrityReactionsFeatureFlag, workflowData) || !hasReactionFieldsInToolConfig(githubTool) { + return + } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage( + "integrity-reactions: endorsement/disapproval reactions are ignored in MCP gateway mode because "+ + "reaction authors cannot be identified from the GitHub MCP server. Reactions are only enforced "+ + "in proxy mode (DIFC proxy / CLI proxy).")) +} + +func (r *MCPConfigRendererUnified) renderGitHubMCPJSONBody(yaml *strings.Builder, githubTool map[string]any, githubType GitHubMCPMode, readOnly bool, lockdown bool, guardPoliciesFromStep bool, toolsets string, features string, guardPolicies map[string]any) { + if githubType == GitHubMCPModeRemote { + r.renderGitHubMCPRemoteJSON(yaml, githubTool, readOnly, lockdown, guardPoliciesFromStep, toolsets, features, guardPolicies) + return + } + r.renderGitHubMCPLocalJSON(yaml, githubTool, readOnly, lockdown, guardPoliciesFromStep, toolsets, features, guardPolicies) +} + +func (r *MCPConfigRendererUnified) renderGitHubMCPRemoteJSON(yaml *strings.Builder, githubTool map[string]any, readOnly bool, lockdown bool, guardPoliciesFromStep bool, toolsets string, features string, guardPolicies map[string]any) { + mcpRendererLog.Printf("GitHub MCP remote mode selected: copilot_fields=%t", r.options.IncludeCopilotFields) + authValue := "******" + if r.options.IncludeCopilotFields { + authValue = "******" + } + RenderGitHubMCPRemoteConfig(yaml, GitHubMCPRemoteOptions{ + ReadOnly: readOnly, Lockdown: lockdown, LockdownFromStep: false, + GuardPoliciesFromStep: guardPoliciesFromStep, Toolsets: toolsets, Features: features, + AuthorizationValue: authValue, IncludeToolsField: r.options.IncludeCopilotFields, + AllowedTools: getGitHubAllowedTools(githubTool), IncludeEnvSection: r.options.IncludeCopilotFields, + GuardPolicies: guardPolicies, + }) +} + +func (r *MCPConfigRendererUnified) renderGitHubMCPLocalJSON(yaml *strings.Builder, githubTool map[string]any, readOnly bool, lockdown bool, guardPoliciesFromStep bool, toolsets string, features string, guardPolicies map[string]any) { + githubDockerImageVersion := getGitHubDockerImageVersion(githubTool) + customArgs := getGitHubCustomArgs(githubTool) + mcpRendererLog.Printf("GitHub MCP local docker mode: image_version=%s, custom_args=%d", githubDockerImageVersion, len(customArgs)) + RenderGitHubMCPDockerConfig(yaml, GitHubMCPDockerOptions{ + ReadOnly: readOnly, Lockdown: lockdown, LockdownFromStep: false, + GuardPoliciesFromStep: guardPoliciesFromStep, Toolsets: toolsets, Features: features, + DockerImageVersion: githubDockerImageVersion, CustomArgs: customArgs, + IncludeTypeField: r.options.IncludeCopilotFields, AllowedTools: getGitHubAllowedTools(githubTool), + EffectiveToken: "", GuardPolicies: guardPolicies, + }) +} + +// renderGitHubTOML generates GitHub MCP configuration in TOML format (for Codex engine) +func (r *MCPConfigRendererUnified) renderGitHubTOML(yaml *strings.Builder, githubTool map[string]any, workflowData *WorkflowData) { + githubType := getGitHubType(githubTool) + readOnly := getGitHubReadOnly() + lockdown := getGitHubLockdown(githubTool) + toolsets := getGitHubToolsets(githubTool) + features := getGitHubFeatures(githubTool) + + mcpRendererLog.Printf("Rendering GitHub MCP TOML: type=%s, read_only=%t, lockdown=%t, toolsets=%s, features=%s", githubType, readOnly, lockdown, toolsets, features) + + yaml.WriteString(" \n") + yaml.WriteString(" [mcp_servers.github]\n") + + // Add user_agent field defaulting to workflow identifier + yaml.WriteString(" user_agent = \"" + gitHubTOMLUserAgent(workflowData) + "\"\n") + + // Use tools.startup-timeout if specified, otherwise default to DefaultMCPStartupTimeout + // For GitHub Actions expressions, fall back to default (TOML format doesn't support expressions) + fmt.Fprintf(yaml, " startup_timeout_sec = %d\n", gitHubTOMLStartupTimeout(workflowData)) + + // Use tools.timeout if specified, otherwise default to DefaultToolTimeout + // For GitHub Actions expressions, fall back to default (TOML format doesn't support expressions) + fmt.Fprintf(yaml, " tool_timeout_sec = %d\n", gitHubTOMLToolTimeout(workflowData)) + + // Check if remote mode is enabled + if githubType == GitHubMCPModeRemote { + renderGitHubTOMLRemote(yaml, readOnly) + } else { + renderGitHubTOMLLocal(yaml, githubTool, readOnly, lockdown, toolsets, features) + } +} + +func gitHubTOMLUserAgent(workflowData *WorkflowData) string { + userAgent := "github-agentic-workflow" + if workflowData == nil { + return userAgent + } + if workflowData.EngineConfig != nil && workflowData.EngineConfig.UserAgent != "" { + return workflowData.EngineConfig.UserAgent + } + if workflowData.Name != "" { + return SanitizeArtifactIdentifier(workflowData.Name) + } + return userAgent +} + +func gitHubTOMLStartupTimeout(workflowData *WorkflowData) int { + startupTimeout := int(constants.DefaultMCPStartupTimeout / time.Second) + if workflowData != nil && workflowData.ToolsStartupTimeout != "" { + if n := templatableIntValue(&workflowData.ToolsStartupTimeout); n > 0 { + return n + } + } + return startupTimeout +} + +func gitHubTOMLToolTimeout(workflowData *WorkflowData) int { + toolTimeout := int(constants.DefaultToolTimeout / time.Second) + if workflowData != nil && workflowData.ToolsTimeout != "" { + if n := templatableIntValue(&workflowData.ToolsTimeout); n > 0 { + return n + } + } + return toolTimeout +} + +func renderGitHubTOMLRemote(yaml *strings.Builder, readOnly bool) { + mcpRendererLog.Printf("GitHub MCP TOML remote mode: readonly_endpoint=%t", readOnly) + if readOnly { + yaml.WriteString(" url = \"https://api.githubcopilot.com/mcp-readonly/\"\n") + } else { + yaml.WriteString(" url = \"https://api.githubcopilot.com/mcp/\"\n") + } + yaml.WriteString(" bearer_token_env_var = \"GH_AW_GITHUB_TOKEN\"\n") +} + +func renderGitHubTOMLLocal(yaml *strings.Builder, githubTool map[string]any, readOnly bool, lockdown bool, toolsets string, features string) { + githubDockerImageVersion := getGitHubDockerImageVersion(githubTool) + customArgs := getGitHubCustomArgs(githubTool) + yaml.WriteString(" container = \"ghcr.io/github/github-mcp-server:" + githubDockerImageVersion + "\"\n") + writeGitHubTOMLCustomArgs(yaml, customArgs) + envVars := buildGitHubMCPEnvVars("$GH_AW_GITHUB_TOKEN", "$GITHUB_SERVER_URL", readOnly, lockdown, toolsets, features) + envKeys := sliceutil.SortedKeys(envVars) + writeTOMLInlineStringMapSection(yaml, " ", "env", envVars) + writeGitHubTOMLEnvVars(yaml, envKeys) +} + +func writeGitHubTOMLCustomArgs(yaml *strings.Builder, customArgs []string) { + if len(customArgs) == 0 { + return + } + yaml.WriteString(" args = [\n") + for _, arg := range customArgs { + yaml.WriteString(" " + strconv.Quote(arg) + ",\n") + } + yaml.WriteString(" ]\n") +} + +func writeGitHubTOMLEnvVars(yaml *strings.Builder, envKeys []string) { + yaml.WriteString(" env_vars = [") + for i, key := range envKeys { + if i > 0 { + yaml.WriteString(", ") + } + fmt.Fprintf(yaml, "\"%s\"", key) + } + yaml.WriteString("]\n") +} + +// RenderGitHubMCPDockerConfig renders the GitHub MCP server configuration for Docker (local mode). +// Per MCP Gateway Specification v1.0.0 section 3.2.1, stdio-based MCP servers MUST be containerized. +// Uses MCP Gateway spec format: container, entrypointArgs, and env fields. +// +// Parameters: +// - yaml: The string builder for YAML output +// - options: GitHub MCP Docker rendering options +func RenderGitHubMCPDockerConfig(yaml *strings.Builder, options GitHubMCPDockerOptions) { + mcpRendererLog.Printf("Rendering GitHub MCP Docker config: image=%s, read_only=%t, lockdown=%t", options.DockerImageVersion, options.ReadOnly, options.Lockdown) + + // Add type field if needed (Copilot requires this, Claude doesn't) + // Per MCP Gateway Specification v1.0.0 section 4.1.2, use "stdio" for containerized servers + if options.IncludeTypeField { + yaml.WriteString(" \"type\": \"stdio\",\n") + } + + // MCP Gateway spec fields for containerized stdio servers + yaml.WriteString(" \"container\": \"ghcr.io/github/github-mcp-server:" + options.DockerImageVersion + "\",\n") + + // Append custom args if present (these are Docker runtime args, go before container image) + if len(options.CustomArgs) > 0 { + yaml.WriteString(" \"args\": [\n") + for _, arg := range options.CustomArgs { + quotedArg, _ := json.Marshal(arg) //nolint:jsonmarshalignoredeerror // marshaling a string cannot fail + yaml.WriteString(" " + string(quotedArg) + ",\n") + } + yaml.WriteString(" ],\n") + } + + // Note: tools field is NOT included here - the converter script adds it back + // for Copilot (see convert_gateway_config_copilot.cjs). This keeps the gateway + // config compatible with the schema which doesn't have the tools field. + + // GitHub token (always required) + tokenValue := "$GITHUB_MCP_SERVER_TOKEN" + hostValue := "$GITHUB_SERVER_URL" + if options.IncludeTypeField { + // Copilot engine: keep shell expansion so gateway input remains valid JSON. + tokenValue = "${GITHUB_MCP_SERVER_TOKEN}" + // GitHub host for enterprise deployments (format: https://hostname, e.g. https://myorg.ghe.com). + // GITHUB_SERVER_URL is set by GitHub Actions as a full URL (https://hostname, no trailing slash), + // which matches the format expected by github-mcp-server for GITHUB_HOST. + hostValue = "${GITHUB_SERVER_URL}" + } + + envVars := buildGitHubMCPEnvVars(tokenValue, hostValue, options.ReadOnly, options.Lockdown, options.Toolsets, options.Features) + hasGuardPolicies := hasGitHubMCPGuardPolicies(options.GuardPolicies, options.GuardPoliciesFromStep) + writeJSONStringMapSection(yaml, " ", "env", envVars, hasGuardPolicies) + renderGitHubMCPGuardPolicies(yaml, options.GuardPolicies, options.GuardPoliciesFromStep, " ") +} + +// RenderGitHubMCPRemoteConfig renders the GitHub MCP server configuration for remote (hosted) mode. +// This shared function extracts the duplicate pattern from Claude and Copilot engines. +// +// Parameters: +// - yaml: The string builder for YAML output +// - options: GitHub MCP remote rendering options +func RenderGitHubMCPRemoteConfig(yaml *strings.Builder, options GitHubMCPRemoteOptions) { + mcpRendererLog.Printf("Rendering GitHub MCP remote config: read_only=%t, lockdown=%t, toolsets=%s", options.ReadOnly, options.Lockdown, options.Toolsets) + + // Remote mode - use hosted GitHub MCP server + yaml.WriteString(" \"type\": \"http\",\n") + yaml.WriteString(" \"url\": \"https://api.githubcopilot.com/mcp/\",\n") + hasGuardPolicies := hasGitHubMCPGuardPolicies(options.GuardPolicies, options.GuardPoliciesFromStep) + // Use writeJSONStringMapSectionRaw so that pre-escaped shell placeholders such as + // \${GITHUB_PERSONAL_ACCESS_TOKEN} (Copilot passthrough syntax) are written with a + // single backslash in the generated lock file. The compiled config is embedded in an + // unquoted bash heredoc; bash collapses \$ → $, delivering the literal ${VAR} string + // that the MCP gateway then expands from its own environment at runtime. + // Using writeJSONStringMapSection instead would double-escape the backslash to \\${VAR}, + // which bash expands to \ — an invalid JSON escape character that causes + // JSON.parse to fail on every run. + writeJSONStringMapSectionRaw( + yaml, + " ", + "headers", + buildGitHubMCPRemoteHeaders(options.AuthorizationValue, options.ReadOnly, options.Lockdown, options.Toolsets, options.Features), + (options.IncludeToolsField && len(options.AllowedTools) > 0) || options.IncludeEnvSection || hasGuardPolicies, + ) + + // Add tools field if requested (Copilot needs it, Claude doesn't) + // Note: This is added here when IncludeToolsField is true, but in some cases + // the converter script also adds it back (see convert_gateway_config_copilot.cjs). + if options.IncludeToolsField && len(options.AllowedTools) > 0 { + yaml.WriteString(" \"tools\": [\n") + for i, tool := range options.AllowedTools { + yaml.WriteString(" \"") + yaml.WriteString(tool) + yaml.WriteString("\"") + if i < len(options.AllowedTools)-1 { + yaml.WriteString(",") + } + yaml.WriteString("\n") + } + if options.IncludeEnvSection || hasGuardPolicies { + yaml.WriteString(" ],\n") + } else { + yaml.WriteString(" ]\n") + } + } + + // Add env section if needed (Copilot uses this, Claude doesn't) + if options.IncludeEnvSection { + writeJSONStringMapSection( + yaml, + " ", + "env", + buildGitHubMCPEnvVars("${GITHUB_MCP_SERVER_TOKEN}", "${GITHUB_SERVER_URL}", false, false, "", ""), + hasGuardPolicies, + ) + } + + // Add guard-policies if configured or from step + renderGitHubMCPGuardPolicies(yaml, options.GuardPolicies, options.GuardPoliciesFromStep, " ") +} diff --git a/.refactor-work/orig_github.go b/.refactor-work/orig_github.go new file mode 100644 index 00000000000..592020f81d5 --- /dev/null +++ b/.refactor-work/orig_github.go @@ -0,0 +1,328 @@ +package workflow + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/github/gh-aw/pkg/console" + "github.com/github/gh-aw/pkg/constants" + "github.com/github/gh-aw/pkg/sliceutil" +) + +// RenderGitHubMCP generates the GitHub MCP server configuration +// Supports both local (Docker) and remote (hosted) modes +func (r *MCPConfigRendererUnified) RenderGitHubMCP(yaml *strings.Builder, githubTool map[string]any, workflowData *WorkflowData) { + githubType := getGitHubType(githubTool) + readOnly := getGitHubReadOnly() + + // Get explicit lockdown value (only used when lockdown is explicitly configured) + lockdown := getGitHubLockdown(githubTool) + + // Guard policies from step: automatically applied for public repositories when no explicit + // guard policy is configured and no GitHub App token is in use. + // The determine-automatic-lockdown step outputs min_integrity and repos for public repos. + explicitGuardPolicies := getGitHubGuardPolicies(githubTool) + // Integrity reaction fields are only supported in proxy mode (DIFC/CLI proxy), + // not in gateway mode. The MCP gateway cannot identify reaction authors because + // the GitHub MCP server protocol does not expose that information. Warn if the + // user configured reactions with the gateway path. + if isFeatureEnabled(constants.IntegrityReactionsFeatureFlag, workflowData) { + if hasReactionFieldsInToolConfig(githubTool) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage( + "integrity-reactions: endorsement/disapproval reactions are ignored in MCP gateway mode because "+ + "reaction authors cannot be identified from the GitHub MCP server. Reactions are only enforced "+ + "in proxy mode (DIFC proxy / CLI proxy).")) + } + } + shouldUseStepOutputForGuardPolicy := len(explicitGuardPolicies) == 0 && !hasGitHubApp(githubTool) + + toolsets := getGitHubToolsets(githubTool) + features := getGitHubFeatures(githubTool) + + mcpRendererLog.Printf("Rendering GitHub MCP: type=%s, read_only=%t, lockdown=%t (explicit=%t), guard_from_step=%t, toolsets=%v, features=%s, format=%s", + githubType, readOnly, lockdown, hasGitHubLockdownExplicitlySet(githubTool), shouldUseStepOutputForGuardPolicy, toolsets, features, r.options.Format) + + if r.options.Format == "toml" { + mcpRendererLog.Print("GitHub MCP format=toml, dispatching to renderGitHubTOML") + r.renderGitHubTOML(yaml, githubTool, workflowData) + return + } + + yaml.WriteString(" \"github\": {\n") + + // Check if remote mode is enabled (type: remote) + if githubType == GitHubMCPModeRemote { + mcpRendererLog.Printf("GitHub MCP remote mode selected: copilot_fields=%t", r.options.IncludeCopilotFields) + // Determine authorization value based on engine requirements + // Copilot uses MCP passthrough syntax: "Bearer \${GITHUB_PERSONAL_ACCESS_TOKEN}" + // Other engines use shell variable: "Bearer $GITHUB_MCP_SERVER_TOKEN" + authValue := "Bearer $GITHUB_MCP_SERVER_TOKEN" + if r.options.IncludeCopilotFields { + authValue = "Bearer \\${GITHUB_PERSONAL_ACCESS_TOKEN}" + } + + RenderGitHubMCPRemoteConfig(yaml, GitHubMCPRemoteOptions{ + ReadOnly: readOnly, + Lockdown: lockdown, + LockdownFromStep: false, + GuardPoliciesFromStep: shouldUseStepOutputForGuardPolicy, + Toolsets: toolsets, + Features: features, + AuthorizationValue: authValue, + IncludeToolsField: r.options.IncludeCopilotFields, + AllowedTools: getGitHubAllowedTools(githubTool), + IncludeEnvSection: r.options.IncludeCopilotFields, + GuardPolicies: explicitGuardPolicies, + }) + } else { + // Local mode - use Docker-based GitHub MCP server (default) + githubDockerImageVersion := getGitHubDockerImageVersion(githubTool) + customArgs := getGitHubCustomArgs(githubTool) + + mcpRendererLog.Printf("GitHub MCP local docker mode: image_version=%s, custom_args=%d", githubDockerImageVersion, len(customArgs)) + + RenderGitHubMCPDockerConfig(yaml, GitHubMCPDockerOptions{ + ReadOnly: readOnly, + Lockdown: lockdown, + LockdownFromStep: false, + GuardPoliciesFromStep: shouldUseStepOutputForGuardPolicy, + Toolsets: toolsets, + Features: features, + DockerImageVersion: githubDockerImageVersion, + CustomArgs: customArgs, + IncludeTypeField: r.options.IncludeCopilotFields, + AllowedTools: getGitHubAllowedTools(githubTool), + EffectiveToken: "", // Token passed via env + GuardPolicies: explicitGuardPolicies, + }) + } + + if r.options.IsLast { + yaml.WriteString(" }\n") + } else { + yaml.WriteString(" },\n") + } +} + +// renderGitHubTOML generates GitHub MCP configuration in TOML format (for Codex engine) +func (r *MCPConfigRendererUnified) renderGitHubTOML(yaml *strings.Builder, githubTool map[string]any, workflowData *WorkflowData) { + githubType := getGitHubType(githubTool) + readOnly := getGitHubReadOnly() + lockdown := getGitHubLockdown(githubTool) + toolsets := getGitHubToolsets(githubTool) + features := getGitHubFeatures(githubTool) + + mcpRendererLog.Printf("Rendering GitHub MCP TOML: type=%s, read_only=%t, lockdown=%t, toolsets=%s, features=%s", githubType, readOnly, lockdown, toolsets, features) + + yaml.WriteString(" \n") + yaml.WriteString(" [mcp_servers.github]\n") + + // Add user_agent field defaulting to workflow identifier + userAgent := "github-agentic-workflow" + if workflowData != nil { + // Check if user_agent is configured in engine config first + if workflowData.EngineConfig != nil && workflowData.EngineConfig.UserAgent != "" { + userAgent = workflowData.EngineConfig.UserAgent + } else if workflowData.Name != "" { + // Fall back to sanitizing the workflow name as an artifact/user-agent identifier + userAgent = SanitizeArtifactIdentifier(workflowData.Name) + } + } + yaml.WriteString(" user_agent = \"" + userAgent + "\"\n") + + // Use tools.startup-timeout if specified, otherwise default to DefaultMCPStartupTimeout + // For GitHub Actions expressions, fall back to default (TOML format doesn't support expressions) + startupTimeout := int(constants.DefaultMCPStartupTimeout / time.Second) + if workflowData != nil && workflowData.ToolsStartupTimeout != "" { + if n := templatableIntValue(&workflowData.ToolsStartupTimeout); n > 0 { + startupTimeout = n + } + } + fmt.Fprintf(yaml, " startup_timeout_sec = %d\n", startupTimeout) + + // Use tools.timeout if specified, otherwise default to DefaultToolTimeout + // For GitHub Actions expressions, fall back to default (TOML format doesn't support expressions) + toolTimeout := int(constants.DefaultToolTimeout / time.Second) + if workflowData != nil && workflowData.ToolsTimeout != "" { + if n := templatableIntValue(&workflowData.ToolsTimeout); n > 0 { + toolTimeout = n + } + } + fmt.Fprintf(yaml, " tool_timeout_sec = %d\n", toolTimeout) + + // Check if remote mode is enabled + if githubType == GitHubMCPModeRemote { + mcpRendererLog.Printf("GitHub MCP TOML remote mode: readonly_endpoint=%t", readOnly) + // Remote mode - use hosted GitHub MCP server with streamable HTTP + // Use readonly endpoint if read-only mode is enabled + if readOnly { + yaml.WriteString(" url = \"https://api.githubcopilot.com/mcp-readonly/\"\n") + } else { + yaml.WriteString(" url = \"https://api.githubcopilot.com/mcp/\"\n") + } + + // Use bearer_token_env_var for authentication + yaml.WriteString(" bearer_token_env_var = \"GH_AW_GITHUB_TOKEN\"\n") + } else { + // Local mode - use Docker-based GitHub MCP server with MCP Gateway spec format + githubDockerImageVersion := getGitHubDockerImageVersion(githubTool) + customArgs := getGitHubCustomArgs(githubTool) + + // MCP Gateway spec fields for containerized stdio servers + yaml.WriteString(" container = \"ghcr.io/github/github-mcp-server:" + githubDockerImageVersion + "\"\n") + + // Append custom args if present (these are Docker runtime args, go before container image) + if len(customArgs) > 0 { + yaml.WriteString(" args = [\n") + for _, arg := range customArgs { + yaml.WriteString(" " + strconv.Quote(arg) + ",\n") + } + yaml.WriteString(" ]\n") + } + + // Build environment variables + envVars := buildGitHubMCPEnvVars( + "$GH_AW_GITHUB_TOKEN", + "$GITHUB_SERVER_URL", + readOnly, + lockdown, + toolsets, + features, + ) + + // Write environment variables in sorted order for deterministic output + envKeys := sliceutil.SortedKeys(envVars) + + writeTOMLInlineStringMapSection(yaml, " ", "env", envVars) + + // Use env_vars array to reference environment variables + yaml.WriteString(" env_vars = [") + for i, key := range envKeys { + if i > 0 { + yaml.WriteString(", ") + } + fmt.Fprintf(yaml, "\"%s\"", key) + } + yaml.WriteString("]\n") + } +} + +// RenderGitHubMCPDockerConfig renders the GitHub MCP server configuration for Docker (local mode). +// Per MCP Gateway Specification v1.0.0 section 3.2.1, stdio-based MCP servers MUST be containerized. +// Uses MCP Gateway spec format: container, entrypointArgs, and env fields. +// +// Parameters: +// - yaml: The string builder for YAML output +// - options: GitHub MCP Docker rendering options +func RenderGitHubMCPDockerConfig(yaml *strings.Builder, options GitHubMCPDockerOptions) { + mcpRendererLog.Printf("Rendering GitHub MCP Docker config: image=%s, read_only=%t, lockdown=%t", options.DockerImageVersion, options.ReadOnly, options.Lockdown) + + // Add type field if needed (Copilot requires this, Claude doesn't) + // Per MCP Gateway Specification v1.0.0 section 4.1.2, use "stdio" for containerized servers + if options.IncludeTypeField { + yaml.WriteString(" \"type\": \"stdio\",\n") + } + + // MCP Gateway spec fields for containerized stdio servers + yaml.WriteString(" \"container\": \"ghcr.io/github/github-mcp-server:" + options.DockerImageVersion + "\",\n") + + // Append custom args if present (these are Docker runtime args, go before container image) + if len(options.CustomArgs) > 0 { + yaml.WriteString(" \"args\": [\n") + for _, arg := range options.CustomArgs { + quotedArg, _ := json.Marshal(arg) //nolint:jsonmarshalignoredeerror // marshaling a string cannot fail + yaml.WriteString(" " + string(quotedArg) + ",\n") + } + yaml.WriteString(" ],\n") + } + + // Note: tools field is NOT included here - the converter script adds it back + // for Copilot (see convert_gateway_config_copilot.cjs). This keeps the gateway + // config compatible with the schema which doesn't have the tools field. + + // GitHub token (always required) + tokenValue := "$GITHUB_MCP_SERVER_TOKEN" + hostValue := "$GITHUB_SERVER_URL" + if options.IncludeTypeField { + // Copilot engine: keep shell expansion so gateway input remains valid JSON. + tokenValue = "${GITHUB_MCP_SERVER_TOKEN}" + // GitHub host for enterprise deployments (format: https://hostname, e.g. https://myorg.ghe.com). + // GITHUB_SERVER_URL is set by GitHub Actions as a full URL (https://hostname, no trailing slash), + // which matches the format expected by github-mcp-server for GITHUB_HOST. + hostValue = "${GITHUB_SERVER_URL}" + } + + envVars := buildGitHubMCPEnvVars(tokenValue, hostValue, options.ReadOnly, options.Lockdown, options.Toolsets, options.Features) + hasGuardPolicies := hasGitHubMCPGuardPolicies(options.GuardPolicies, options.GuardPoliciesFromStep) + writeJSONStringMapSection(yaml, " ", "env", envVars, hasGuardPolicies) + renderGitHubMCPGuardPolicies(yaml, options.GuardPolicies, options.GuardPoliciesFromStep, " ") +} + +// RenderGitHubMCPRemoteConfig renders the GitHub MCP server configuration for remote (hosted) mode. +// This shared function extracts the duplicate pattern from Claude and Copilot engines. +// +// Parameters: +// - yaml: The string builder for YAML output +// - options: GitHub MCP remote rendering options +func RenderGitHubMCPRemoteConfig(yaml *strings.Builder, options GitHubMCPRemoteOptions) { + mcpRendererLog.Printf("Rendering GitHub MCP remote config: read_only=%t, lockdown=%t, toolsets=%s", options.ReadOnly, options.Lockdown, options.Toolsets) + + // Remote mode - use hosted GitHub MCP server + yaml.WriteString(" \"type\": \"http\",\n") + yaml.WriteString(" \"url\": \"https://api.githubcopilot.com/mcp/\",\n") + hasGuardPolicies := hasGitHubMCPGuardPolicies(options.GuardPolicies, options.GuardPoliciesFromStep) + // Use writeJSONStringMapSectionRaw so that pre-escaped shell placeholders such as + // \${GITHUB_PERSONAL_ACCESS_TOKEN} (Copilot passthrough syntax) are written with a + // single backslash in the generated lock file. The compiled config is embedded in an + // unquoted bash heredoc; bash collapses \$ → $, delivering the literal ${VAR} string + // that the MCP gateway then expands from its own environment at runtime. + // Using writeJSONStringMapSection instead would double-escape the backslash to \\${VAR}, + // which bash expands to \ — an invalid JSON escape character that causes + // JSON.parse to fail on every run. + writeJSONStringMapSectionRaw( + yaml, + " ", + "headers", + buildGitHubMCPRemoteHeaders(options.AuthorizationValue, options.ReadOnly, options.Lockdown, options.Toolsets, options.Features), + (options.IncludeToolsField && len(options.AllowedTools) > 0) || options.IncludeEnvSection || hasGuardPolicies, + ) + + // Add tools field if requested (Copilot needs it, Claude doesn't) + // Note: This is added here when IncludeToolsField is true, but in some cases + // the converter script also adds it back (see convert_gateway_config_copilot.cjs). + if options.IncludeToolsField && len(options.AllowedTools) > 0 { + yaml.WriteString(" \"tools\": [\n") + for i, tool := range options.AllowedTools { + yaml.WriteString(" \"") + yaml.WriteString(tool) + yaml.WriteString("\"") + if i < len(options.AllowedTools)-1 { + yaml.WriteString(",") + } + yaml.WriteString("\n") + } + if options.IncludeEnvSection || hasGuardPolicies { + yaml.WriteString(" ],\n") + } else { + yaml.WriteString(" ]\n") + } + } + + // Add env section if needed (Copilot uses this, Claude doesn't) + if options.IncludeEnvSection { + writeJSONStringMapSection( + yaml, + " ", + "env", + buildGitHubMCPEnvVars("${GITHUB_MCP_SERVER_TOKEN}", "${GITHUB_SERVER_URL}", false, false, "", ""), + hasGuardPolicies, + ) + } + + // Add guard-policies if configured or from step + renderGitHubMCPGuardPolicies(yaml, options.GuardPolicies, options.GuardPoliciesFromStep, " ") +} diff --git a/.scratch_refactor/INSTRUCTIONS.md b/.scratch_refactor/INSTRUCTIONS.md new file mode 100644 index 00000000000..cdec9fc07df --- /dev/null +++ b/.scratch_refactor/INSTRUCTIONS.md @@ -0,0 +1,42 @@ +# Refactoring task: reduce function-length lint findings in pkg/cli + +You are refactoring Go functions in `pkg/cli/` (package `cli`) that exceed 60 lines. +Goal: reduce the `largefunc` lint findings by extracting focused helper functions. + +## Hard rules +1. **Pure refactoring — preserve ALL existing behavior.** No logic, output, or control-flow changes. +2. Extract logical sub-sections of a long function into new helper functions so the + original function drops to **≤ 60 lines** (counting the `func` signature line through the + closing `}` inclusive). Each new helper should also be ≤ 60 lines. +3. **Keep helpers in the SAME file** as the function you extracted them from. +4. **Naming (critical — avoid collisions):** every NEW helper function MUST be named by + using the original function's name as a prefix, then a descriptive suffix. + Example: extracting from `buildAuditData` → `buildAuditDataMetrics`, `buildAuditDataFirewall`. + For extracted anonymous/`func literal` findings (e.g. cobra `RunE:` closures), create a + named package function derived from the command/context, e.g. `runLogsCommand`, + and reference it from the closure: `RunE: func(cmd *cobra.Command, args []string) error { return runLogsCommand(cmd, args, &opts) }`. + Do NOT reuse a generic name that another file might also use. +5. **Do NOT modify `_test.go` files.** +6. Do not change exported function signatures or public API. Helpers should be unexported. +7. Preserve comments; move them with the code they describe. +8. Keep imports correct. Do not add new external dependencies. + +## Technique +- Read the full function first. Identify natural blocks (often separated by blank lines or + comments, or handling distinct phases: validation, data gathering, API calls, rendering). +- Extract each block into a helper that takes exactly the inputs it needs and returns exactly + the outputs the caller needs. Pass pointers to structs when the block mutates shared state. +- If a block references many local variables, consider passing a small params struct OR + passing the needed values; prefer passing the existing struct/receiver when available. +- Be careful with `return`, `continue`, `break`, named return values, `defer`, and closures + capturing loop variables — extracting these requires returning values/errors to preserve flow. +- For methods (receiver funcs), helpers can be methods on the same receiver. + +## After editing each file +- Run `gofmt -w` on the file to ensure it is syntactically valid and formatted. + If gofmt reports a syntax error, fix it. + +## Do NOT +- Do NOT run `go build ./pkg/cli/...` or the package-wide linter — other agents are editing + the same package concurrently and it will show spurious errors. Only use `gofmt` per-file. +- Do NOT touch files outside your assigned list. diff --git a/.scratch_refactor/PARAM_INSTRUCTIONS.md b/.scratch_refactor/PARAM_INSTRUCTIONS.md new file mode 100644 index 00000000000..c37cffe251d --- /dev/null +++ b/.scratch_refactor/PARAM_INSTRUCTIONS.md @@ -0,0 +1,34 @@ +# Fix "too many parameters (limit 8)" findings in pkg/cli + +Some helper functions were recently extracted with more than 8 parameters, which trips +a linter. Your job: refactor each listed function so it has **≤ 8 parameters**, WITHOUT +changing behavior. + +## Preferred technique: options/params struct +For each offending function `fooBar(a, b, c, ... many ...)`: +1. Define a struct in the SAME file, named `fooBarParams`, with fields for the parameters + that logically group together (or all of them). Use exported-style Go field names + (e.g. field `Verbose bool`), but the struct itself stays unexported (`fooBarParams`). +2. Change the function signature to accept the struct, e.g. + `func fooBar(p fooBarParams) ... {` and update the body to reference `p.FieldName`. + Keep a small number of separate params only if it reads better, but the TOTAL parameter + count MUST be ≤ 8. Simplest is to move ALL params into one struct (1 param total). +3. Update EVERY call site (usually just one — the function that this helper was extracted + from). Build the struct literal with the same values in the same order/meaning. + +## Alternative technique +If several params are already fields of an existing struct that the caller has in scope, +you may pass that struct/pointer instead of individual fields. + +## Hard rules +- Preserve behavior EXACTLY. Same values passed, same order of operations. +- Do NOT modify `_test.go` files. +- Keep the new struct and function in the same file. +- Struct field types must match the original parameter types exactly (watch pointers vs values). +- Run `gofmt -w ` on each edited file. Fix any syntax errors gofmt reports. +- Do NOT run the package-wide `go build`/linter (other agents edit the same package). +- Only edit the files assigned to you. + +## Verifying parameter count +After refactor, the function signature should list ≤ 8 comma-separated parameters +(a single struct param counts as 1). Double-check nested/variadic params. diff --git a/.scratch_refactor/batch_01 b/.scratch_refactor/batch_01 new file mode 100644 index 00000000000..8fd634e4868 --- /dev/null +++ b/.scratch_refactor/batch_01 @@ -0,0 +1,19 @@ +pkg/cli/access_log.go +pkg/cli/actions_build_command.go +pkg/cli/add_interactive_engine.go +pkg/cli/add_interactive_git.go +pkg/cli/add_interactive_orchestrator.go +pkg/cli/add_interactive_schedule.go +pkg/cli/add_interactive_workflow.go +pkg/cli/add_package_manifest.go +pkg/cli/add_wizard_command.go +pkg/cli/add_workflow_pr.go +pkg/cli/add_workflow_resolution.go +pkg/cli/audit_agentic_analysis.go +pkg/cli/audit_comparison.go +pkg/cli/audit_diff.go +pkg/cli/audit_diff_command.go +pkg/cli/audit_diff_render.go +pkg/cli/audit_expanded.go +pkg/cli/audit_report.go +pkg/cli/audit_report_analysis.go diff --git a/.scratch_refactor/batch_02 b/.scratch_refactor/batch_02 new file mode 100644 index 00000000000..1be7af26513 --- /dev/null +++ b/.scratch_refactor/batch_02 @@ -0,0 +1,14 @@ +pkg/cli/audit_report_render.go +pkg/cli/bootstrap_profile_manifest.go +pkg/cli/bootstrap_profile_runner.go +pkg/cli/bootstrap_shared.go +pkg/cli/codemod_agent_session.go +pkg/cli/codemod_assign_to_agent.go +pkg/cli/codemod_bash_single_quoted_args.go +pkg/cli/codemod_checkout_persist_credentials_false.go +pkg/cli/codemod_cli_proxy_mode.go +pkg/cli/codemod_dependabot_permissions.go +pkg/cli/codemod_difc_proxy.go +pkg/cli/codemod_discussion_flag.go +pkg/cli/codemod_discussion_trigger_categories.go +pkg/cli/codemod_effective_tokens_to_ai_credits.go diff --git a/.scratch_refactor/batch_03 b/.scratch_refactor/batch_03 new file mode 100644 index 00000000000..076731dc82e --- /dev/null +++ b/.scratch_refactor/batch_03 @@ -0,0 +1,14 @@ +pkg/cli/codemod_engine_env_secrets.go +pkg/cli/codemod_engine_steps.go +pkg/cli/codemod_engine_to_top_level_helpers.go +pkg/cli/codemod_factory.go +pkg/cli/codemod_github_app.go +pkg/cli/codemod_github_repos.go +pkg/cli/codemod_mcp_network.go +pkg/cli/codemod_mentions_allow_team_members.go +pkg/cli/codemod_messages_effective_tokens_suffix_to_ai_credits_suffix.go +pkg/cli/codemod_network_firewall.go +pkg/cli/codemod_permissions_read.go +pkg/cli/codemod_permissions_write.go +pkg/cli/codemod_playwright_domains.go +pkg/cli/codemod_run_install_scripts.go diff --git a/.scratch_refactor/batch_04 b/.scratch_refactor/batch_04 new file mode 100644 index 00000000000..b7daec8f3ab --- /dev/null +++ b/.scratch_refactor/batch_04 @@ -0,0 +1,15 @@ +pkg/cli/codemod_safe_output_add_reviewer_allowlists.go +pkg/cli/codemod_safe_output_merge_pr_constraints.go +pkg/cli/codemod_safe_output_require_title_prefix.go +pkg/cli/codemod_schedule.go +pkg/cli/codemod_serena_import.go +pkg/cli/codemod_slash_command.go +pkg/cli/codemod_steps_run_secrets_env.go +pkg/cli/codemod_upload_assets.go +pkg/cli/codemod_user_rate_limit.go +pkg/cli/codemod_workflow_dispatch_required.go +pkg/cli/codemod_workflow_run_branches.go +pkg/cli/commands.go +pkg/cli/compile_compiler_setup.go +pkg/cli/compile_file_operations.go +pkg/cli/compile_orchestrator.go diff --git a/.scratch_refactor/batch_05 b/.scratch_refactor/batch_05 new file mode 100644 index 00000000000..7e851e7426e --- /dev/null +++ b/.scratch_refactor/batch_05 @@ -0,0 +1,19 @@ +pkg/cli/compile_pipeline.go +pkg/cli/compile_repository_manifest.go +pkg/cli/compile_schedule_calendar.go +pkg/cli/compile_stats.go +pkg/cli/compile_validation.go +pkg/cli/compile_watch.go +pkg/cli/compile_workflow_processor.go +pkg/cli/completion_command.go +pkg/cli/copilot_agent_logs.go +pkg/cli/copilot_events_jsonl.go +pkg/cli/copilot_setup.go +pkg/cli/dependency_graph.go +pkg/cli/deps_report.go +pkg/cli/deps_security.go +pkg/cli/devcontainer.go +pkg/cli/dispatch.go +pkg/cli/docker_images.go +pkg/cli/download_workflow.go +pkg/cli/drain3_integration.go diff --git a/.scratch_refactor/batch_06 b/.scratch_refactor/batch_06 new file mode 100644 index 00000000000..0e3e3bff2f6 --- /dev/null +++ b/.scratch_refactor/batch_06 @@ -0,0 +1,20 @@ +pkg/cli/drain3_train.go +pkg/cli/enable.go +pkg/cli/engine_secrets.go +pkg/cli/experiments_analyze_statistics.go +pkg/cli/experiments_command.go +pkg/cli/fetch.go +pkg/cli/firewall_log.go +pkg/cli/firewall_policy.go +pkg/cli/fix_codemods.go +pkg/cli/fix_command.go +pkg/cli/forecast.go +pkg/cli/forecast_command.go +pkg/cli/forecast_compute.go +pkg/cli/forecast_montecarlo.go +pkg/cli/forecast_render.go +pkg/cli/frontmatter_editor.go +pkg/cli/gateway_logs_mcp.go +pkg/cli/gateway_logs_parsing.go +pkg/cli/gateway_logs_render.go +pkg/cli/gateway_logs_rpc.go diff --git a/.scratch_refactor/batch_07 b/.scratch_refactor/batch_07 new file mode 100644 index 00000000000..0c6eaff7e53 --- /dev/null +++ b/.scratch_refactor/batch_07 @@ -0,0 +1,21 @@ +pkg/cli/gateway_logs_timeline_render.go +pkg/cli/generate_action_metadata_command.go +pkg/cli/git.go +pkg/cli/health_command.go +pkg/cli/health_metrics.go +pkg/cli/import_url_fetcher.go +pkg/cli/importpath.go +pkg/cli/imports.go +pkg/cli/includes.go +pkg/cli/init.go +pkg/cli/init_command.go +pkg/cli/interactive.go +pkg/cli/jsonworkflow_to_markdown.go +pkg/cli/list_workflows_command.go +pkg/cli/log_aggregation.go +pkg/cli/logs_cache.go +pkg/cli/logs_command.go +pkg/cli/logs_download.go +pkg/cli/logs_episode.go +pkg/cli/logs_format_compact.go +pkg/cli/logs_format_tsv.go diff --git a/.scratch_refactor/batch_08 b/.scratch_refactor/batch_08 new file mode 100644 index 00000000000..96b53e49dd9 --- /dev/null +++ b/.scratch_refactor/batch_08 @@ -0,0 +1,18 @@ +pkg/cli/logs_github_api.go +pkg/cli/logs_github_rate_limit_usage.go +pkg/cli/logs_metrics.go +pkg/cli/logs_orchestrator_download.go +pkg/cli/logs_orchestrator_filters.go +pkg/cli/logs_orchestrator_render.go +pkg/cli/logs_orchestrator_stdin.go +pkg/cli/logs_parsing_firewall.go +pkg/cli/logs_parsing_javascript.go +pkg/cli/logs_report.go +pkg/cli/logs_report_mcp.go +pkg/cli/logs_report_tools.go +pkg/cli/logs_safe_output_chains.go +pkg/cli/logs_usage_activity.go +pkg/cli/logs_utils.go +pkg/cli/mcp_add.go +pkg/cli/mcp_inspect.go +pkg/cli/mcp_inspect_inspector.go diff --git a/.scratch_refactor/batch_09 b/.scratch_refactor/batch_09 new file mode 100644 index 00000000000..093daf9c631 --- /dev/null +++ b/.scratch_refactor/batch_09 @@ -0,0 +1,20 @@ +pkg/cli/mcp_inspect_mcp.go +pkg/cli/mcp_inspect_mcp_scripts_files.go +pkg/cli/mcp_inspect_mcp_scripts_server.go +pkg/cli/mcp_list.go +pkg/cli/mcp_list_tools.go +pkg/cli/mcp_logs_guardrail.go +pkg/cli/mcp_permissions.go +pkg/cli/mcp_registry.go +pkg/cli/mcp_registry_list.go +pkg/cli/mcp_secrets.go +pkg/cli/mcp_server.go +pkg/cli/mcp_server_command.go +pkg/cli/mcp_tool_table.go +pkg/cli/mcp_tools_management.go +pkg/cli/mcp_tools_privileged.go +pkg/cli/mcp_tools_readonly.go +pkg/cli/mcp_validation.go +pkg/cli/model_costs.go +pkg/cli/observability_insights.go +pkg/cli/org_runner.go diff --git a/.scratch_refactor/batch_10 b/.scratch_refactor/batch_10 new file mode 100644 index 00000000000..8be69af1f9b --- /dev/null +++ b/.scratch_refactor/batch_10 @@ -0,0 +1,19 @@ +pkg/cli/outcome_domain_breakdown.go +pkg/cli/outcome_eval.go +pkg/cli/outcome_eval_agent.go +pkg/cli/outcome_eval_comment.go +pkg/cli/outcome_eval_enrich.go +pkg/cli/outcome_eval_issue.go +pkg/cli/outcome_eval_label.go +pkg/cli/outcome_eval_pr.go +pkg/cli/outcome_eval_review.go +pkg/cli/outcome_eval_update.go +pkg/cli/outcome_eval_workflow.go +pkg/cli/outcome_evaluation.go +pkg/cli/outcomes_command.go +pkg/cli/outcomes_history.go +pkg/cli/packages.go +pkg/cli/poutine.go +pkg/cli/pr_automerge.go +pkg/cli/pr_command.go +pkg/cli/preconditions.go diff --git a/.scratch_refactor/batch_11 b/.scratch_refactor/batch_11 new file mode 100644 index 00000000000..32d1b9316c7 --- /dev/null +++ b/.scratch_refactor/batch_11 @@ -0,0 +1,22 @@ +pkg/cli/project_command.go +pkg/cli/redacted_domains.go +pkg/cli/remove_command.go +pkg/cli/resources.go +pkg/cli/retry.go +pkg/cli/run_interactive.go +pkg/cli/run_push.go +pkg/cli/run_workflow_execution.go +pkg/cli/run_workflow_tracking.go +pkg/cli/run_workflow_validation.go +pkg/cli/runner_guard.go +pkg/cli/secret_set_command.go +pkg/cli/setup_repository.go +pkg/cli/shell_completion.go +pkg/cli/signal_aware_poll.go +pkg/cli/spec.go +pkg/cli/status_command.go +pkg/cli/token_usage.go +pkg/cli/tokens_bootstrap.go +pkg/cli/tool_graph.go +pkg/cli/trial_command.go +pkg/cli/trial_confirmation.go diff --git a/.scratch_refactor/batch_12 b/.scratch_refactor/batch_12 new file mode 100644 index 00000000000..d0c24e55183 --- /dev/null +++ b/.scratch_refactor/batch_12 @@ -0,0 +1,20 @@ +pkg/cli/trial_helpers.go +pkg/cli/trial_repository.go +pkg/cli/trial_runner.go +pkg/cli/trial_support.go +pkg/cli/update_actions.go +pkg/cli/update_command.go +pkg/cli/update_container_pins.go +pkg/cli/update_extension_check.go +pkg/cli/update_manifest.go +pkg/cli/update_merge.go +pkg/cli/update_org.go +pkg/cli/update_redirects.go +pkg/cli/update_workflows.go +pkg/cli/upgrade_command.go +pkg/cli/upgrade_org.go +pkg/cli/validate_command.go +pkg/cli/view_command.go +pkg/cli/workflows.go +pkg/cli/yaml_frontmatter_utils.go +pkg/cli/zizmor.go diff --git a/.scratch_worktree b/.scratch_worktree new file mode 160000 index 00000000000..8c143edc211 --- /dev/null +++ b/.scratch_worktree @@ -0,0 +1 @@ +Subproject commit 8c143edc211063b6364ca19742e25e72efd7f7a8 diff --git a/pkg/cli/access_log.go b/pkg/cli/access_log.go index 2e66f38d9d0..a0160af541e 100644 --- a/pkg/cli/access_log.go +++ b/pkg/cli/access_log.go @@ -103,54 +103,7 @@ func parseSquidAccessLog(logPath string, verbose bool) (*DomainAnalysis, error) scanner := bufio.NewScanner(file) for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - entry, err := parseSquidLogLine(line) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse log line: %v", err))) - } - continue - } - - analysis.TotalRequests++ - - // Extract domain from URL - domain := stringutil.ExtractDomainFromURL(entry.URL) - if domain == "" { - continue - } - - // Determine if request was allowed or blocked based on status code - // Squid typically returns: - // - 200, 206, 304: Allowed/successful - // - 403: Forbidden (blocked by ACL) - // - 407: Proxy authentication required - // - 502, 503: Connection/upstream errors - statusCode := entry.Status - isAllowed := statusCode == "TCP_HIT/200" || statusCode == "TCP_MISS/200" || - statusCode == "TCP_REFRESH_MODIFIED/200" || statusCode == "TCP_IMS_HIT/304" || - strings.Contains(statusCode, "/200") || strings.Contains(statusCode, "/206") || - strings.Contains(statusCode, "/304") - - if isAllowed { - analysis.AllowedRequests++ - if !setutil.Contains(allowedDomainsSet, domain) { - allowedDomainsSet[domain] = struct { - }{} - analysis.AllowedDomains = append(analysis.AllowedDomains, domain) - } - } else { - analysis.BlockedRequests++ - if !setutil.Contains(blockedDomainsSet, domain) { - blockedDomainsSet[domain] = struct { - }{} - analysis.BlockedDomains = append(analysis.BlockedDomains, domain) - } - } + parseSquidAccessLogLine(scanner.Text(), verbose, analysis, allowedDomainsSet, blockedDomainsSet) } if err := scanner.Err(); err != nil { @@ -167,6 +120,58 @@ func parseSquidAccessLog(logPath string, verbose bool) (*DomainAnalysis, error) return analysis, nil } +func parseSquidAccessLogLine(line string, verbose bool, analysis *DomainAnalysis, allowedDomainsSet, blockedDomainsSet map[string]struct{}) { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + return + } + + entry, err := parseSquidLogLine(line) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse log line: %v", err))) + } + return + } + + analysis.TotalRequests++ + + // Extract domain from URL + domain := stringutil.ExtractDomainFromURL(entry.URL) + if domain == "" { + return + } + + if parseSquidAccessLogEntryAllowed(entry) { + analysis.AllowedRequests++ + if !setutil.Contains(allowedDomainsSet, domain) { + allowedDomainsSet[domain] = struct{}{} + analysis.AllowedDomains = append(analysis.AllowedDomains, domain) + } + return + } + + analysis.BlockedRequests++ + if !setutil.Contains(blockedDomainsSet, domain) { + blockedDomainsSet[domain] = struct{}{} + analysis.BlockedDomains = append(analysis.BlockedDomains, domain) + } +} + +func parseSquidAccessLogEntryAllowed(entry *AccessLogEntry) bool { + // Determine if request was allowed or blocked based on status code + // Squid typically returns: + // - 200, 206, 304: Allowed/successful + // - 403: Forbidden (blocked by ACL) + // - 407: Proxy authentication required + // - 502, 503: Connection/upstream errors + statusCode := entry.Status + return statusCode == "TCP_HIT/200" || statusCode == "TCP_MISS/200" || + statusCode == "TCP_REFRESH_MODIFIED/200" || statusCode == "TCP_IMS_HIT/304" || + strings.Contains(statusCode, "/200") || strings.Contains(statusCode, "/206") || + strings.Contains(statusCode, "/304") +} + // parseSquidLogLine parses a single squid access log line // Squid log format: timestamp duration client status size method url user hierarchy type func parseSquidLogLine(line string) (*AccessLogEntry, error) { diff --git a/pkg/cli/actions_build_command.go b/pkg/cli/actions_build_command.go index 4c8f193eec7..4ddd0023e07 100644 --- a/pkg/cli/actions_build_command.go +++ b/pkg/cli/actions_build_command.go @@ -228,6 +228,10 @@ func buildAction(actionsDir, actionName string) error { return nil } + return buildActionJavaScript(actionPath, actionName) +} + +func buildActionJavaScript(actionPath, actionName string) error { srcPath := filepath.Join(actionPath, "src", "index.js") outputPath := filepath.Join(actionPath, "index.js") @@ -246,6 +250,24 @@ func buildAction(actionsDir, actionName string) error { dependencies := getActionDependencies(actionName) fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf(" ✓ Found %d dependencies", len(dependencies)))) + files := buildActionDependencyFiles(dependencies) + outputContent, err := buildActionOutputContent(sourceContent, files) + if err != nil { + return err + } + + // Write output file with restrictive permissions (0600 for security) + if err := os.WriteFile(outputPath, outputContent, constants.FilePermSensitive); err != nil { + return fmt.Errorf("failed to write output file: %w", err) + } + + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" ✓ Built "+outputPath)) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf(" ✓ Embedded %d files", len(files)))) + + return nil +} + +func buildActionDependencyFiles(dependencies []string) map[string]string { // Get all JavaScript sources sources := workflow.GetJavaScriptSources() @@ -259,11 +281,14 @@ func buildAction(actionsDir, actionName string) error { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(" ⚠ Warning: Could not find "+dep)) } } + return files +} +func buildActionOutputContent(sourceContent []byte, files map[string]string) ([]byte, error) { // Generate FILES object with embedded content filesJSON, err := json.MarshalIndent(files, "", " ") if err != nil { - return fmt.Errorf("failed to marshal files: %w", err) + return nil, fmt.Errorf("failed to marshal files: %w", err) } // Indent the JSON for proper embedding @@ -273,16 +298,7 @@ func buildAction(actionsDir, actionName string) error { // Replace the FILES placeholder in source // Match: const FILES = { ... }; outputContent := filesConstPattern.ReplaceAllString(string(sourceContent), fmt.Sprintf("const FILES = %s;", strings.TrimSpace(indentedJSON))) - - // Write output file with restrictive permissions (0600 for security) - if err := os.WriteFile(outputPath, []byte(outputContent), constants.FilePermSensitive); err != nil { - return fmt.Errorf("failed to write output file: %w", err) - } - - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" ✓ Built "+outputPath)) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf(" ✓ Embedded %d files", len(files)))) - - return nil + return []byte(outputContent), nil } // isCompositeAction checks if an action uses the 'composite' runtime diff --git a/pkg/cli/add_interactive_engine.go b/pkg/cli/add_interactive_engine.go index 194f5873555..0af9dc7c884 100644 --- a/pkg/cli/add_interactive_engine.go +++ b/pkg/cli/add_interactive_engine.go @@ -24,72 +24,98 @@ func (c *AddInteractiveConfig) selectAIEngineAndKey() error { return err } - // Determine default engine based on existing secrets, workflow preference, then environment - // Priority order: flag override > existing secrets > workflow frontmatter > environment > default - defaultEngine := string(constants.DefaultEngine) - workflowSpecifiedEngine := "" + workflowSpecifiedEngine := c.selectAIEngineAndKeyWorkflowSpecifiedEngine() + defaultEngine := c.selectAIEngineAndKeyDefaultEngine(workflowSpecifiedEngine) + + // If engine is already overridden, skip selection + if c.EngineOverride != "" { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Using coding agent: "+c.EngineOverride)) + return c.configureEngineAPISecret(c.EngineOverride) + } + + // Inform user if workflow specifies an engine + if workflowSpecifiedEngine != "" { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Workflow specifies engine: "+workflowSpecifiedEngine)) + } + + engineOptions := c.selectAIEngineAndKeyOptions(workflowSpecifiedEngine) + + var selectedEngine string + + if err := c.selectAIEngineAndKeyPrompt(engineOptions, defaultEngine, &selectedEngine); err != nil { + return fmt.Errorf("failed to select coding agent: %w", err) + } + c.EngineOverride = selectedEngine + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Selected engine: "+selectedEngine)) + + return c.configureEngineAPISecret(selectedEngine) +} + +func (c *AddInteractiveConfig) selectAIEngineAndKeyWorkflowSpecifiedEngine() string { // Check if workflow specifies a preferred engine in frontmatter if c.resolvedWorkflows != nil && len(c.resolvedWorkflows.Workflows) > 0 { for _, wf := range c.resolvedWorkflows.Workflows { if wf.Engine != "" { - workflowSpecifiedEngine = wf.Engine addInteractiveLog.Printf("Workflow specifies engine in frontmatter: %s", wf.Engine) - break + return wf.Engine } } } + return "" +} - // If engine is explicitly overridden via flag, use that +func (c *AddInteractiveConfig) selectAIEngineAndKeyDefaultEngine(workflowSpecifiedEngine string) string { + // Determine default engine based on existing secrets, workflow preference, then environment + // Priority order: flag override > existing secrets > workflow frontmatter > environment > default if c.EngineOverride != "" { - defaultEngine = c.EngineOverride - } else { - // Priority 1: Check existing repository secrets using EngineOptions - // This takes precedence over workflow preference since users should use what's already available - for _, opt := range constants.EngineOptions { - if setutil.Contains(c.existingSecrets, opt.SecretName) { - defaultEngine = opt.Value - addInteractiveLog.Printf("Found existing secret %s, recommending engine: %s", opt.SecretName, opt.Value) - break - } - } - - // Priority 2: If no existing secret found, use workflow frontmatter preference - if defaultEngine == string(constants.DefaultEngine) && workflowSpecifiedEngine != "" { - defaultEngine = workflowSpecifiedEngine - } + return c.EngineOverride + } - // Priority 3: Check environment variables if no existing secret or workflow preference found - if defaultEngine == string(constants.DefaultEngine) && workflowSpecifiedEngine == "" { - for _, opt := range constants.EngineOptions { - envVar := opt.SecretName - if opt.EnvVarName != "" { - envVar = opt.EnvVarName - } - if lookupEnv(envVar) != "" { - defaultEngine = opt.Value - addInteractiveLog.Printf("Found env var %s, recommending engine: %s", envVar, opt.Value) - break - } - } - } + if engine, ok := c.selectAIEngineAndKeyDefaultEngineFromSecrets(); ok { + return engine } + if workflowSpecifiedEngine != "" { + return workflowSpecifiedEngine + } + if engine, ok := c.selectAIEngineAndKeyDefaultEngineFromEnv(); ok { + return engine + } + return string(constants.DefaultEngine) +} - // If engine is already overridden, skip selection - if c.EngineOverride != "" { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Using coding agent: "+c.EngineOverride)) - return c.configureEngineAPISecret(c.EngineOverride) +func (c *AddInteractiveConfig) selectAIEngineAndKeyDefaultEngineFromSecrets() (string, bool) { + // Priority 1: Check existing repository secrets using EngineOptions + // This takes precedence over workflow preference since users should use what's already available + for _, opt := range constants.EngineOptions { + if setutil.Contains(c.existingSecrets, opt.SecretName) { + addInteractiveLog.Printf("Found existing secret %s, recommending engine: %s", opt.SecretName, opt.Value) + return opt.Value, true + } } + return "", false +} - // Inform user if workflow specifies an engine - if workflowSpecifiedEngine != "" { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Workflow specifies engine: "+workflowSpecifiedEngine)) +func (c *AddInteractiveConfig) selectAIEngineAndKeyDefaultEngineFromEnv() (string, bool) { + // Priority 3: Check environment variables if no existing secret or workflow preference found + for _, opt := range constants.EngineOptions { + envVar := opt.SecretName + if opt.EnvVarName != "" { + envVar = opt.EnvVarName + } + if lookupEnv(envVar) != "" { + addInteractiveLog.Printf("Found env var %s, recommending engine: %s", envVar, opt.Value) + return opt.Value, true + } } + return "", false +} +func (c *AddInteractiveConfig) selectAIEngineAndKeyOptions(workflowSpecifiedEngine string) []huh.Option[string] { // Build engine options with notes about existing secrets and workflow specification. // The list of engines is derived from the catalog to ensure all registered engines appear. catalog := workflow.NewEngineCatalog(workflow.NewEngineRegistry()) - engineOptions := sliceutil.Map(catalog.All(), func(def *workflow.EngineDefinition) huh.Option[string] { + return sliceutil.Map(catalog.All(), func(def *workflow.EngineDefinition) huh.Option[string] { opt := constants.GetEngineOption(def.ID) label := fmt.Sprintf("%s - %s", def.DisplayName, def.Description) // Add markers for secret availability and workflow specification. @@ -105,9 +131,9 @@ func (c *AddInteractiveConfig) selectAIEngineAndKey() error { } return huh.NewOption(label, def.ID) }) +} - var selectedEngine string - +func (c *AddInteractiveConfig) selectAIEngineAndKeyPrompt(engineOptions []huh.Option[string], defaultEngine string, selectedEngine *string) error { // Set the default selection by moving it to front for i, opt := range engineOptions { if opt.Value == defaultEngine { @@ -124,17 +150,9 @@ func (c *AddInteractiveConfig) selectAIEngineAndKey() error { Title("Which coding agent would you like to use?"). Description("This determines which coding agent processes your workflows"). Options(engineOptions...). - Value(&selectedEngine), + Value(selectedEngine), ) - - if err := form.RunWithContext(c.Ctx); err != nil { - return fmt.Errorf("failed to select coding agent: %w", err) - } - - c.EngineOverride = selectedEngine - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Selected engine: "+selectedEngine)) - - return c.configureEngineAPISecret(selectedEngine) + return form.RunWithContext(c.Ctx) } // configureEngineAPISecret collects the API key for the selected engine using the unified engine secrets functions @@ -145,12 +163,7 @@ func (c *AddInteractiveConfig) configureEngineAPISecret(engine string) error { // Note: for Copilot workflows, --no-secret implies the PAT path; users who want // copilot-requests (org billing) should not pass --no-secret. if c.SkipSecret { - opt := constants.GetEngineOption(engine) - if opt != nil { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping %s secret setup (--no-secret flag set).", opt.SecretName))) - } else { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping secret setup (--no-secret flag set).")) - } + c.configureEngineAPISecretSkipSecret(engine) return nil } @@ -168,14 +181,7 @@ func (c *AddInteractiveConfig) configureEngineAPISecret(engine string) error { // If user doesn't have write access, skip secrets configuration. // Users without write access cannot configure repository secrets. if !c.hasWriteAccess { - opt := constants.GetEngineOption(engine) - if opt != nil { - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping %s secret setup — write access is required to configure repository secrets.", opt.SecretName))) - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, "Once you have write access or an admin configures the repository, set the secret with:") - fmt.Fprintln(os.Stderr, console.FormatCommandMessage(fmt.Sprintf(" gh aw secrets set %s --repo %s", opt.SecretName, c.RepoOverride))) - } + c.configureEngineAPISecretNoWriteAccess(engine) return nil } @@ -205,6 +211,27 @@ func (c *AddInteractiveConfig) configureEngineAPISecret(engine string) error { return nil } +func (c *AddInteractiveConfig) configureEngineAPISecretSkipSecret(engine string) { + opt := constants.GetEngineOption(engine) + if opt != nil { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping %s secret setup (--no-secret flag set).", opt.SecretName))) + } else { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping secret setup (--no-secret flag set).")) + } +} + +func (c *AddInteractiveConfig) configureEngineAPISecretNoWriteAccess(engine string) { + opt := constants.GetEngineOption(engine) + if opt == nil { + return + } + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping %s secret setup — write access is required to configure repository secrets.", opt.SecretName))) + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, "Once you have write access or an admin configures the repository, set the secret with:") + fmt.Fprintln(os.Stderr, console.FormatCommandMessage(fmt.Sprintf(" gh aw secrets set %s --repo %s", opt.SecretName, c.RepoOverride))) +} + // authMethodCopilotRequests is the wizard option value for Copilot org-billing authentication // (permissions.copilot-requests: write). Extracted as a package-level constant so both the // form definition and applyCopilotAuthMethodChoice reference the same sentinel. @@ -218,12 +245,45 @@ func (c *AddInteractiveConfig) selectCopilotAuthMethod() error { const authMethodPAT = "pat" + copilotRequestsLabel := "Use copilot-requests (org's Copilot billing, no PAT)" + probe := c.selectCopilotAuthMethodProbe() + copilotRequestsLabel += probe.LabelSuffix + + fmt.Fprintln(os.Stderr, "") + + options := selectCopilotAuthMethodOptions(probe, copilotRequestsLabel, authMethodPAT) + + var authMethod string + selectField := huh.NewSelect[string](). + Title("How would you like Copilot workflows to authenticate?"). + Description("copilot-requests uses the org's Copilot billing seat — no PAT required.\nPAT uses a fine-grained personal access token stored as COPILOT_GITHUB_TOKEN (requires repo write access to configure)."). + Options(options...). + Value(&authMethod) + + if probe.Disabled { + selectField = selectField.Validate(func(v string) error { + if v == authMethodCopilotRequests { + return errors.New("org Copilot CLI billing is disabled — please choose PAT") + } + return nil + }) + } + + form := console.NewSelectForm(selectField) + + if err := form.RunWithContext(c.Ctx); err != nil { + return fmt.Errorf("failed to select Copilot authentication method: %w", err) + } + + c.applyCopilotAuthMethodChoice(authMethod) + return nil +} + +func (c *AddInteractiveConfig) selectCopilotAuthMethodProbe() orgCopilotBillingProbeResult { // Detect org Copilot CLI billing status before building the form. // c.RepoOverride is in "owner/repo" format; we need just the org login. // When no org login is available the result is inconclusive (same as a // non-200 response) so the user still sees the info note. - copilotRequestsLabel := "Use copilot-requests (org's Copilot billing, no PAT)" - var probe orgCopilotBillingProbeResult if orgLogin, _, found := strings.Cut(c.RepoOverride, "/"); found && orgLogin != "" { probe = probeCopilotBillingForOrg(c.Ctx, orgLogin) @@ -233,13 +293,13 @@ func (c *AddInteractiveConfig) selectCopilotAuthMethod() error { } } c.copilotCLIBillingStatus = probe.BillingStatus - copilotRequestsLabel += probe.LabelSuffix if probe.InfoNote != "" { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(probe.InfoNote)) } + return probe +} - fmt.Fprintln(os.Stderr, "") - +func selectCopilotAuthMethodOptions(probe orgCopilotBillingProbeResult, copilotRequestsLabel, authMethodPAT string) []huh.Option[string] { // Build select options. // When billing is confirmed enabled, copilot-requests is listed first (pre-selected). // When billing is disabled or inconclusive, PAT is listed first (default selection). @@ -248,40 +308,14 @@ func (c *AddInteractiveConfig) selectCopilotAuthMethod() error { patOpt := huh.NewOption("Use a Personal Access Token (PAT) as COPILOT_GITHUB_TOKEN", authMethodPAT) copilotRequestsOpt := huh.NewOption(copilotRequestsLabel, authMethodCopilotRequests) - var options []huh.Option[string] switch probe.BillingStatus { case "enabled": // copilot-requests pre-selected - options = []huh.Option[string]{copilotRequestsOpt.Selected(true), patOpt} + return []huh.Option[string]{copilotRequestsOpt.Selected(true), patOpt} default: // PAT is default (first) for disabled or inconclusive - options = []huh.Option[string]{patOpt.Selected(true), copilotRequestsOpt} + return []huh.Option[string]{patOpt.Selected(true), copilotRequestsOpt} } - - var authMethod string - selectField := huh.NewSelect[string](). - Title("How would you like Copilot workflows to authenticate?"). - Description("copilot-requests uses the org's Copilot billing seat — no PAT required.\nPAT uses a fine-grained personal access token stored as COPILOT_GITHUB_TOKEN (requires repo write access to configure)."). - Options(options...). - Value(&authMethod) - - if probe.Disabled { - selectField = selectField.Validate(func(v string) error { - if v == authMethodCopilotRequests { - return errors.New("org Copilot CLI billing is disabled — please choose PAT") - } - return nil - }) - } - - form := console.NewSelectForm(selectField) - - if err := form.RunWithContext(c.Ctx); err != nil { - return fmt.Errorf("failed to select Copilot authentication method: %w", err) - } - - c.applyCopilotAuthMethodChoice(authMethod) - return nil } // applyCopilotAuthMethodChoice records the user's Copilot auth method selection and prints diff --git a/pkg/cli/add_interactive_git.go b/pkg/cli/add_interactive_git.go index bb4ae9a995e..8b7feba0b55 100644 --- a/pkg/cli/add_interactive_git.go +++ b/pkg/cli/add_interactive_git.go @@ -28,6 +28,22 @@ func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecret(ctx context.Co fmt.Fprintln(os.Stderr, "") + result, err := c.createWorkflowPRAndConfigureSecretAddWorkflows(ctx) + if err != nil { + return fmt.Errorf("failed to add workflow: %w", err) + } + c.addResult = result + + // Step 8b: Optionally merge the PR – loop until merged, confirmed-merged, or user exits + if err := c.createWorkflowPRAndConfigureSecretMergePR(result); err != nil { + return err + } + + // Step 8c: Add the secret (skip if no secret configured or already exists in repository) + return c.createWorkflowPRAndConfigureSecretAddSecret(secretName, secretValue) +} + +func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecretAddWorkflows(ctx context.Context) (*AddWorkflowsResult, error) { // Add the workflow using existing implementation with --create-pull-request // Pass the resolved workflows to avoid re-fetching them // Pass Quiet=true to suppress detailed output (already shown earlier in interactive mode) @@ -47,13 +63,20 @@ func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecret(ctx context.Co DisableSecurityScanner: c.DisableSecurityScanner, AddCopilotRequestsPermission: c.UseCopilotRequests, } - result, err := AddResolvedWorkflows(ctx, c.WorkflowSpecs, c.resolvedWorkflows, opts) - if err != nil { - return fmt.Errorf("failed to add workflow: %w", err) - } - c.addResult = result + return AddResolvedWorkflows(ctx, c.WorkflowSpecs, c.resolvedWorkflows, opts) +} - // Step 8b: Optionally merge the PR – loop until merged, confirmed-merged, or user exits +type createWorkflowPRAndConfigureSecretMergeAction string + +const ( + createWorkflowPRAndConfigureSecretMergeActionAttempt createWorkflowPRAndConfigureSecretMergeAction = "attempt" + createWorkflowPRAndConfigureSecretMergeActionEditTitle createWorkflowPRAndConfigureSecretMergeAction = "editTitle" + createWorkflowPRAndConfigureSecretMergeActionReview createWorkflowPRAndConfigureSecretMergeAction = "review" + createWorkflowPRAndConfigureSecretMergeActionConfirmed createWorkflowPRAndConfigureSecretMergeAction = "confirmed" + createWorkflowPRAndConfigureSecretMergeActionExit createWorkflowPRAndConfigureSecretMergeAction = "exit" +) + +func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecretMergePR(result *AddWorkflowsResult) error { if result.PRNumber == 0 { if result.PRURL == "" { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Requested workflow files already exist locally; no pull request was created.")) @@ -61,115 +84,126 @@ func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecret(ctx context.Co } fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Could not determine PR number")) fmt.Fprintln(os.Stderr, "Please merge the PR manually from the GitHub web interface.") - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Pull request created: "+result.PRURL)) - fmt.Fprintln(os.Stderr, "") - - // mergeAction values used in the select loop - type mergeAction string - const ( - mergeActionAttempt mergeAction = "attempt" - mergeActionEditTitle mergeAction = "editTitle" - mergeActionReview mergeAction = "review" - mergeActionConfirmed mergeAction = "confirmed" - mergeActionExit mergeAction = "exit" - ) - - mergeDone := false // true when the PR is merged (or confirmed merged) - mergeFailed := false // true after an unsuccessful merge attempt - userReviewing := false // true after the user chose "I'll review myself" - - for !mergeDone { - // Build option list based on current state - var options []huh.Option[mergeAction] + return nil + } - options = append(options, huh.NewOption("Attempt to merge", mergeActionAttempt)) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Pull request created: "+result.PRURL)) + fmt.Fprintln(os.Stderr, "") + return c.createWorkflowPRAndConfigureSecretMergeLoop(result) +} - if mergeFailed { - options = append(options, huh.NewOption("Edit PR title and retry", mergeActionEditTitle)) - } +func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecretMergeLoop(result *AddWorkflowsResult) error { + mergeDone := false // true when the PR is merged (or confirmed merged) + mergeFailed := false // true after an unsuccessful merge attempt + userReviewing := false // true after the user chose "I'll review myself" - if userReviewing { - options = append(options, huh.NewOption("PR has been manually merged", mergeActionConfirmed)) - } else { - options = append(options, huh.NewOption("I'll review/merge myself", mergeActionReview)) - } + for !mergeDone { + chosen, err := createWorkflowPRAndConfigureSecretMergeChoice(result.PRURL, mergeFailed, userReviewing) + if err != nil { + return err + } - if userReviewing { - options = append(options, huh.NewOption("Exit, I'm done here", mergeActionExit)) - } else { - options = append(options, huh.NewOption("Exit", mergeActionExit)) + switch chosen { + case createWorkflowPRAndConfigureSecretMergeActionAttempt: + mergeDone = c.createWorkflowPRAndConfigureSecretAttemptMerge(result, mergeFailed) + mergeFailed = !mergeDone + case createWorkflowPRAndConfigureSecretMergeActionEditTitle: + if updated, err := c.createWorkflowPRAndConfigureSecretEditTitle(result.PRNumber); err != nil { + return err + } else if updated { + mergeFailed = false } + case createWorkflowPRAndConfigureSecretMergeActionReview: + userReviewing = true + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please review and merge the pull request: "+result.PRURL)) + fmt.Fprintln(os.Stderr, "") + case createWorkflowPRAndConfigureSecretMergeActionConfirmed: + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Great – continuing with the merged pull request")) + mergeDone = true + case createWorkflowPRAndConfigureSecretMergeActionExit: + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Exiting. You can merge the pull request later: "+result.PRURL)) + return errors.New("user exited before PR was merged") + } + } + return nil +} - var chosen mergeAction - selectForm := console.NewSelectForm( - huh.NewSelect[mergeAction](). - Title("What would you like to do with pull request " + result.PRURL + "?"). - Options(options...). - Value(&chosen), - ) +func createWorkflowPRAndConfigureSecretMergeChoice(prURL string, mergeFailed, userReviewing bool) (createWorkflowPRAndConfigureSecretMergeAction, error) { + options := createWorkflowPRAndConfigureSecretMergeOptions(mergeFailed, userReviewing) + var chosen createWorkflowPRAndConfigureSecretMergeAction + selectForm := console.NewSelectForm( + huh.NewSelect[createWorkflowPRAndConfigureSecretMergeAction](). + Title("What would you like to do with pull request " + prURL + "?"). + Options(options...). + Value(&chosen), + ) + + if selectErr := selectForm.Run(); selectErr != nil { + return "", fmt.Errorf("failed to get user input: %w", selectErr) + } + return chosen, nil +} - if selectErr := selectForm.Run(); selectErr != nil { - return fmt.Errorf("failed to get user input: %w", selectErr) - } +func createWorkflowPRAndConfigureSecretMergeOptions(mergeFailed, userReviewing bool) []huh.Option[createWorkflowPRAndConfigureSecretMergeAction] { + // Build option list based on current state + options := []huh.Option[createWorkflowPRAndConfigureSecretMergeAction]{ + huh.NewOption("Attempt to merge", createWorkflowPRAndConfigureSecretMergeActionAttempt), + } + if mergeFailed { + options = append(options, huh.NewOption("Edit PR title and retry", createWorkflowPRAndConfigureSecretMergeActionEditTitle)) + } + if userReviewing { + options = append(options, huh.NewOption("PR has been manually merged", createWorkflowPRAndConfigureSecretMergeActionConfirmed)) + options = append(options, huh.NewOption("Exit, I'm done here", createWorkflowPRAndConfigureSecretMergeActionExit)) + } else { + options = append(options, huh.NewOption("I'll review/merge myself", createWorkflowPRAndConfigureSecretMergeActionReview)) + options = append(options, huh.NewOption("Exit", createWorkflowPRAndConfigureSecretMergeActionExit)) + } + return options +} - switch chosen { - case mergeActionAttempt: - if mergeErr := c.mergePullRequest(result.PRNumber); mergeErr != nil { - if isAlreadyMergedGHError(mergeErr) { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Merged pull request "+result.PRURL)) - mergeDone = true - } else { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to merge PR: %v", mergeErr))) - if mergeFailed { - fmt.Fprintln(os.Stderr, "Please merge the PR manually: "+result.PRURL) - } - mergeFailed = true - } - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Merged pull request "+result.PRURL)) - mergeDone = true - } - - case mergeActionEditTitle: - var newTitle string - titleForm := console.NewInputForm( - huh.NewInput(). - Title("Enter new PR title"). - Description("Add a prefix if required, for example: feat: or fix:"). - Value(&newTitle), - ) - if titleErr := titleForm.Run(); titleErr != nil { - return fmt.Errorf("failed to get user input: %w", titleErr) - } - newTitle = strings.TrimSpace(newTitle) - if newTitle == "" { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("PR title cannot be empty, keeping current title")) - } else if editErr := editPRTitle(result.PRNumber, newTitle, c.RepoOverride); editErr != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to update PR title: %v", editErr))) - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("PR title updated to: "+newTitle)) - mergeFailed = false - } - - case mergeActionReview: - userReviewing = true - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please review and merge the pull request: "+result.PRURL)) - fmt.Fprintln(os.Stderr, "") - - case mergeActionConfirmed: - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Great – continuing with the merged pull request")) - mergeDone = true - - case mergeActionExit: - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Exiting. You can merge the pull request later: "+result.PRURL)) - return errors.New("user exited before PR was merged") - } +func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecretAttemptMerge(result *AddWorkflowsResult, mergeFailed bool) bool { + if mergeErr := c.mergePullRequest(result.PRNumber); mergeErr != nil { + if isAlreadyMergedGHError(mergeErr) { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Merged pull request "+result.PRURL)) + return true } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to merge PR: %v", mergeErr))) + if mergeFailed { + fmt.Fprintln(os.Stderr, "Please merge the PR manually: "+result.PRURL) + } + return false + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Merged pull request "+result.PRURL)) + return true +} + +func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecretEditTitle(prNumber int) (bool, error) { + var newTitle string + titleForm := console.NewInputForm( + huh.NewInput(). + Title("Enter new PR title"). + Description("Add a prefix if required, for example: feat: or fix:"). + Value(&newTitle), + ) + if titleErr := titleForm.Run(); titleErr != nil { + return false, fmt.Errorf("failed to get user input: %w", titleErr) + } + newTitle = strings.TrimSpace(newTitle) + if newTitle == "" { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("PR title cannot be empty, keeping current title")) + return false, nil + } + if editErr := editPRTitle(prNumber, newTitle, c.RepoOverride); editErr != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to update PR title: %v", editErr))) + return false, nil } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("PR title updated to: "+newTitle)) + return true, nil +} - // Step 8c: Add the secret (skip if no secret configured or already exists in repository) +func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecretAddSecret(secretName, secretValue string) error { if secretName == "" { // No secret to configure (e.g., user doesn't have write access to the repository) } else if secretValue == "" { @@ -203,27 +237,7 @@ func (c *AddInteractiveConfig) createWorkflowPRAndConfigureSecret(ctx context.Co func (c *AddInteractiveConfig) updateLocalBranch() error { addInteractiveLog.Print("Updating local branch with merged changes") - // Get the default branch name using gh - output, err := workflow.RunGHCombined("Getting default branch...", "repo", "view", "--repo", c.RepoOverride, "--json", "defaultBranchRef", "--jq", ".defaultBranchRef.name") - defaultBranch := "" - if err == nil { - defaultBranch = strings.TrimSpace(string(output)) - } - - // Fallback: query the local origin remote directly (works even when gh repo - // view fails, e.g. forks without a default remote set). - if defaultBranch == "" { - addInteractiveLog.Print("gh repo view failed, trying git ls-remote to detect default branch") - lsCmd := exec.Command("git", "ls-remote", "--symref", "origin", "HEAD") - lsOutput, lsErr := lsCmd.CombinedOutput() - if lsErr == nil { - defaultBranch = parseDefaultBranchFromLsRemote(string(lsOutput)) - } - } - - if defaultBranch == "" { - defaultBranch = "main" - } + defaultBranch := c.updateLocalBranchDefaultBranch() addInteractiveLog.Printf("Default branch: %s", defaultBranch) // Fetch the latest changes from origin @@ -266,6 +280,31 @@ func (c *AddInteractiveConfig) updateLocalBranch() error { return nil } +func (c *AddInteractiveConfig) updateLocalBranchDefaultBranch() string { + // Get the default branch name using gh + output, err := workflow.RunGHCombined("Getting default branch...", "repo", "view", "--repo", c.RepoOverride, "--json", "defaultBranchRef", "--jq", ".defaultBranchRef.name") + defaultBranch := "" + if err == nil { + defaultBranch = strings.TrimSpace(string(output)) + } + + // Fallback: query the local origin remote directly (works even when gh repo + // view fails, e.g. forks without a default remote set). + if defaultBranch == "" { + addInteractiveLog.Print("gh repo view failed, trying git ls-remote to detect default branch") + lsCmd := exec.Command("git", "ls-remote", "--symref", "origin", "HEAD") + lsOutput, lsErr := lsCmd.CombinedOutput() + if lsErr == nil { + defaultBranch = parseDefaultBranchFromLsRemote(string(lsOutput)) + } + } + + if defaultBranch == "" { + defaultBranch = "main" + } + return defaultBranch +} + // checkCleanWorkingDirectory verifies the working directory has no uncommitted changes. // This is checked early in the interactive flow to avoid failing later during PR creation. func (c *AddInteractiveConfig) checkCleanWorkingDirectory() error { diff --git a/pkg/cli/add_interactive_orchestrator.go b/pkg/cli/add_interactive_orchestrator.go index 866f6dc28cb..3cfc8373d37 100644 --- a/pkg/cli/add_interactive_orchestrator.go +++ b/pkg/cli/add_interactive_orchestrator.go @@ -71,6 +71,40 @@ type AddInteractiveConfig struct { func RunAddInteractive(ctx context.Context, config *AddInteractiveConfig) error { addInteractiveLog.Print("Starting interactive add workflow") + if err := runAddInteractivePrepare(ctx, config); err != nil { + return err + } + + // Step 1: Welcome message + console.ShowWelcomeBanner("This tool will walk you through adding an automated workflow to your repository.") + + if err := runAddInteractiveCheckRepository(config); err != nil { + return err + } + + initFiles, err := runAddInteractiveConfigure(config) + if err != nil { + return err + } + + filesToAdd, secretName, secretValue, err := runAddInteractivePlanChanges(config, initFiles) + if err != nil { + return err + } + + if err := runAddInteractiveApplyChanges(ctx, config, filesToAdd, initFiles, secretName, secretValue); err != nil { + return err + } + + // Step 10: Check status and offer to run + if err := config.checkStatusAndOfferRun(ctx); err != nil { + return err + } + + return nil +} + +func runAddInteractivePrepare(ctx context.Context, config *AddInteractiveConfig) error { // Assert this function is not running in automated unit tests or CI. // GO_TEST_MODE intentionally uses GetBoolFromEnv so common boolean spellings // are treated consistently across test and automation environments, while @@ -81,110 +115,100 @@ func RunAddInteractive(ctx context.Context, config *AddInteractiveConfig) error // Set context on the config config.Ctx = ctx + runAddInteractiveDetectGHESHost(config) + return nil +} +func runAddInteractiveDetectGHESHost(config *AddInteractiveConfig) { // Auto-detect GHES host from git remote if not already set - if os.Getenv("GH_HOST") == "" { //nolint:osgetenvlibrary - detectedHost := getHostFromOriginRemote() - if detectedHost != "github.com" { - addInteractiveLog.Printf("Auto-detected GHES host from git remote: %s", detectedHost) - workflow.SetDefaultGHHost(detectedHost) - if config.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Auto-detected GitHub Enterprise host: "+detectedHost)) - } + if os.Getenv("GH_HOST") != "" { //nolint:osgetenvlibrary + return + } + detectedHost := getHostFromOriginRemote() + if detectedHost != "github.com" { + addInteractiveLog.Printf("Auto-detected GHES host from git remote: %s", detectedHost) + workflow.SetDefaultGHHost(detectedHost) + if config.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Auto-detected GitHub Enterprise host: "+detectedHost)) } } +} - // Step 1: Welcome message - console.ShowWelcomeBanner("This tool will walk you through adding an automated workflow to your repository.") - +func runAddInteractiveCheckRepository(config *AddInteractiveConfig) error { // Step 1b: Resolve workflows early to get descriptions and validate specs if err := config.resolveWorkflows(); err != nil { return err } - // Step 1c: Show workflow descriptions if available config.showWorkflowDescriptions() - - // Step 2: Check gh auth status - if err := config.checkGHAuthStatus(); err != nil { - return err - } - - // Step 3: Check git repository and get org/repo - if err := config.checkGitRepository(); err != nil { - return err - } - - // Step 3b: Check working directory is clean (must be clean for PR creation later) - if err := config.checkCleanWorkingDirectory(); err != nil { - return err + checks := []func() error{ + config.checkGHAuthStatus, // Step 2: Check gh auth status + config.checkGitRepository, // Step 3: Check git repository and get org/repo + config.checkCleanWorkingDirectory, // Step 3b: Check working directory is clean + config.checkActionsEnabled, // Step 4: Check GitHub Actions is enabled + config.checkUserPermissions, // Step 5: Check user permissions } - - // Step 4: Check GitHub Actions is enabled - if err := config.checkActionsEnabled(); err != nil { - return err - } - - // Step 5: Check user permissions - if err := config.checkUserPermissions(); err != nil { - return err + for _, check := range checks { + if err := check(); err != nil { + return err + } } + return nil +} +func runAddInteractiveConfigure(config *AddInteractiveConfig) ([]string, error) { // Step 6: Select coding agent and collect API key if err := config.selectAIEngineAndKey(); err != nil { - return err - } - - initFiles, err := ensureAddRepositoryInitializedWithDetails(config.EngineOverride, config.Verbose, config.NoGitattributes) - if err != nil { - return err + return nil, err } + return ensureAddRepositoryInitializedWithDetails(config.EngineOverride, config.Verbose, config.NoGitattributes) +} +func runAddInteractivePlanChanges(config *AddInteractiveConfig, initFiles []string) ([]string, string, string, error) { // Step 7: Determine files to add filesToAdd, _, err := config.determineFilesToAdd() if err != nil { - return err + return nil, "", "", err } - // Step 7b: Offer schedule frequency selection for scheduled workflows if err := config.selectScheduleFrequency(); err != nil { - return err + return nil, "", "", err } - // Step 8: Confirm with user - var secretName, secretValue string - if config.hasWriteAccess && !config.SkipSecret && !config.UseCopilotRequests { - secretName, secretValue, err = config.resolveEngineApiKeyCredential() - if err != nil { - return err - } + secretName, secretValue, err := runAddInteractiveResolveSecret(config) + if err != nil { + return nil, "", "", err } - if err := config.confirmChanges(filesToAdd, initFiles, secretName, secretValue); err != nil { - return err + return nil, "", "", err } + return filesToAdd, secretName, secretValue, nil +} +func runAddInteractiveResolveSecret(config *AddInteractiveConfig) (string, string, error) { + if config.hasWriteAccess && !config.SkipSecret && !config.UseCopilotRequests { + return config.resolveEngineApiKeyCredential() + } + return "", "", nil +} + +func runAddInteractiveApplyChanges(ctx context.Context, config *AddInteractiveConfig, filesToAdd, initFiles []string, secretName, secretValue string) error { // Step 9: Apply changes (create PR, merge, add secret) if err := config.createWorkflowPRAndConfigureSecret(ctx, filesToAdd, initFiles, secretName, secretValue); err != nil { return err } + return runAddInteractiveBootstrap(ctx, config) +} +func runAddInteractiveBootstrap(ctx context.Context, config *AddInteractiveConfig) error { // Step 9b: Apply bootstrap config steps interactively (if the package declares any) - if config.resolvedWorkflows != nil && config.resolvedWorkflows.BootstrapProfile != nil { - if config.hasWriteAccess { - if err := executeBootstrapConfigForAdd(ctx, config.RepoOverride, config.WorkflowSpecs, config.resolvedWorkflows.BootstrapProfile, config.UseCopilotRequests, config.Verbose); err != nil { - return err - } - } else { - printBootstrapConfigTODO(os.Stderr, config.resolvedWorkflows.BootstrapProfile) - } + if config.resolvedWorkflows == nil || config.resolvedWorkflows.BootstrapProfile == nil { + return nil } - - // Step 10: Check status and offer to run - if err := config.checkStatusAndOfferRun(ctx); err != nil { - return err + if config.hasWriteAccess { + return executeBootstrapConfigForAdd(ctx, config.RepoOverride, config.WorkflowSpecs, config.resolvedWorkflows.BootstrapProfile, config.UseCopilotRequests, config.Verbose) } - + printBootstrapConfigTODO(os.Stderr, config.resolvedWorkflows.BootstrapProfile) return nil } diff --git a/pkg/cli/add_interactive_schedule.go b/pkg/cli/add_interactive_schedule.go index 5e3961d6625..0262422a7d5 100644 --- a/pkg/cli/add_interactive_schedule.go +++ b/pkg/cli/add_interactive_schedule.go @@ -73,54 +73,68 @@ func detectWorkflowScheduleInfo(content string) scheduleDetection { // Case 2: on is a map — extract schedule value if present if onMap, ok := onValue.(map[string]any); ok { - schedValue, hasSchedule := onMap["schedule"] - if !hasSchedule { - return scheduleDetection{} + return detectWorkflowScheduleInfoFromOnMap(onMap) + } + + return scheduleDetection{} +} + +func detectWorkflowScheduleInfoFromOnMap(onMap map[string]any) scheduleDetection { + schedValue, hasSchedule := onMap["schedule"] + if !hasSchedule { + return scheduleDetection{} + } + + isMultiTrigger := detectWorkflowScheduleInfoMultiTrigger(onMap) + + // Schedule as string shorthand (e.g., "schedule: daily") + if schedStr, ok := schedValue.(string); ok { + return scheduleDetection{ + RawExpr: schedStr, + Frequency: classifyScheduleFrequency(schedStr), + IsUpdatable: true, + IsMultiTrigger: isMultiTrigger, + IsOnMap: true, } + } - // Determine if on: has triggers beyond schedule / workflow_dispatch - isMultiTrigger := false - for key := range onMap { - if key != "schedule" && key != "workflow_dispatch" { - isMultiTrigger = true - scheduleWizardLog.Printf("Multi-trigger on: map detected (trigger '%s')", key) - break - } + return detectWorkflowScheduleInfoFromScheduleArray(schedValue, isMultiTrigger) +} + +func detectWorkflowScheduleInfoMultiTrigger(onMap map[string]any) bool { + // Determine if on: has triggers beyond schedule / workflow_dispatch + for key := range onMap { + if key != "schedule" && key != "workflow_dispatch" { + scheduleWizardLog.Printf("Multi-trigger on: map detected (trigger '%s')", key) + return true } + } + return false +} - // Schedule as string shorthand (e.g., "schedule: daily") - if schedStr, ok := schedValue.(string); ok { +func detectWorkflowScheduleInfoFromScheduleArray(schedValue any, isMultiTrigger bool) scheduleDetection { + // Schedule as array (e.g., "schedule:\n - cron: daily") + schedArray, ok := schedValue.([]any) + if !ok || len(schedArray) == 0 { + return scheduleDetection{} + } + // Workflows with multiple cron entries cannot be safely rewritten to a single + // frequency, so mark them as not updatable. + if len(schedArray) > 1 { + scheduleWizardLog.Printf("Multiple cron entries (%d) detected — not updatable", len(schedArray)) + return scheduleDetection{} + } + if item, ok := schedArray[0].(map[string]any); ok { + if cronVal, ok := item["cron"].(string); ok { return scheduleDetection{ - RawExpr: schedStr, - Frequency: classifyScheduleFrequency(schedStr), + RawExpr: cronVal, + Frequency: classifyScheduleFrequency(cronVal), IsUpdatable: true, IsMultiTrigger: isMultiTrigger, IsOnMap: true, } } - - // Schedule as array (e.g., "schedule:\n - cron: daily") - if schedArray, ok := schedValue.([]any); ok && len(schedArray) > 0 { - // Workflows with multiple cron entries cannot be safely rewritten to a single - // frequency, so mark them as not updatable. - if len(schedArray) > 1 { - scheduleWizardLog.Printf("Multiple cron entries (%d) detected — not updatable", len(schedArray)) - return scheduleDetection{} - } - if item, ok := schedArray[0].(map[string]any); ok { - if cronVal, ok := item["cron"].(string); ok { - return scheduleDetection{ - RawExpr: cronVal, - Frequency: classifyScheduleFrequency(cronVal), - IsUpdatable: true, - IsMultiTrigger: isMultiTrigger, - IsOnMap: true, - } - } - } - } } - return scheduleDetection{} } @@ -129,30 +143,11 @@ func detectWorkflowScheduleInfo(content string) scheduleDetection { func classifyScheduleFrequency(scheduleStr string) string { normalized := strings.ToLower(strings.TrimSpace(scheduleStr)) - // Direct friendly-format matches - switch normalized { - case "hourly", "every 1h", "every 1 hour", "every 1 hours": - return "hourly" - case "every 3h", "every 3 hours": - return "3-hourly" - case "daily": - return "daily" - case "weekly": - return "weekly" - } - - // Fuzzy cron placeholder matches (produced by the compiler during preprocessing) - if strings.HasPrefix(normalized, "fuzzy:hourly/1 ") || normalized == "fuzzy:hourly/1" { //nolint:tolowerequalfold - return "hourly" + if frequency := classifyScheduleFrequencyFriendly(normalized); frequency != "" { + return frequency } - if strings.HasPrefix(normalized, "fuzzy:hourly/3 ") || normalized == "fuzzy:hourly/3" { //nolint:tolowerequalfold - return "3-hourly" - } - if strings.HasPrefix(normalized, "fuzzy:daily") { - return "daily" - } - if strings.HasPrefix(normalized, "fuzzy:weekly") { - return "weekly" + if frequency := classifyScheduleFrequencyFuzzy(normalized); frequency != "" { + return frequency } // Cron expression checks @@ -178,6 +173,47 @@ func classifyScheduleFrequency(scheduleStr string) string { return "weekly" } + if classifyScheduleFrequencyMonthly(scheduleStr) { + return "monthly" + } + + return "custom" +} + +func classifyScheduleFrequencyFriendly(normalized string) string { + // Direct friendly-format matches + switch normalized { + case "hourly", "every 1h", "every 1 hour", "every 1 hours": + return "hourly" + case "every 3h", "every 3 hours": + return "3-hourly" + case "daily": + return "daily" + case "weekly": + return "weekly" + default: + return "" + } +} + +func classifyScheduleFrequencyFuzzy(normalized string) string { + // Fuzzy cron placeholder matches (produced by the compiler during preprocessing) + if strings.HasPrefix(normalized, "fuzzy:hourly/1 ") || normalized == "fuzzy:hourly/1" { //nolint:tolowerequalfold + return "hourly" + } + if strings.HasPrefix(normalized, "fuzzy:hourly/3 ") || normalized == "fuzzy:hourly/3" { //nolint:tolowerequalfold + return "3-hourly" + } + if strings.HasPrefix(normalized, "fuzzy:daily") { + return "daily" + } + if strings.HasPrefix(normalized, "fuzzy:weekly") { + return "weekly" + } + return "" +} + +func classifyScheduleFrequencyMonthly(scheduleStr string) bool { // Monthly cron: M H * * where is a specific numeric date (e.g. "1", "15"). // fields: [0]=minute [1]=hour [2]=day-of-month [3]=month [4]=day-of-week // Excludes interval expressions like "*/2" so that "0 0 */2 * *" (every-2-days) is @@ -186,11 +222,11 @@ func classifyScheduleFrequency(scheduleStr string) string { if len(fields) == 5 && fields[3] == "*" && fields[4] == "*" { day := fields[2] // day-of-month field if day != "*" && !strings.ContainsAny(day, "*/-,") { - return "monthly" + return true } } - return "custom" + return false } // selectScheduleFrequency presents a schedule-frequency selection form to the user when the @@ -202,79 +238,99 @@ func (c *AddInteractiveConfig) selectScheduleFrequency() error { } for _, wf := range c.resolvedWorkflows.Workflows { - content := string(wf.Content) - detection := detectWorkflowScheduleInfo(content) - if !detection.IsUpdatable { - continue + if err := c.selectScheduleFrequencyForWorkflow(wf); err != nil { + return err } + } - rawExpr := detection.RawExpr - currentFreq := detection.Frequency - scheduleWizardLog.Printf("Detected schedule: expr=%q, freq=%s, multiTrigger=%v", rawExpr, currentFreq, detection.IsMultiTrigger) + return nil +} - // Build the ordered option list - options := buildScheduleOptions(rawExpr, currentFreq) +func (c *AddInteractiveConfig) selectScheduleFrequencyForWorkflow(wf *ResolvedWorkflow) error { + content := string(wf.Content) + detection := detectWorkflowScheduleInfo(content) + if !detection.IsUpdatable { + return nil + } - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("This workflow runs on a schedule.")) + rawExpr := detection.RawExpr + currentFreq := detection.Frequency + scheduleWizardLog.Printf("Detected schedule: expr=%q, freq=%s, multiTrigger=%v", rawExpr, currentFreq, detection.IsMultiTrigger) - var selected string - form := console.NewSelectForm( - huh.NewSelect[string](). - Title("How often should this workflow run?"). - Description("Current schedule: " + rawExpr). - Options(options...). - Value(&selected), - ) + selected, err := c.selectScheduleFrequencyPrompt(rawExpr, currentFreq) + if err != nil { + return err + } - if err := form.RunWithContext(c.Ctx); err != nil { - return fmt.Errorf("failed to select schedule frequency: %w", err) - } + // "custom" or same frequency means keep as-is + if selected == "custom" || selected == currentFreq { + scheduleWizardLog.Printf("Schedule unchanged: keeping %q", rawExpr) + return nil + } - scheduleWizardLog.Printf("User selected frequency: %s", selected) + newExpr := selectScheduleFrequencyExpression(selected) + if newExpr == "" { + return nil + } + return selectScheduleFrequencyUpdateWorkflow(wf, content, detection, newExpr, selected) +} - // "custom" or same frequency means keep as-is - if selected == "custom" || selected == currentFreq { - scheduleWizardLog.Printf("Schedule unchanged: keeping %q", rawExpr) - continue - } +func (c *AddInteractiveConfig) selectScheduleFrequencyPrompt(rawExpr, currentFreq string) (string, error) { + // Build the ordered option list + options := buildScheduleOptions(rawExpr, currentFreq) - // Look up the schedule expression for the chosen frequency - var newExpr string - for _, opt := range standardScheduleFrequencies { - if opt.Value == selected { - newExpr = opt.Expression - break - } - } - if newExpr == "" { - continue - } + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("This workflow runs on a schedule.")) - // Update the workflow content in memory. - // When on: is a mapping, update only the schedule sub-key so other triggers - // (e.g., workflow_dispatch, push) are preserved. - // When on: is a scalar string, replace the on: field value directly. - var updatedContent string - var updateErr error - if detection.IsOnMap { - updatedContent, updateErr = UpdateScheduleInOnBlock(content, newExpr) - } else { - updatedContent, updateErr = UpdateFieldInFrontmatter(content, "on", newExpr) - } - if updateErr != nil { - scheduleWizardLog.Printf("Failed to update schedule (isOnMap=%v): %v", detection.IsOnMap, updateErr) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not update schedule: %v", updateErr))) - continue - } + var selected string + form := console.NewSelectForm( + huh.NewSelect[string](). + Title("How often should this workflow run?"). + Description("Current schedule: " + rawExpr). + Options(options...). + Value(&selected), + ) + + if err := form.RunWithContext(c.Ctx); err != nil { + return "", fmt.Errorf("failed to select schedule frequency: %w", err) + } + scheduleWizardLog.Printf("User selected frequency: %s", selected) + return selected, nil +} - wf.Content = []byte(updatedContent) - if wf.SourceInfo != nil { - wf.SourceInfo.Content = []byte(updatedContent) +func selectScheduleFrequencyExpression(selected string) string { + // Look up the schedule expression for the chosen frequency + for _, opt := range standardScheduleFrequencies { + if opt.Value == selected { + return opt.Expression } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Schedule updated to: "+selected)) + } + return "" +} + +func selectScheduleFrequencyUpdateWorkflow(wf *ResolvedWorkflow, content string, detection scheduleDetection, newExpr, selected string) error { + // Update the workflow content in memory. + // When on: is a mapping, update only the schedule sub-key so other triggers + // (e.g., workflow_dispatch, push) are preserved. + // When on: is a scalar string, replace the on: field value directly. + var updatedContent string + var updateErr error + if detection.IsOnMap { + updatedContent, updateErr = UpdateScheduleInOnBlock(content, newExpr) + } else { + updatedContent, updateErr = UpdateFieldInFrontmatter(content, "on", newExpr) + } + if updateErr != nil { + scheduleWizardLog.Printf("Failed to update schedule (isOnMap=%v): %v", detection.IsOnMap, updateErr) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not update schedule: %v", updateErr))) + return nil } + wf.Content = []byte(updatedContent) + if wf.SourceInfo != nil { + wf.SourceInfo.Content = []byte(updatedContent) + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Schedule updated to: "+selected)) return nil } diff --git a/pkg/cli/add_interactive_workflow.go b/pkg/cli/add_interactive_workflow.go index cb011f1dc2a..e0fd7be0e05 100644 --- a/pkg/cli/add_interactive_workflow.go +++ b/pkg/cli/add_interactive_workflow.go @@ -20,82 +20,88 @@ func (c *AddInteractiveConfig) checkStatusAndOfferRun(ctx context.Context) error // Wait a moment for GitHub to process the merge fmt.Fprintln(os.Stderr, "") + // Try a few times to see the workflow in status + workflowFound, err := c.checkStatusAndOfferRunWaitForWorkflow(ctx) + if err != nil { + return err + } + + if !workflowFound { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Could not verify workflow status.")) + fmt.Fprintf(os.Stderr, "You can check status with: %s status\n", string(constants.CLIExtensionPrefix)) + c.showFinalInstructions() + return nil + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Workflow is ready")) + + return c.checkStatusAndOfferRunOffer(ctx) +} + +func (c *AddInteractiveConfig) checkStatusAndOfferRunWaitForWorkflow(ctx context.Context) (bool, error) { // Use spinner only in non-verbose mode (spinner can't be restarted after stop) var spinner *console.SpinnerWrapper if !c.Verbose { spinner = console.NewSpinner("Waiting for workflow to be available...") spinner.Start() } - - // Try a few times to see the workflow in status - var workflowFound bool - for i := range 5 { - // Wait 2 seconds before each check (including the first) - timer := time.NewTimer(2 * time.Second) - select { - case <-ctx.Done(): - timer.Stop() - if spinner != nil { - spinner.Stop() - } - return ctx.Err() - case <-timer.C: - // Continue with check + defer func() { + if spinner != nil { + spinner.Stop() } + }() - workflowName := c.primaryWorkflowName() - if workflowName != "" { - if c.Verbose { - fmt.Fprintf(os.Stderr, "Checking workflow status (attempt %d/5) for: %s\n", i+1, workflowName) - } - // Check if workflow is in status - statuses, err := findWorkflowsByFilenamePattern(workflowName, c.RepoOverride, c.Verbose) - if err != nil { - if c.Verbose { - fmt.Fprintf(os.Stderr, "Status check error: %v\n", err) - } - } else if len(statuses) > 0 { - if c.Verbose { - fmt.Fprintf(os.Stderr, "Found %d workflow(s) matching pattern\n", len(statuses)) - } - workflowFound = true - break - } else if c.Verbose { - fmt.Fprintln(os.Stderr, "No workflows found matching pattern yet") - } + for i := range 5 { + found, err := c.checkStatusAndOfferRunAttempt(ctx, i) + if err != nil || found { + return found, err } } + return false, nil +} - if spinner != nil { - spinner.Stop() +func (c *AddInteractiveConfig) checkStatusAndOfferRunAttempt(ctx context.Context, i int) (bool, error) { + // Wait 2 seconds before each check (including the first) + timer := time.NewTimer(2 * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return false, ctx.Err() + case <-timer.C: + // Continue with check } - if !workflowFound { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Could not verify workflow status.")) - fmt.Fprintf(os.Stderr, "You can check status with: %s status\n", string(constants.CLIExtensionPrefix)) - c.showFinalInstructions() - return nil + workflowName := c.primaryWorkflowName() + if workflowName == "" { + return false, nil } - - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Workflow is ready")) - - // Only offer to run if workflow has workflow_dispatch trigger - if c.addResult == nil || !c.addResult.HasWorkflowDispatch { - addInteractiveLog.Print("Workflow does not have workflow_dispatch trigger, skipping run offer") - c.showFinalInstructions() - return nil + if c.Verbose { + fmt.Fprintf(os.Stderr, "Checking workflow status (attempt %d/5) for: %s\n", i+1, workflowName) + } + // Check if workflow is in status + statuses, err := findWorkflowsByFilenamePattern(workflowName, c.RepoOverride, c.Verbose) + if err != nil { + if c.Verbose { + fmt.Fprintf(os.Stderr, "Status check error: %v\n", err) + } + return false, nil + } + if len(statuses) > 0 { + if c.Verbose { + fmt.Fprintf(os.Stderr, "Found %d workflow(s) matching pattern\n", len(statuses)) + } + return true, nil } + if c.Verbose { + fmt.Fprintln(os.Stderr, "No workflows found matching pattern yet") + } + return false, nil +} - // In Codespaces, don't offer to trigger - provide link to Actions page instead - if isRunningInCodespace() { - addInteractiveLog.Print("Running in Codespaces, skipping run offer and showing Actions link") - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Running in GitHub Codespaces - please trigger the workflow manually from the Actions page")) - fmt.Fprintf(os.Stderr, "🔗 https://github.com/%s/actions\n", c.RepoOverride) - c.showFinalInstructions() +func (c *AddInteractiveConfig) checkStatusAndOfferRunOffer(ctx context.Context) error { + if c.checkStatusAndOfferRunShouldSkipOffer() { return nil } - // Ask if user wants to run the workflow fmt.Fprintln(os.Stderr, "") runNow := true // Default to yes @@ -117,52 +123,87 @@ func (c *AddInteractiveConfig) checkStatusAndOfferRun(ctx context.Context) error return nil } - // Run the workflow interactively (collects inputs if the workflow has them) - workflowName := c.primaryWorkflowName() - if workflowName != "" { - fmt.Fprintln(os.Stderr, "") + if err := c.checkStatusAndOfferRunRunWorkflow(ctx); err != nil { + return err + } - // Pull the merged workflow files now that we know GitHub has processed the - // merge (workflowFound is true). Doing this here—rather than immediately - // after the PR merge—avoids a race where git fetch runs before GitHub's git - // objects have been updated, which caused "workflow file not found" errors. - if !c.Verbose { - fmt.Fprintln(os.Stderr, "Updating local branch (this may take a few seconds)...") - } - if err := c.updateLocalBranch(); err != nil { - addInteractiveLog.Printf("Failed to update local branch: %v", err) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not update local branch: %v", err))) - fmt.Fprintln(os.Stderr, "You may need to switch to your repository's default branch (for example 'main') and run 'git pull' manually before running the workflow.") - } - if !c.Verbose { - fmt.Fprintln(os.Stderr, "Finished updating local branch.") - } + c.showFinalInstructions() + return nil +} - if err := RunSpecificWorkflowInteractively(ctx, RunWorkflowOptions{ - WorkflowName: workflowName, - Verbose: c.Verbose, - EngineOverride: c.EngineOverride, - RepoOverride: c.RepoOverride, - }); err != nil { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Failed to run workflow: %v", err))) - c.showFinalInstructions() - return nil - } +func (c *AddInteractiveConfig) checkStatusAndOfferRunShouldSkipOffer() bool { + // Only offer to run if workflow has workflow_dispatch trigger + if c.addResult == nil || !c.addResult.HasWorkflowDispatch { + addInteractiveLog.Print("Workflow does not have workflow_dispatch trigger, skipping run offer") + c.showFinalInstructions() + return true + } - // Get the run URL for step 10 - runInfo, err := getLatestWorkflowRunWithRetry(workflowName+".lock.yml", c.RepoOverride, c.Verbose) - if err == nil && runInfo.URL != "" { - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Workflow triggered successfully!")) - fmt.Fprintln(os.Stderr, "") - fmt.Fprintf(os.Stderr, "🔗 View workflow run: %s\n", runInfo.URL) - } + // In Codespaces, don't offer to trigger - provide link to Actions page instead + if isRunningInCodespace() { + addInteractiveLog.Print("Running in Codespaces, skipping run offer and showing Actions link") + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Running in GitHub Codespaces - please trigger the workflow manually from the Actions page")) + fmt.Fprintf(os.Stderr, "🔗 https://github.com/%s/actions\n", c.RepoOverride) + c.showFinalInstructions() + return true } + return false +} - c.showFinalInstructions() +func (c *AddInteractiveConfig) checkStatusAndOfferRunRunWorkflow(ctx context.Context) error { + // Run the workflow interactively (collects inputs if the workflow has them) + workflowName := c.primaryWorkflowName() + if workflowName == "" { + return nil + } + fmt.Fprintln(os.Stderr, "") + c.checkStatusAndOfferRunUpdateLocalBranch() + + if err := RunSpecificWorkflowInteractively(ctx, RunWorkflowOptions{ + WorkflowName: workflowName, + Verbose: c.Verbose, + EngineOverride: c.EngineOverride, + RepoOverride: c.RepoOverride, + }); err != nil { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Failed to run workflow: %v", err))) + c.showFinalInstructions() + return nil + } + + c.checkStatusAndOfferRunPrintRunURL(workflowName) return nil } +func (c *AddInteractiveConfig) checkStatusAndOfferRunUpdateLocalBranch() { + // Pull the merged workflow files now that we know GitHub has processed the + // merge (workflowFound is true). Doing this here—rather than immediately + // after the PR merge—avoids a race where git fetch runs before GitHub's git + // objects have been updated, which caused "workflow file not found" errors. + if !c.Verbose { + fmt.Fprintln(os.Stderr, "Updating local branch (this may take a few seconds)...") + } + if err := c.updateLocalBranch(); err != nil { + addInteractiveLog.Printf("Failed to update local branch: %v", err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not update local branch: %v", err))) + fmt.Fprintln(os.Stderr, "You may need to switch to your repository's default branch (for example 'main') and run 'git pull' manually before running the workflow.") + } + if !c.Verbose { + fmt.Fprintln(os.Stderr, "Finished updating local branch.") + } +} + +func (c *AddInteractiveConfig) checkStatusAndOfferRunPrintRunURL(workflowName string) { + // Get the run URL for step 10 + runInfo, err := getLatestWorkflowRunWithRetry(workflowName+".lock.yml", c.RepoOverride, c.Verbose) + if err == nil && runInfo.URL != "" { + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Workflow triggered successfully!")) + fmt.Fprintln(os.Stderr, "") + fmt.Fprintf(os.Stderr, "🔗 View workflow run: %s\n", runInfo.URL) + } +} + // findWorkflowsByFilenamePattern is a helper to find workflows registered in GitHub by filename pattern. // The pattern is matched against the workflow filename (basename without extension) func findWorkflowsByFilenamePattern(pattern, repoOverride string, verbose bool) ([]WorkflowStatus, error) { diff --git a/pkg/cli/add_package_manifest.go b/pkg/cli/add_package_manifest.go index cde8cc6b42c..e68c4d13d78 100644 --- a/pkg/cli/add_package_manifest.go +++ b/pkg/cli/add_package_manifest.go @@ -84,25 +84,7 @@ func resolveRepositoryPackage(ctx context.Context, repoSpec *RepoSpec, host stri owner := parts[0] repo := parts[1] - // At manifest-fetch time there is no resolved package metadata yet. - ref := repositoryPackageEffectiveRef(repoSpec, nil) - if ref == "" { - if isGhAwRepository(repoSpec.RepoSlug) { - if latestRelease, err := getRepositoryPackageLatestRelease(repoSpec.RepoSlug, host); err == nil { - ref = latestRelease - } else { - addPackageManifestLog.Printf("failed to resolve latest release for %s (host=%q): %v", repoSpec.RepoSlug, host, err) - } - } - if ref == "" { - ref = "main" - if defaultBranch, err := getRepositoryPackageDefaultBranch(repoSpec.RepoSlug, host); err == nil { - ref = defaultBranch - } else { - addPackageManifestLog.Printf("failed to resolve default branch for %s (host=%q), falling back to %q: %v", repoSpec.RepoSlug, host, ref, err) - } - } - } + ref := resolveRepositoryPackageRef(repoSpec, host) packagePath := strings.Trim(repoSpec.PackagePath, "/") manifestPath, manifestContent, err := loadRepositoryPackageManifestFile(ctx, owner, repo, packagePath, ref, host) @@ -115,42 +97,21 @@ func resolveRepositoryPackage(ctx context.Context, repoSpec *RepoSpec, host stri return nil, err } - includeInstallablePaths, includeSkillDirs, includeAgentFiles := splitManifestIncludePaths(manifest.Includes) - includeInstallablePaths = append(includeInstallablePaths, manifest.Files...) - - installationSources := normalizePackageInstallablePaths(includeInstallablePaths, packagePath) - if len(installationSources) == 0 { - installationSources, err = scanRepositoryPackageInstallablePaths(ctx, owner, repo, packagePath, ref, host) - if err != nil { - return nil, err - } - } - if err := validateUniqueManifestWorkflowFilenames(installationSources, manifestPath); err != nil { - return nil, err - } - - docsPath, err := resolveRepositoryPackageDocsPath(ctx, owner, repo, packagePath, ref, host) + installationSources, includeSkillDirs, includeAgentFiles, err := resolveRepositoryPackageInstallablePaths(ctx, owner, repo, packagePath, ref, host, manifest) if err != nil { return nil, err } - // Resolve skill files: explicit from manifest or auto-scanned. - explicitSkillDirs := append([]string{}, manifest.Skills...) - explicitSkillDirs = append(explicitSkillDirs, includeSkillDirs...) - skillFiles, skillWarnings, err := resolvePackageSkillFiles(ctx, owner, repo, packagePath, ref, host, explicitSkillDirs) + docsPath, err := resolveRepositoryPackageDocsPath(ctx, owner, repo, packagePath, ref, host) if err != nil { return nil, err } - warnings = append(warnings, skillWarnings...) - // Resolve agent files: explicit from manifest or auto-scanned. - explicitAgentFiles := append([]string{}, manifest.Agents...) - explicitAgentFiles = append(explicitAgentFiles, includeAgentFiles...) - agentFiles, agentWarnings, err := resolvePackageAgentFiles(ctx, owner, repo, packagePath, ref, host, explicitAgentFiles) + skillFiles, agentFiles, packageWarnings, err := resolveRepositoryPackageExtras(resolveRepositoryPackageExtrasParams{Ctx: ctx, Owner: owner, Repo: repo, PackagePath: packagePath, Ref: ref, Host: host, Manifest: manifest, IncludeSkillDirs: includeSkillDirs, IncludeAgentFiles: includeAgentFiles}) if err != nil { return nil, err } - warnings = append(warnings, agentWarnings...) + warnings = append(warnings, packageWarnings...) if len(installationSources) == 0 && len(skillFiles) == 0 && len(agentFiles) == 0 { return nil, fmt.Errorf("repository %q does not contain any installable workflows, skills, or agents (either explicitly declared or auto-discovered)", repositoryPackageIdentifier(repoSpec.RepoSlug, packagePath)) @@ -172,6 +133,77 @@ func resolveRepositoryPackage(ctx context.Context, repoSpec *RepoSpec, host stri }, nil } +func resolveRepositoryPackageRef(repoSpec *RepoSpec, host string) string { + // At manifest-fetch time there is no resolved package metadata yet. + ref := repositoryPackageEffectiveRef(repoSpec, nil) + if ref != "" { + return ref + } + if isGhAwRepository(repoSpec.RepoSlug) { + if latestRelease, err := getRepositoryPackageLatestRelease(repoSpec.RepoSlug, host); err == nil { + return latestRelease + } else { + addPackageManifestLog.Printf("failed to resolve latest release for %s (host=%q): %v", repoSpec.RepoSlug, host, err) + } + } + ref = "main" + if defaultBranch, err := getRepositoryPackageDefaultBranch(repoSpec.RepoSlug, host); err == nil { + ref = defaultBranch + } else { + addPackageManifestLog.Printf("failed to resolve default branch for %s (host=%q), falling back to %q: %v", repoSpec.RepoSlug, host, ref, err) + } + return ref +} + +func resolveRepositoryPackageInstallablePaths(ctx context.Context, owner, repo, packagePath, ref, host string, manifest *repositoryPackageManifest) ([]string, []string, []string, error) { + includeInstallablePaths, includeSkillDirs, includeAgentFiles := splitManifestIncludePaths(manifest.Includes) + includeInstallablePaths = append(includeInstallablePaths, manifest.Files...) + + installationSources := normalizePackageInstallablePaths(includeInstallablePaths, packagePath) + if len(installationSources) == 0 { + var err error + installationSources, err = scanRepositoryPackageInstallablePaths(ctx, owner, repo, packagePath, ref, host) + if err != nil { + return nil, nil, nil, err + } + } + if err := validateUniqueManifestWorkflowFilenames(installationSources, joinRepositoryPackagePath(packagePath, repositoryPackageManifestFileName)); err != nil { + return nil, nil, nil, err + } + return installationSources, includeSkillDirs, includeAgentFiles, nil +} + +type resolveRepositoryPackageExtrasParams struct { + Ctx context.Context + Owner string + Repo string + PackagePath string + Ref string + Host string + Manifest *repositoryPackageManifest + IncludeSkillDirs []string + IncludeAgentFiles []string +} + +func resolveRepositoryPackageExtras(p resolveRepositoryPackageExtrasParams) ([]resolvedPackageSkillFile, []string, []string, error) { + // Resolve skill files: explicit from manifest or auto-scanned. + explicitSkillDirs := append([]string{}, p.Manifest.Skills...) + explicitSkillDirs = append(explicitSkillDirs, p.IncludeSkillDirs...) + skillFiles, skillWarnings, err := resolvePackageSkillFiles(p.Ctx, p.Owner, p.Repo, p.PackagePath, p.Ref, p.Host, explicitSkillDirs) + if err != nil { + return nil, nil, nil, err + } + + // Resolve agent files: explicit from manifest or auto-scanned. + explicitAgentFiles := append([]string{}, p.Manifest.Agents...) + explicitAgentFiles = append(explicitAgentFiles, p.IncludeAgentFiles...) + agentFiles, agentWarnings, err := resolvePackageAgentFiles(p.Ctx, p.Owner, p.Repo, p.PackagePath, p.Ref, p.Host, explicitAgentFiles) + if err != nil { + return nil, nil, nil, err + } + return skillFiles, agentFiles, append(skillWarnings, agentWarnings...), nil +} + func loadRepositoryPackageManifestFile(ctx context.Context, owner, repo, packagePath, ref, host string) (string, []byte, error) { manifestPath := joinRepositoryPackagePath(packagePath, repositoryPackageManifestFileName) repoSlug := owner + "/" + repo @@ -231,6 +263,22 @@ func parseRepositoryPackageManifest(manifestPath string, content []byte) (*repos } var warnings []string + if err := parseRepositoryPackageManifestVersions(root, manifestPath, manifest); err != nil { + return nil, nil, err + } + warnings = append(warnings, parseRepositoryPackageManifestMetadata(root, manifestPath, manifest)...) + warnings = append(warnings, parseRepositoryPackageManifestIncludes(root, manifestPath, manifest)...) + warnings = append(warnings, parseRepositoryPackageManifestAssets(root, manifestPath, manifest)...) + configWarnings, err := parseRepositoryPackageManifestConfig(root, manifestPath, manifest) + if err != nil { + return nil, nil, err + } + warnings = append(warnings, configWarnings...) + + return manifest, warnings, nil +} + +func parseRepositoryPackageManifestVersions(root map[string]any, manifestPath string, manifest *repositoryPackageManifest) error { if manifestVersion, ok := stringValue(root["manifest-version"]); ok { manifest.ManifestVersion = strings.TrimSpace(manifestVersion) } else { @@ -240,39 +288,44 @@ func parseRepositoryPackageManifest(manifestPath string, content []byte) (*repos if minVersion, ok := stringValue(root["min-version"]); ok { manifest.MinVersion = strings.TrimSpace(minVersion) if !isSupportedManifestMinVersion(manifest.MinVersion) { - return nil, nil, fmt.Errorf("invalid Agentic Workflow manifest %q: min-version must use vMAJOR.minor.patch, got %q", manifestPath, minVersion) + return fmt.Errorf("invalid Agentic Workflow manifest %q: min-version must use vMAJOR.minor.patch, got %q", manifestPath, minVersion) } currentVersion := GetVersion() if !semverutil.IsValid(currentVersion) { - return nil, nil, fmt.Errorf("invalid Agentic Workflow manifest %q: min-version validation requires a semantic-versioned compiler, but the current compiler version %q is not a valid semantic version (this indicates a build issue)", manifestPath, currentVersion) + return fmt.Errorf("invalid Agentic Workflow manifest %q: min-version validation requires a semantic-versioned compiler, but the current compiler version %q is not a valid semantic version (this indicates a build issue)", manifestPath, currentVersion) } currentVersion = semverutil.NormalizeGitDescribeSemver(currentVersion) if semverutil.Compare(currentVersion, manifest.MinVersion) < 0 { - return nil, nil, fmt.Errorf("invalid Agentic Workflow manifest %q: min-version %q requires gh-aw %s or newer (current: %s)", manifestPath, manifest.MinVersion, manifest.MinVersion, currentVersion) + return fmt.Errorf("invalid Agentic Workflow manifest %q: min-version %q requires gh-aw %s or newer (current: %s)", manifestPath, manifest.MinVersion, manifest.MinVersion, currentVersion) } } + return nil +} +func parseRepositoryPackageManifestMetadata(root map[string]any, manifestPath string, manifest *repositoryPackageManifest) []string { + var warnings []string if description, ok := stringValue(root["description"]); ok { manifest.Description = description if len(description) > 255 { warnings = append(warnings, fmt.Sprintf("Manifest %s description exceeds the 255-character marketplace display limit", manifestPath)) } } - if emoji, ok := stringValue(root["emoji"]); ok { manifest.Emoji = emoji } - if license, ok := stringValue(root["license"]); ok { manifest.License = license } + return warnings +} +func parseRepositoryPackageManifestIncludes(root map[string]any, manifestPath string, manifest *repositoryPackageManifest) []string { + var warnings []string if includesValue, ok := root["includes"]; ok { includes, includeWarnings := extractManifestIncludes(includesValue, manifestPath) manifest.Includes = includes warnings = append(warnings, includeWarnings...) } - if filesValue, ok := root["files"]; ok { files, fileWarnings := extractManifestFiles(filesValue, manifestPath) manifest.Files = files @@ -282,29 +335,34 @@ func parseRepositoryPackageManifest(manifestPath string, content []byte) (*repos warnings = append(warnings, "Codemod suggestion:\n"+formatIncludesCodemodSuggestion(codemodManifestFilesToIncludes(files))) } } + return warnings +} +func parseRepositoryPackageManifestAssets(root map[string]any, manifestPath string, manifest *repositoryPackageManifest) []string { + var warnings []string if skillsValue, ok := root["skills"]; ok { skills, skillWarnings := extractManifestSkillDirs(skillsValue, manifestPath) manifest.Skills = skills warnings = append(warnings, skillWarnings...) } - if agentsValue, ok := root["agents"]; ok { agents, agentWarnings := extractManifestAgentFiles(agentsValue, manifestPath) manifest.Agents = agents warnings = append(warnings, agentWarnings...) } + return warnings +} +func parseRepositoryPackageManifestConfig(root map[string]any, manifestPath string, manifest *repositoryPackageManifest) ([]string, error) { if configValue, ok := root["config"]; ok { - warnings = append(warnings, "Using experimental feature: config") bootstrap, err := extractManifestConfig(configValue, manifestPath) if err != nil { - return nil, nil, err + return nil, err } manifest.Bootstrap = bootstrap + return []string{"Using experimental feature: config"}, nil } - - return manifest, warnings, nil + return nil, nil } func extractManifestIncludes(value any, manifestPath string) ([]string, []string) { @@ -542,9 +600,6 @@ func agentDirectoryRoot(cleaned string) string { // contain a SKILL.md file but are not already covered by the manifest. Each skill folder // is traversed recursively so that all nested files are included. func resolvePackageSkillFiles(ctx context.Context, owner, repo, packagePath, ref, host string, explicitSkillDirs []string) ([]resolvedPackageSkillFile, []string, error) { - // seenSkillDirs tracks full skill directories already added so that auto-scanned - // duplicates of manifest-specified skills are not added a second time. - seenSkillDirs := make(map[string]struct{}) var warnings []string // Step 1: resolve manifest skills first (explicit dirs). @@ -565,7 +620,17 @@ func resolvePackageSkillFiles(ctx context.Context, owner, repo, packagePath, ref } } - // Build the final ordered list: manifest skills first, then auto-scanned extras. + skillDirs := resolvePackageSkillFilesOrderedDirs(manifestSkillDirs, autoScanned) + skillFiles, fileWarnings, err := resolvePackageSkillFilesList(ctx, owner, repo, ref, host, manifestSkillDirs, skillDirs) + if err != nil { + return nil, nil, err + } + warnings = append(warnings, fileWarnings...) + return skillFiles, warnings, nil +} + +func resolvePackageSkillFilesOrderedDirs(manifestSkillDirs, autoScanned []string) []string { + seenSkillDirs := make(map[string]struct{}) var skillDirs []string appendIfNew := func(dir string) { if _, exists := seenSkillDirs[dir]; !exists { @@ -579,48 +644,68 @@ func resolvePackageSkillFiles(ctx context.Context, owner, repo, packagePath, ref for _, dir := range autoScanned { appendIfNew(dir) } + return skillDirs +} - // manifestSkillDirSet is used to know which dirs require a SKILL.md marker check. +func resolvePackageSkillFilesList(ctx context.Context, owner, repo, ref, host string, manifestSkillDirs, skillDirs []string) ([]resolvedPackageSkillFile, []string, error) { manifestSkillDirSet := make(map[string]struct{}, len(manifestSkillDirs)) for _, d := range manifestSkillDirs { manifestSkillDirSet[d] = struct{}{} } var skillFiles []resolvedPackageSkillFile + var warnings []string for _, skillDir := range skillDirs { - // For skills that came from the manifest, validate that the SKILL.md marker - // exists so that typos in the manifest surface as clear warnings. - if _, fromManifest := manifestSkillDirSet[skillDir]; fromManifest { - markerPath := joinRepositoryPackagePath(skillDir, packageSkillMarkerFile) - if _, err := downloadPackageFileFromGitHubForHost(ctx, owner, repo, markerPath, ref, host); err != nil { - if isRepositoryFileNotFound(err) { - warnings = append(warnings, fmt.Sprintf("Skill directory %q is missing required %s marker file", skillDir, packageSkillMarkerFile)) - continue - } - return nil, nil, fmt.Errorf("failed to validate skill marker %q: %w", markerPath, err) - } + ok, warning, err := resolvePackageSkillFilesValidateMarker(ctx, owner, repo, ref, host, manifestSkillDirSet, skillDir) + if err != nil { + return nil, nil, err } - skillName := filepath.Base(skillDir) - // Use recursive listing so that the entire skill folder (including any - // subdirectories) is copied, not just the top-level files. - files, err := listPackageDirFilesRecursivelyForHost(ctx, owner, repo, ref, skillDir, host) + if warning != "" { + warnings = append(warnings, warning) + } + if !ok { + continue + } + files, err := resolvePackageSkillFilesListDir(ctx, owner, repo, ref, host, skillDir) if err != nil { if isRepositoryFileNotFound(err) { warnings = append(warnings, fmt.Sprintf("Skill directory %q not found in package, skipping", skillDir)) continue } - return nil, nil, fmt.Errorf("failed to list files in skill directory %q: %w", skillDir, err) + return nil, nil, err } + skillName := filepath.Base(skillDir) for _, file := range files { - skillFiles = append(skillFiles, resolvedPackageSkillFile{ - SourcePath: file, - SkillName: skillName, - }) + skillFiles = append(skillFiles, resolvedPackageSkillFile{SourcePath: file, SkillName: skillName}) } } return skillFiles, warnings, nil } +func resolvePackageSkillFilesValidateMarker(ctx context.Context, owner, repo, ref, host string, manifestSkillDirSet map[string]struct{}, skillDir string) (bool, string, error) { + if _, fromManifest := manifestSkillDirSet[skillDir]; !fromManifest { + return true, "", nil + } + markerPath := joinRepositoryPackagePath(skillDir, packageSkillMarkerFile) + if _, err := downloadPackageFileFromGitHubForHost(ctx, owner, repo, markerPath, ref, host); err != nil { + if isRepositoryFileNotFound(err) { + return false, fmt.Sprintf("Skill directory %q is missing required %s marker file", skillDir, packageSkillMarkerFile), nil + } + return false, "", fmt.Errorf("failed to validate skill marker %q: %w", markerPath, err) + } + return true, "", nil +} + +func resolvePackageSkillFilesListDir(ctx context.Context, owner, repo, ref, host, skillDir string) ([]string, error) { + // Use recursive listing so that the entire skill folder (including any + // subdirectories) is copied, not just the top-level files. + files, err := listPackageDirFilesRecursivelyForHost(ctx, owner, repo, ref, skillDir, host) + if err != nil { + return nil, fmt.Errorf("failed to list files in skill directory %q: %w", skillDir, err) + } + return files, nil +} + // resolvePackageAgentFiles returns the list of agent file source paths for a package. // If explicitAgentFiles is non-empty it is used; otherwise the agents/ directory is // auto-scanned for .md files. diff --git a/pkg/cli/add_wizard_command.go b/pkg/cli/add_wizard_command.go index f3ddaabaab1..cf8572639e1 100644 --- a/pkg/cli/add_wizard_command.go +++ b/pkg/cli/add_wizard_command.go @@ -11,12 +11,7 @@ import ( var addWizardLog = logger.New("cli:add_wizard_command") -// NewAddWizardCommand creates the add-wizard command, which is always interactive. -func NewAddWizardCommand(validateEngine func(string) error) *cobra.Command { - cmd := &cobra.Command{ - Use: "add-wizard ...", - Short: "Interactively add one or more agentic workflows with guided setup", - Long: `Interactively add one or more agentic workflows with guided setup. +const addWizardCommandLong = `Interactively add one or more agentic workflows with guided setup. This command walks you through: - Selecting an AI engine (Copilot, Claude, Codex, or Gemini) @@ -42,8 +37,9 @@ Note: Requires an interactive terminal. Use 'add' for CI/automation environments Note: In GitHub Enterprise repos, shorthand source specs resolve on your enterprise host by default. For github/*, githubnext/*, and microsoft/* sources, shorthand resolves on github.com. Use full https://github.com/... source URLs for other public github.com workflows. -Note: To create a new workflow from scratch, use the 'new' command instead.`, - Example: ` ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics # Guided setup for repository-root aw.yml package +Note: To create a new workflow from scratch, use the 'new' command instead.` + +const addWizardCommandExample = ` ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics # Guided setup for repository-root aw.yml package ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/packages/repo-assist # Guided setup for nested aw.yml package ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/daily-repo-status # Guided setup ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/ci-doctor@v1.0.0 # Guided setup with version @@ -54,7 +50,15 @@ Note: To create a new workflow from scratch, use the 'new' command instead.`, ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/ci-doctor --no-secret # Skip secret prompt ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/ci-doctor --append "custom footer" # Append custom content ` + string(constants.CLIExtensionPrefix) + ` add-wizard githubnext/agentics/ci-doctor --no-security-scanner # Skip security scan -`, +` + +// NewAddWizardCommand creates the add-wizard command, which is always interactive. +func NewAddWizardCommand(validateEngine func(string) error) *cobra.Command { + cmd := &cobra.Command{ + Use: "add-wizard ...", + Short: "Interactively add one or more agentic workflows with guided setup", + Long: addWizardCommandLong, + Example: addWizardCommandExample, Args: func(cmd *cobra.Command, args []string) error { if len(args) < 1 { return errors.New("missing workflow specification\n\nRun 'gh aw add-wizard --help' for usage information") @@ -62,77 +66,76 @@ Note: To create a new workflow from scratch, use the 'new' command instead.`, return nil }, RunE: func(cmd *cobra.Command, args []string) error { - workflows := args - engineOverride, _ := cmd.Flags().GetString("engine") - verbose, _ := cmd.Flags().GetBool("verbose") - noGitattributes, _ := cmd.Flags().GetBool("no-gitattributes") - workflowDir, _ := cmd.Flags().GetString("dir") - noStopAfter, _ := cmd.Flags().GetBool("no-stop-after") - stopAfter, _ := cmd.Flags().GetString("stop-after") - noSecret, _ := cmd.Flags().GetBool("no-secret") - skipSecretLegacy, _ := cmd.Flags().GetBool("skip-secret") - skipSecret := noSecret || skipSecretLegacy - appendText, _ := cmd.Flags().GetString("append") - disableSecurityScanner, _ := cmd.Flags().GetBool("no-security-scanner") - - addWizardLog.Printf("Starting add-wizard: workflows=%v, engine=%s, verbose=%v", workflows, engineOverride, verbose) - - if err := validateEngine(engineOverride); err != nil { - return err - } + return newAddWizardCommandRunE(cmd, args, validateEngine) + }, + } - // add-wizard requires an interactive terminal - isTerminal := tty.IsStdoutTerminal() - isCIEnv := IsRunningInCI() - addWizardLog.Printf("Terminal check: is_terminal=%v, is_ci=%v", isTerminal, isCIEnv) - if !isTerminal || isCIEnv { - return errors.New("add-wizard requires an interactive terminal; use 'add' for non-interactive environments") - } + newAddWizardCommandFlags(cmd) - return RunAddInteractive(cmd.Context(), &AddInteractiveConfig{ - WorkflowSpecs: workflows, - Verbose: verbose, - EngineOverride: engineOverride, - NoGitattributes: noGitattributes, - WorkflowDir: workflowDir, - NoStopAfter: noStopAfter, - StopAfter: stopAfter, - SkipSecret: skipSecret, - AppendText: appendText, - DisableSecurityScanner: disableSecurityScanner, - }) - }, + return cmd +} + +func newAddWizardCommandRunE(cmd *cobra.Command, args []string, validateEngine func(string) error) error { + config := newAddWizardCommandConfig(cmd, args) + addWizardLog.Printf("Starting add-wizard: workflows=%v, engine=%s, verbose=%v", config.WorkflowSpecs, config.EngineOverride, config.Verbose) + + if err := validateEngine(config.EngineOverride); err != nil { + return err + } + if err := newAddWizardCommandRequireInteractive(); err != nil { + return err } + return RunAddInteractive(cmd.Context(), config) +} - // Add AI engine flag - addEngineFlag(cmd) +func newAddWizardCommandConfig(cmd *cobra.Command, workflows []string) *AddInteractiveConfig { + engineOverride, _ := cmd.Flags().GetString("engine") + verbose, _ := cmd.Flags().GetBool("verbose") + noGitattributes, _ := cmd.Flags().GetBool("no-gitattributes") + workflowDir, _ := cmd.Flags().GetString("dir") + noStopAfter, _ := cmd.Flags().GetBool("no-stop-after") + stopAfter, _ := cmd.Flags().GetString("stop-after") + noSecret, _ := cmd.Flags().GetBool("no-secret") + skipSecretLegacy, _ := cmd.Flags().GetBool("skip-secret") + appendText, _ := cmd.Flags().GetString("append") + disableSecurityScanner, _ := cmd.Flags().GetBool("no-security-scanner") + + return &AddInteractiveConfig{ + WorkflowSpecs: workflows, + Verbose: verbose, + EngineOverride: engineOverride, + NoGitattributes: noGitattributes, + WorkflowDir: workflowDir, + NoStopAfter: noStopAfter, + StopAfter: stopAfter, + SkipSecret: noSecret || skipSecretLegacy, + AppendText: appendText, + DisableSecurityScanner: disableSecurityScanner, + } +} - // Add no-gitattributes flag - cmd.Flags().Bool("no-gitattributes", false, "Skip updating .gitattributes file") +func newAddWizardCommandRequireInteractive() error { + // add-wizard requires an interactive terminal + isTerminal := tty.IsStdoutTerminal() + isCIEnv := IsRunningInCI() + addWizardLog.Printf("Terminal check: is_terminal=%v, is_ci=%v", isTerminal, isCIEnv) + if !isTerminal || isCIEnv { + return errors.New("add-wizard requires an interactive terminal; use 'add' for non-interactive environments") + } + return nil +} - // Add workflow directory flag +func newAddWizardCommandFlags(cmd *cobra.Command) { + addEngineFlag(cmd) + cmd.Flags().Bool("no-gitattributes", false, "Skip updating .gitattributes file") cmd.Flags().StringP("dir", "d", "", "Workflow directory (default: $GH_AW_WORKFLOWS_DIR or .github/workflows)") - - // Add no-stop-after flag cmd.Flags().Bool("no-stop-after", false, "Remove any stop-after field from the workflow") - - // Add stop-after flag cmd.Flags().String("stop-after", "", "Override stop-after value in the workflow (e.g., '+48h', '2025-12-31 23:59:59')") - - // Add no-secret flag (--skip-secret is kept as an undocumented alias) cmd.Flags().Bool("no-secret", false, "Skip the API secret prompt (use when the secret is already set at the org or repo level)") cmd.Flags().Bool("skip-secret", false, "Skip the API secret prompt (use when the secret is already set at the org or repo level)") _ = cmd.Flags().MarkHidden("skip-secret") - - // Add append flag (matches --append in add command) cmd.Flags().String("append", "", "Append extra content to the end of the agentic workflow on installation") - - // Add no-security-scanner flag (matches --no-security-scanner in add command) cmd.Flags().Bool("no-security-scanner", false, "Skip security scanning of workflow markdown content") - - // Register completions RegisterEngineFlagCompletion(cmd) RegisterDirFlagCompletion(cmd, "dir") - - return cmd } diff --git a/pkg/cli/add_workflow_pr.go b/pkg/cli/add_workflow_pr.go index 7d80ed32dab..0de364ce90b 100644 --- a/pkg/cli/add_workflow_pr.go +++ b/pkg/cli/add_workflow_pr.go @@ -53,30 +53,11 @@ func sanitizeBranchName(name string) string { func addWorkflowsWithPR(ctx context.Context, workflows []*ResolvedWorkflow, opts AddOptions) (int, string, error) { addWorkflowPRLog.Printf("Adding %d workflow(s) with PR creation (resolved)", len(workflows)) - // Get current branch for restoration later - currentBranch, err := getCurrentBranch() + currentBranch, branchName, tracker, err := addWorkflowsWithPRPrepare(workflows, opts) if err != nil { - addWorkflowPRLog.Printf("Failed to get current branch: %v", err) - return 0, "", fmt.Errorf("failed to get current branch: %w", err) - } - - addWorkflowPRLog.Printf("Current branch: %s", currentBranch) - - // Create temporary branch with random 4-digit number - // Use sanitized workflow name to avoid invalid git ref characters - randomNum := rand.Intn(9000) + 1000 // Generate number between 1000-9999 - sanitizedName := sanitizeBranchName(workflows[0].Spec.WorkflowPath) - branchName := fmt.Sprintf("add-workflow-%s-%04d", sanitizedName, randomNum) - - addWorkflowPRLog.Printf("Creating temporary branch: %s", branchName) - - if err := createAndSwitchBranch(branchName, opts.Verbose); err != nil { - return 0, "", fmt.Errorf("failed to create branch %s: %w", branchName, err) + return 0, "", err } - // Create file tracker for rollback capability - tracker := NewFileTracker() - // Ensure we switch back to original branch on exit defer func() { if switchErr := switchBranch(currentBranch, opts.Verbose); switchErr != nil && opts.Verbose { @@ -92,57 +73,37 @@ func addWorkflowsWithPR(ctx context.Context, workflows []*ResolvedWorkflow, opts return 0, "", fmt.Errorf("failed to add workflows: %w", err) } - // Stage all files before creating PR - addWorkflowPRLog.Print("Staging workflow files") - if err := tracker.StageAllFiles(opts.Verbose); err != nil { - if rollbackErr := tracker.RollbackAllFiles(opts.Verbose); rollbackErr != nil && opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to rollback files: %v", rollbackErr))) - } - return 0, "", fmt.Errorf("failed to stage workflow files: %w", err) + if err := addWorkflowsWithPRStage(tracker, opts); err != nil { + return 0, "", err } - // Update .gitattributes and stage it if changed - if err := stageGitAttributesIfChanged(); err != nil && opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to stage .gitattributes: %v", err))) + commitMessage, prTitle, prBody := addWorkflowsWithPRMessages(workflows) + + if err := commitChanges(commitMessage, opts.Verbose); err != nil { + return 0, "", addWorkflowsWithPRCommitError(err, commitMessage, branchName, prTitle) } - // Commit changes - var commitMessage, prTitle, prBody, joinedNames string - if len(workflows) == 1 { - joinedNames = workflows[0].Spec.WorkflowName - commitMessage = "Add agentic workflow " + joinedNames - prTitle = "Add agentic workflow " + joinedNames - prBody = "Add agentic workflow " + joinedNames - } else { - workflowNames := sliceutil.Map(workflows, func(wf *ResolvedWorkflow) string { - return wf.Spec.WorkflowName - }) - joinedNames = strings.Join(workflowNames, ", ") - commitMessage = "Add agentic workflows: " + joinedNames - prTitle = "Add agentic workflows: " + joinedNames - prBody = "Add agentic workflows: " + joinedNames + if err := addWorkflowsWithPRPushBranch(branchName, prTitle, opts); err != nil { + return 0, "", err } - if err := commitChanges(commitMessage, opts.Verbose); err != nil { - // Don't rollback - leave the workflow files on disk for manual recovery. - // Return a richly formatted error with clear instructions so the user can - // commit and push manually. The top-level error handler will print this. - return 0, "", fmt.Errorf( - "failed to commit workflow files: %w\n\n"+ - "The workflow files have been written to disk and staged in git.\n"+ - "Please commit the files manually, then either push them to the\n"+ - "repository or create a pull request:\n\n"+ - " git commit -m %q\n"+ - " git push\n\n"+ - "Or to create a pull request:\n\n"+ - " git checkout -b %s\n"+ - " git commit -m %q\n"+ - " git push -u origin %s\n"+ - " gh pr create --title %q", - err, commitMessage, branchName, commitMessage, branchName, prTitle, - ) + prNumber, prURL, err := addWorkflowsWithPRCreatePR(branchName, prTitle, prBody, tracker, opts) + if err != nil { + return 0, "", err } + addWorkflowPRLog.Printf("Successfully created PR #%d: %s", prNumber, prURL) + + // Switch back to original branch + if err := switchBranch(currentBranch, opts.Verbose); err != nil { + return prNumber, prURL, fmt.Errorf("failed to switch back to branch %s: %w", currentBranch, err) + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Created pull request "+prURL)) + return prNumber, prURL, nil +} + +func addWorkflowsWithPRPushBranch(branchName, prTitle string, opts AddOptions) error { // Push branch addWorkflowPRLog.Printf("Pushing branch %s to remote", branchName) if err := pushBranch(branchName, opts.Verbose); err != nil { @@ -156,9 +117,12 @@ func addWorkflowsWithPR(ctx context.Context, workflows []*ResolvedWorkflow, opts " git push -u origin "+branchName+"\n"+ " gh pr create --title "+fmt.Sprintf("%q", prTitle), )) - return 0, "", fmt.Errorf("failed to push branch %s: %w", branchName, err) + return fmt.Errorf("failed to push branch %s: %w", branchName, err) } + return nil +} +func addWorkflowsWithPRCreatePR(branchName, prTitle, prBody string, tracker *FileTracker, opts AddOptions) (int, string, error) { // Create PR addWorkflowPRLog.Printf("Creating pull request: %s", prTitle) prNumber, prURL, err := createPR(branchName, prTitle, prBody, opts.Verbose) @@ -169,14 +133,77 @@ func addWorkflowsWithPR(ctx context.Context, workflows []*ResolvedWorkflow, opts } return 0, "", fmt.Errorf("failed to create PR: %w", err) } + return prNumber, prURL, nil +} - addWorkflowPRLog.Printf("Successfully created PR #%d: %s", prNumber, prURL) +func addWorkflowsWithPRPrepare(workflows []*ResolvedWorkflow, opts AddOptions) (string, string, *FileTracker, error) { + // Get current branch for restoration later + currentBranch, err := getCurrentBranch() + if err != nil { + addWorkflowPRLog.Printf("Failed to get current branch: %v", err) + return "", "", nil, fmt.Errorf("failed to get current branch: %w", err) + } + addWorkflowPRLog.Printf("Current branch: %s", currentBranch) - // Switch back to original branch - if err := switchBranch(currentBranch, opts.Verbose); err != nil { - return prNumber, prURL, fmt.Errorf("failed to switch back to branch %s: %w", currentBranch, err) + // Create temporary branch with random 4-digit number + // Use sanitized workflow name to avoid invalid git ref characters + randomNum := rand.Intn(9000) + 1000 // Generate number between 1000-9999 + sanitizedName := sanitizeBranchName(workflows[0].Spec.WorkflowPath) + branchName := fmt.Sprintf("add-workflow-%s-%04d", sanitizedName, randomNum) + + addWorkflowPRLog.Printf("Creating temporary branch: %s", branchName) + if err := createAndSwitchBranch(branchName, opts.Verbose); err != nil { + return "", "", nil, fmt.Errorf("failed to create branch %s: %w", branchName, err) } + return currentBranch, branchName, NewFileTracker(), nil +} - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Created pull request "+prURL)) - return prNumber, prURL, nil +func addWorkflowsWithPRStage(tracker *FileTracker, opts AddOptions) error { + // Stage all files before creating PR + addWorkflowPRLog.Print("Staging workflow files") + if err := tracker.StageAllFiles(opts.Verbose); err != nil { + if rollbackErr := tracker.RollbackAllFiles(opts.Verbose); rollbackErr != nil && opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to rollback files: %v", rollbackErr))) + } + return fmt.Errorf("failed to stage workflow files: %w", err) + } + + // Update .gitattributes and stage it if changed + if err := stageGitAttributesIfChanged(); err != nil && opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to stage .gitattributes: %v", err))) + } + return nil +} + +func addWorkflowsWithPRMessages(workflows []*ResolvedWorkflow) (string, string, string) { + var joinedNames string + if len(workflows) == 1 { + joinedNames = workflows[0].Spec.WorkflowName + return "Add agentic workflow " + joinedNames, "Add agentic workflow " + joinedNames, "Add agentic workflow " + joinedNames + } + workflowNames := sliceutil.Map(workflows, func(wf *ResolvedWorkflow) string { + return wf.Spec.WorkflowName + }) + joinedNames = strings.Join(workflowNames, ", ") + return "Add agentic workflows: " + joinedNames, "Add agentic workflows: " + joinedNames, "Add agentic workflows: " + joinedNames +} + +func addWorkflowsWithPRCommitError(err error, commitMessage, branchName, prTitle string) error { + // Don't rollback - leave the workflow files on disk for manual recovery. + // Return a richly formatted error with clear instructions so the user can + // commit and push manually. The top-level error handler will print this. + return fmt.Errorf( + "failed to commit workflow files: %w\n\n"+ + "The workflow files have been written to disk and staged in git.\n"+ + "Please commit the files manually, then either push them to the\n"+ + "repository or create a pull request:\n\n"+ + " git commit -m %q\n"+ + " git push\n\n"+ + "Or to create a pull request:\n\n"+ + " git checkout -b %s\n"+ + " git commit -m %q\n"+ + " git push -u origin %s\n"+ + " gh pr create --title %q", + err, commitMessage, branchName, commitMessage, branchName, prTitle, + ) } diff --git a/pkg/cli/add_workflow_resolution.go b/pkg/cli/add_workflow_resolution.go index 900d357f3ba..52e96e08331 100644 --- a/pkg/cli/add_workflow_resolution.go +++ b/pkg/cli/add_workflow_resolution.go @@ -462,30 +462,16 @@ func resolveLocalRepositoryPackage(source string) (*resolvedRepositoryPackage, e return nil, err } - includeInstallablePaths, includeSkillDirs, includeAgentFiles := splitManifestIncludePaths(manifest.Includes) - includeInstallablePaths = append(includeInstallablePaths, manifest.Files...) - installationSources := normalizeLocalPackageInstallablePaths(includeInstallablePaths, packageDir) - if len(installationSources) == 0 { - installationSources, err = scanLocalRepositoryPackageInstallablePaths(packageDir) - if err != nil { - return nil, err - } - } - if err := validateUniqueManifestWorkflowFilenames(installationSources, manifestPath); err != nil { - return nil, err - } - - skillFiles, skillWarnings, err := resolveLocalPackageSkillFiles(packageDir, append(append([]string{}, manifest.Skills...), includeSkillDirs...)) + installationSources, includeSkillDirs, includeAgentFiles, err := resolveLocalRepositoryPackageInstallablePaths(packageDir, manifestPath, manifest) if err != nil { return nil, err } - warnings = append(warnings, skillWarnings...) - agentFiles, agentWarnings, err := resolveLocalPackageAgentFiles(packageDir, append(append([]string{}, manifest.Agents...), includeAgentFiles...)) + skillFiles, agentFiles, packageWarnings, err := resolveLocalRepositoryPackageExtras(packageDir, manifest, includeSkillDirs, includeAgentFiles) if err != nil { return nil, err } - warnings = append(warnings, agentWarnings...) + warnings = append(warnings, packageWarnings...) if len(installationSources) == 0 && len(skillFiles) == 0 && len(agentFiles) == 0 { return nil, fmt.Errorf("repository package at %q does not contain any installable workflows, skills, or agents (either explicitly declared or auto-discovered)", packageDir) @@ -506,6 +492,35 @@ func resolveLocalRepositoryPackage(source string) (*resolvedRepositoryPackage, e }, nil } +func resolveLocalRepositoryPackageInstallablePaths(packageDir, manifestPath string, manifest *repositoryPackageManifest) ([]string, []string, []string, error) { + includeInstallablePaths, includeSkillDirs, includeAgentFiles := splitManifestIncludePaths(manifest.Includes) + includeInstallablePaths = append(includeInstallablePaths, manifest.Files...) + installationSources := normalizeLocalPackageInstallablePaths(includeInstallablePaths, packageDir) + if len(installationSources) == 0 { + var err error + installationSources, err = scanLocalRepositoryPackageInstallablePaths(packageDir) + if err != nil { + return nil, nil, nil, err + } + } + if err := validateUniqueManifestWorkflowFilenames(installationSources, manifestPath); err != nil { + return nil, nil, nil, err + } + return installationSources, includeSkillDirs, includeAgentFiles, nil +} + +func resolveLocalRepositoryPackageExtras(packageDir string, manifest *repositoryPackageManifest, includeSkillDirs, includeAgentFiles []string) ([]resolvedPackageSkillFile, []string, []string, error) { + skillFiles, skillWarnings, err := resolveLocalPackageSkillFiles(packageDir, append(append([]string{}, manifest.Skills...), includeSkillDirs...)) + if err != nil { + return nil, nil, nil, err + } + agentFiles, agentWarnings, err := resolveLocalPackageAgentFiles(packageDir, append(append([]string{}, manifest.Agents...), includeAgentFiles...)) + if err != nil { + return nil, nil, nil, err + } + return skillFiles, agentFiles, append(skillWarnings, agentWarnings...), nil +} + func localRepositoryPackageManifest(source string) (string, string, error) { resolvedPath, err := filepath.Abs(source) if err != nil { @@ -586,22 +601,9 @@ func appendLocalRepositoryPackageWorkflowSpecs(parsedSpecs []*WorkflowSpec, pkg } func resolveLocalPackageSkillFiles(packageDir string, explicitSkillDirs []string) ([]resolvedPackageSkillFile, []string, error) { - seenSkillDirs := make(map[string]struct{}) var warnings []string - var skillDirs []string - appendIfNew := func(dir string) { - cleaned := filepath.Clean(dir) - if _, exists := seenSkillDirs[cleaned]; exists { - return - } - seenSkillDirs[cleaned] = struct{}{} - skillDirs = append(skillDirs, cleaned) - } - - for _, dir := range explicitSkillDirs { - appendIfNew(filepath.Join(packageDir, filepath.FromSlash(dir))) - } + skillDirs := resolveLocalPackageSkillFilesExplicitDirs(packageDir, explicitSkillDirs) autoScanned, err := scanLocalPackageSkillDirs(packageDir) if err != nil { if len(skillDirs) == 0 { @@ -610,48 +612,97 @@ func resolveLocalPackageSkillFiles(packageDir string, explicitSkillDirs []string warnings = append(warnings, fmt.Sprintf("failed to auto-scan skills directory, proceeding with manifest skills only: %v", err)) } for _, dir := range autoScanned { - appendIfNew(dir) + skillDirs = resolveLocalPackageSkillFilesAppendDir(skillDirs, dir) + } + + skillFiles, fileWarnings, err := resolveLocalPackageSkillFilesList(packageDir, explicitSkillDirs, skillDirs) + if err != nil { + return nil, nil, err + } + warnings = append(warnings, fileWarnings...) + + return skillFiles, warnings, nil +} + +func resolveLocalPackageSkillFilesExplicitDirs(packageDir string, explicitSkillDirs []string) []string { + var skillDirs []string + for _, dir := range explicitSkillDirs { + skillDirs = resolveLocalPackageSkillFilesAppendDir(skillDirs, filepath.Join(packageDir, filepath.FromSlash(dir))) + } + return skillDirs +} + +func resolveLocalPackageSkillFilesAppendDir(skillDirs []string, dir string) []string { + cleaned := filepath.Clean(dir) + for _, existing := range skillDirs { + if existing == cleaned { + return skillDirs + } } + return append(skillDirs, cleaned) +} +func resolveLocalPackageSkillFilesList(packageDir string, explicitSkillDirs, skillDirs []string) ([]resolvedPackageSkillFile, []string, error) { manifestSkillDirSet := make(map[string]struct{}, len(explicitSkillDirs)) for _, dir := range explicitSkillDirs { manifestSkillDirSet[filepath.Clean(filepath.Join(packageDir, filepath.FromSlash(dir)))] = struct{}{} } var skillFiles []resolvedPackageSkillFile + var warnings []string for _, skillDir := range skillDirs { - if _, fromManifest := manifestSkillDirSet[skillDir]; fromManifest { - markerPath := filepath.Join(skillDir, packageSkillMarkerFile) - if _, err := os.Stat(markerPath); err != nil { - if errors.Is(err, os.ErrNotExist) { - warnings = append(warnings, fmt.Sprintf("Skill directory %q is missing required %s marker file", skillDir, packageSkillMarkerFile)) - continue - } - return nil, nil, fmt.Errorf("failed to validate skill marker %q: %w", markerPath, err) - } + ok, warning, err := resolveLocalPackageSkillFilesValidateMarker(manifestSkillDirSet, skillDir) + if err != nil { + return nil, nil, err } - skillName := filepath.Base(skillDir) - err := filepath.WalkDir(skillDir, func(currentPath string, d os.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr - } - if d.IsDir() { - return nil - } - skillFiles = append(skillFiles, resolvedPackageSkillFile{ - SourcePath: currentPath, - SkillName: skillName, - }) - return nil - }) + if warning != "" { + warnings = append(warnings, warning) + } + if !ok { + continue + } + files, err := resolveLocalPackageSkillFilesWalk(skillDir) if err != nil { - return nil, nil, fmt.Errorf("failed to list files in skill directory %q: %w", skillDir, err) + return nil, nil, err } + skillFiles = append(skillFiles, files...) } - return skillFiles, warnings, nil } +func resolveLocalPackageSkillFilesValidateMarker(manifestSkillDirSet map[string]struct{}, skillDir string) (bool, string, error) { + if _, fromManifest := manifestSkillDirSet[skillDir]; !fromManifest { + return true, "", nil + } + markerPath := filepath.Join(skillDir, packageSkillMarkerFile) + if _, err := os.Stat(markerPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return false, fmt.Sprintf("Skill directory %q is missing required %s marker file", skillDir, packageSkillMarkerFile), nil + } + return false, "", fmt.Errorf("failed to validate skill marker %q: %w", markerPath, err) + } + return true, "", nil +} + +func resolveLocalPackageSkillFilesWalk(skillDir string) ([]resolvedPackageSkillFile, error) { + var skillFiles []resolvedPackageSkillFile + skillName := filepath.Base(skillDir) + err := filepath.WalkDir(skillDir, func(currentPath string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + skillFiles = append(skillFiles, resolvedPackageSkillFile{SourcePath: currentPath, SkillName: skillName}) + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to list files in skill directory %q: %w", skillDir, err) + } + return skillFiles, nil +} + func resolveLocalPackageAgentFiles(packageDir string, explicitAgentFiles []string) ([]string, []string, error) { if len(explicitAgentFiles) > 0 { agentFiles := make([]string, 0, len(explicitAgentFiles)) @@ -711,6 +762,20 @@ func appendRepositoryPackageWorkflowSpecs(parsedSpecs []*WorkflowSpec, repoSpec } host := explicitHostForRepo(repoSpec.RepoSlug) effectiveVersion := repositoryPackageEffectiveRef(repoSpec, pkg) + parsedSpecs = appendRepositoryPackageWorkflowSpecsInstallable(parsedSpecs, repoSpec, pkg, host, effectiveVersion) + parsedSpecs = appendRepositoryPackageWorkflowSpecsSkills(parsedSpecs, repoSpec, pkg, host, effectiveVersion) + return appendRepositoryPackageWorkflowSpecsAgents(parsedSpecs, repoSpec, pkg, host, effectiveVersion) +} + +func appendRepositoryPackageWorkflowSpecsRepoSpec(repoSpec *RepoSpec, effectiveVersion string) RepoSpec { + return RepoSpec{ + RepoSlug: repoSpec.RepoSlug, + Version: effectiveVersion, + PackagePath: repoSpec.PackagePath, + } +} + +func appendRepositoryPackageWorkflowSpecsInstallable(parsedSpecs []*WorkflowSpec, repoSpec *RepoSpec, pkg *resolvedRepositoryPackage, host, effectiveVersion string) []*WorkflowSpec { for _, installationSource := range pkg.InstallationSource { // installationSource is guaranteed by isSupportedPackageInstallablePath to be // either a .md agentic workflow or a .yml action workflow file; no other @@ -719,18 +784,17 @@ func appendRepositoryPackageWorkflowSpecs(parsedSpecs []*WorkflowSpec, repoSpec // Use filepath.Ext for case-insensitive extension removal (e.g. ".YML" or ".MD"). workflowName := strings.TrimSuffix(base, filepath.Ext(base)) parsedSpecs = append(parsedSpecs, &WorkflowSpec{ - RepoSpec: RepoSpec{ - RepoSlug: repoSpec.RepoSlug, - Version: effectiveVersion, - PackagePath: repoSpec.PackagePath, - }, + RepoSpec: appendRepositoryPackageWorkflowSpecsRepoSpec(repoSpec, effectiveVersion), WorkflowPath: installationSource, WorkflowName: workflowName, Host: host, FromRepositoryManifest: true, }) } + return parsedSpecs +} +func appendRepositoryPackageWorkflowSpecsSkills(parsedSpecs []*WorkflowSpec, repoSpec *RepoSpec, pkg *resolvedRepositoryPackage, host, effectiveVersion string) []*WorkflowSpec { // Append skill file specs. Each spec carries IsPackageSkillFile=true and the SkillName // so that the installation step can route the file to the correct skill directory. for _, skillFile := range pkg.SkillFiles { @@ -738,11 +802,7 @@ func appendRepositoryPackageWorkflowSpecs(parsedSpecs []*WorkflowSpec, repoSpec // WorkflowName is unused for skill files but set to a stable value for logging. workflowName := skillFile.SkillName + "/" + strings.TrimSuffix(base, filepath.Ext(base)) parsedSpecs = append(parsedSpecs, &WorkflowSpec{ - RepoSpec: RepoSpec{ - RepoSlug: repoSpec.RepoSlug, - Version: effectiveVersion, - PackagePath: repoSpec.PackagePath, - }, + RepoSpec: appendRepositoryPackageWorkflowSpecsRepoSpec(repoSpec, effectiveVersion), WorkflowPath: skillFile.SourcePath, WorkflowName: workflowName, Host: host, @@ -750,25 +810,23 @@ func appendRepositoryPackageWorkflowSpecs(parsedSpecs []*WorkflowSpec, repoSpec SkillName: skillFile.SkillName, }) } + return parsedSpecs +} +func appendRepositoryPackageWorkflowSpecsAgents(parsedSpecs []*WorkflowSpec, repoSpec *RepoSpec, pkg *resolvedRepositoryPackage, host, effectiveVersion string) []*WorkflowSpec { // Append agent file specs. Each spec carries IsPackageAgentFile=true so the installation // step routes the file to the correct agents directory. for _, agentFile := range pkg.AgentFiles { base := filepath.Base(agentFile) workflowName := strings.TrimSuffix(base, filepath.Ext(base)) parsedSpecs = append(parsedSpecs, &WorkflowSpec{ - RepoSpec: RepoSpec{ - RepoSlug: repoSpec.RepoSlug, - Version: effectiveVersion, - PackagePath: repoSpec.PackagePath, - }, + RepoSpec: appendRepositoryPackageWorkflowSpecsRepoSpec(repoSpec, effectiveVersion), WorkflowPath: agentFile, WorkflowName: workflowName, Host: host, IsPackageAgentFile: true, }) } - return parsedSpecs } @@ -799,47 +857,66 @@ func resolveAddWorkflowSpecAndContent(ctx context.Context, initialSpec *Workflow } visited[locationKey] = struct{}{} - redirect, err := extractRedirectFromContent(string(fetched.Content)) + redirectResult, err := resolveAddWorkflowSpecAndContentRedirect(¤tSpec, initialSpec, fetched, locationKey, followedRedirect) if err != nil { return nil, nil, err } - if redirect == "" { - // Preserve the original WorkflowName from the user's request only when - // one or more redirects were followed, so the final local file keeps - // the requested name. - // Without redirects, keep any name derived during fetch, such as JSON - // imports where conversion picks a better filename from `name`. - if followedRedirect { - currentSpec.WorkflowName = initialSpec.WorkflowName - } + if redirectResult.done { return ¤tSpec, fetched, nil } - - redirectedSource, err := normalizeRedirectToSourceSpec(redirect) - if err != nil { - return nil, nil, fmt.Errorf("invalid redirect %q in %s: %w", redirect, locationKey, err) - } - - nextSpec := &WorkflowSpec{ - RepoSpec: RepoSpec{ - RepoSlug: redirectedSource.Repo, - Version: redirectedSource.Ref, - }, - WorkflowPath: redirectedSource.Path, - WorkflowName: normalizeWorkflowID(redirectedSource.Path), - Host: currentSpec.Host, - } - resolutionLog.Printf("Following redirect for add: from=%s to=%s", locationKey, nextSpec.String()) + followedRedirect = true + currentSpec = redirectResult.nextSpec if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Workflow redirect: %s -> %s", locationKey, nextSpec.String()))) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Workflow redirect: %s -> %s", locationKey, currentSpec.String()))) } - followedRedirect = true - currentSpec = *nextSpec } return nil, nil, fmt.Errorf("redirect chain exceeded maximum depth (%d) for workflow '%s'", maxRedirectDepth, initialSpec.String()) } +type resolveAddWorkflowSpecAndContentRedirectResult struct { + done bool + nextSpec WorkflowSpec +} + +func resolveAddWorkflowSpecAndContentRedirect(currentSpec, initialSpec *WorkflowSpec, fetched *FetchedWorkflow, locationKey string, followedRedirect bool) (resolveAddWorkflowSpecAndContentRedirectResult, error) { + redirect, err := extractRedirectFromContent(string(fetched.Content)) + if err != nil { + return resolveAddWorkflowSpecAndContentRedirectResult{}, err + } + if redirect == "" { + // Preserve the original WorkflowName from the user's request only when + // one or more redirects were followed, so the final local file keeps + // the requested name. + // Without redirects, keep any name derived during fetch, such as JSON + // imports where conversion picks a better filename from `name`. + if followedRedirect { + currentSpec.WorkflowName = initialSpec.WorkflowName + } + return resolveAddWorkflowSpecAndContentRedirectResult{done: true}, nil + } + + redirectedSource, err := normalizeRedirectToSourceSpec(redirect) + if err != nil { + return resolveAddWorkflowSpecAndContentRedirectResult{}, fmt.Errorf("invalid redirect %q in %s: %w", redirect, locationKey, err) + } + nextSpec := resolveAddWorkflowSpecAndContentNextSpec(currentSpec, redirectedSource) + resolutionLog.Printf("Following redirect for add: from=%s to=%s", locationKey, nextSpec.String()) + return resolveAddWorkflowSpecAndContentRedirectResult{nextSpec: nextSpec}, nil +} + +func resolveAddWorkflowSpecAndContentNextSpec(currentSpec *WorkflowSpec, redirectedSource *SourceSpec) WorkflowSpec { + return WorkflowSpec{ + RepoSpec: RepoSpec{ + RepoSlug: redirectedSource.Repo, + Version: redirectedSource.Ref, + }, + WorkflowPath: redirectedSource.Path, + WorkflowName: normalizeWorkflowID(redirectedSource.Path), + Host: currentSpec.Host, + } +} + // expandLocalWildcardWorkflows expands wildcard workflow specifications for local workflows only. func expandLocalWildcardWorkflows(specs []*WorkflowSpec, verbose bool) ([]*WorkflowSpec, error) { expandedWorkflows := []*WorkflowSpec{} diff --git a/pkg/cli/audit_agentic_analysis.go b/pkg/cli/audit_agentic_analysis.go index ad7afe3c636..be15a6b1156 100644 --- a/pkg/cli/audit_agentic_analysis.go +++ b/pkg/cli/audit_agentic_analysis.go @@ -105,59 +105,22 @@ func mergeMCPToolUsageInfo(toolUsage []ToolUsageInfo, mcpToolUsage *MCPToolUsage toolStats[info.Name] = &cloned } - addOrUpdateToolUsage := func(name string, callCount, maxInputSize, maxOutputSize int, maxDuration string) { - normalizedName := strings.TrimSpace(name) - if normalizedName == "" { - return - } - displayKey := workflow.PrettifyToolName(normalizedName) - if existing, exists := toolStats[displayKey]; exists { - existing.CallCount += callCount - if maxInputSize > existing.MaxInputSize { - existing.MaxInputSize = maxInputSize - } - if maxOutputSize > existing.MaxOutputSize { - existing.MaxOutputSize = maxOutputSize - } - if maxDuration != "" { - maxDurationValue := parseDurationString(maxDuration) - if existing.MaxDuration == "" { - existing.MaxDuration = maxDuration - } else { - existingMaxDurationValue := parseDurationString(existing.MaxDuration) - if maxDurationValue > existingMaxDurationValue { - existing.MaxDuration = maxDuration - } - } - } - return - } - - toolStats[displayKey] = &ToolUsageInfo{ - Name: displayKey, - CallCount: callCount, - MaxInputSize: maxInputSize, - MaxOutputSize: maxOutputSize, - MaxDuration: maxDuration, - } - } - if len(mcpToolUsage.Summary) > 0 { for _, summary := range mcpToolUsage.Summary { switch { case summary.ServerName != "" && summary.ToolName != "": - addOrUpdateToolUsage(summary.ServerName+"."+summary.ToolName, summary.CallCount, summary.MaxInputSize, summary.MaxOutputSize, summary.MaxDuration) + mergeMCPToolUsageInfoAdd(toolStats, summary.ServerName+"."+summary.ToolName, summary.CallCount, summary.MaxInputSize, summary.MaxOutputSize, summary.MaxDuration) case summary.ToolName != "": - addOrUpdateToolUsage(summary.ToolName, summary.CallCount, summary.MaxInputSize, summary.MaxOutputSize, summary.MaxDuration) + mergeMCPToolUsageInfoAdd(toolStats, summary.ToolName, summary.CallCount, summary.MaxInputSize, summary.MaxOutputSize, summary.MaxDuration) } } } else { for _, call := range mcpToolUsage.ToolCalls { switch { case call.ServerName != "" && call.ToolName != "": - addOrUpdateToolUsage(call.ServerName+"."+call.ToolName, 1, call.InputSize, call.OutputSize, call.Duration) + mergeMCPToolUsageInfoAdd(toolStats, call.ServerName+"."+call.ToolName, 1, call.InputSize, call.OutputSize, call.Duration) case call.ToolName != "": - addOrUpdateToolUsage(call.ToolName, 1, call.InputSize, call.OutputSize, call.Duration) + mergeMCPToolUsageInfoAdd(toolStats, call.ToolName, 1, call.InputSize, call.OutputSize, call.Duration) } } } @@ -177,6 +140,43 @@ func mergeMCPToolUsageInfo(toolUsage []ToolUsageInfo, mcpToolUsage *MCPToolUsage return mergedToolUsage } +func mergeMCPToolUsageInfoAdd(toolStats map[string]*ToolUsageInfo, name string, callCount, maxInputSize, maxOutputSize int, maxDuration string) { + normalizedName := strings.TrimSpace(name) + if normalizedName == "" { + return + } + displayKey := workflow.PrettifyToolName(normalizedName) + if existing, exists := toolStats[displayKey]; exists { + mergeMCPToolUsageInfoUpdateExisting(existing, callCount, maxInputSize, maxOutputSize, maxDuration) + return + } + + toolStats[displayKey] = &ToolUsageInfo{ + Name: displayKey, + CallCount: callCount, + MaxInputSize: maxInputSize, + MaxOutputSize: maxOutputSize, + MaxDuration: maxDuration, + } +} + +func mergeMCPToolUsageInfoUpdateExisting(existing *ToolUsageInfo, callCount, maxInputSize, maxOutputSize int, maxDuration string) { + existing.CallCount += callCount + if maxInputSize > existing.MaxInputSize { + existing.MaxInputSize = maxInputSize + } + if maxOutputSize > existing.MaxOutputSize { + existing.MaxOutputSize = maxOutputSize + } + if maxDuration == "" { + return + } + maxDurationValue := parseDurationString(maxDuration) + if existing.MaxDuration == "" || maxDurationValue > parseDurationString(existing.MaxDuration) { + existing.MaxDuration = maxDuration + } +} + func deriveRunAgenticAnalysis(processedRun ProcessedRun, metrics LogMetrics) (*AwContext, []ToolUsageInfo, []CreatedItemReport, *TaskDomainInfo, *BehaviorFingerprint, []AgenticAssessment) { auditAgenticLog.Printf("Deriving agentic analysis for run: id=%d workflow=%s", processedRun.Run.DatabaseID, processedRun.Run.WorkflowName) var awContext *AwContext @@ -316,20 +316,33 @@ func buildAgenticAssessments(processedRun ProcessedRun, metrics MetricsData, too frictionEvents := len(processedRun.MissingTools) + len(processedRun.MCPFailures) + len(processedRun.MissingData) writeCount := len(createdItems) + processedRun.Run.SafeItemsCount - if fingerprint.ResourceProfile == "heavy" { - severity := "medium" - if metrics.Turns >= 14 || toolTypes >= 7 || processedRun.Run.Duration >= 20*time.Minute { - severity = "high" - } - assessments = append(assessments, AgenticAssessment{ - Kind: "resource_heavy_for_domain", - Severity: severity, - Summary: fmt.Sprintf("This %s run consumed a heavy execution profile for its task shape.", domain.Label), - Evidence: fmt.Sprintf("turns=%d tool_types=%d duration=%s write_actions=%d", metrics.Turns, toolTypes, formatAssessmentDuration(processedRun.Run.Duration), writeCount), - Recommendation: "Compare this run to similar successful runs and trim unnecessary turns, tools, or write actions.", - }) + assessments = buildAgenticAssessmentsResource(processedRun, metrics, domain, fingerprint, assessments, toolTypes, writeCount) + assessments = buildAgenticAssessmentsControl(processedRun, metrics, domain, fingerprint, assessments, toolTypes, frictionEvents, writeCount) + assessments = buildAgenticAssessmentsOptimization(metrics, domain, fingerprint, assessments) + assessments = buildAgenticAssessmentsDelegated(awContext, assessments) + + auditAgenticLog.Printf("Built %d agentic assessments", len(assessments)) + return assessments +} + +func buildAgenticAssessmentsResource(processedRun ProcessedRun, metrics MetricsData, domain *TaskDomainInfo, fingerprint *BehaviorFingerprint, assessments []AgenticAssessment, toolTypes, writeCount int) []AgenticAssessment { + if fingerprint.ResourceProfile != "heavy" { + return assessments + } + severity := "medium" + if metrics.Turns >= 14 || toolTypes >= 7 || processedRun.Run.Duration >= 20*time.Minute { + severity = "high" } + return append(assessments, AgenticAssessment{ + Kind: "resource_heavy_for_domain", + Severity: severity, + Summary: fmt.Sprintf("This %s run consumed a heavy execution profile for its task shape.", domain.Label), + Evidence: fmt.Sprintf("turns=%d tool_types=%d duration=%s write_actions=%d", metrics.Turns, toolTypes, formatAssessmentDuration(processedRun.Run.Duration), writeCount), + Recommendation: "Compare this run to similar successful runs and trim unnecessary turns, tools, or write actions.", + }) +} +func buildAgenticAssessmentsControl(processedRun ProcessedRun, metrics MetricsData, domain *TaskDomainInfo, fingerprint *BehaviorFingerprint, assessments []AgenticAssessment, toolTypes, frictionEvents, writeCount int) []AgenticAssessment { if (domain.Name == "triage" || domain.Name == "repo_maintenance" || domain.Name == "issue_response") && fingerprint.ResourceProfile == "lean" && fingerprint.ExecutionStyle == "directed" && fingerprint.ToolBreadth == "narrow" { assessments = append(assessments, AgenticAssessment{ Kind: "overkill_for_agentic", @@ -339,7 +352,6 @@ func buildAgenticAssessments(processedRun ProcessedRun, metrics MetricsData, too Recommendation: "Consider whether a scripted rule or deterministic workflow step could replace this agentic path.", }) } - if frictionEvents >= 3 || (frictionEvents > 0 && writeCount >= 3) || ((domain.Name == "triage" || domain.Name == "repo_maintenance" || domain.Name == "issue_response") && fingerprint.ExecutionStyle == "exploratory") { severity := "medium" if frictionEvents >= 4 || (frictionEvents > 0 && fingerprint.ActuationStyle == "write_heavy") { @@ -353,29 +365,12 @@ func buildAgenticAssessments(processedRun ProcessedRun, metrics MetricsData, too Recommendation: "Tighten instructions, reduce unnecessary tools, or delay write actions until the workflow has stronger evidence.", }) } + _ = processedRun + return assessments +} - // Partially reducible: the workflow has a low agentic fraction, meaning - // many turns are data-gathering that could be moved to deterministic steps: - // or post-steps: in the frontmatter. Only flag when there's substantive work - // (not lean/directed runs which overkill_for_agentic already covers). - if fingerprint.AgenticFraction > 0 && fingerprint.AgenticFraction < 0.6 && - fingerprint.ResourceProfile != "lean" { - severity := "low" - if fingerprint.AgenticFraction < 0.4 { - severity = "medium" - } - deterministicPct := int((1.0 - fingerprint.AgenticFraction) * 100) - assessments = append(assessments, AgenticAssessment{ - Kind: "partially_reducible", - Severity: severity, - Summary: fmt.Sprintf("About %d%% of this run's turns appear to be data-gathering that could move to deterministic steps.", deterministicPct), - Evidence: fmt.Sprintf("agentic_fraction=%.2f turns=%d", fingerprint.AgenticFraction, metrics.Turns), - Recommendation: "Move data-fetching work to frontmatter steps: (pre-agent) writing to /tmp/gh-aw/agent/ or post-steps: (post-agent) to reduce inference cost. See the DeterministicOps guide.", - }) - } - - // Model downgrade suggestion: the run uses a heavy resource profile but - // the task domain is simple enough that a smaller model would likely suffice. +func buildAgenticAssessmentsOptimization(metrics MetricsData, domain *TaskDomainInfo, fingerprint *BehaviorFingerprint, assessments []AgenticAssessment) []AgenticAssessment { + assessments = buildAgenticAssessmentsReducible(metrics, fingerprint, assessments) if fingerprint.ResourceProfile != "lean" && (domain.Name == "triage" || domain.Name == "repo_maintenance" || domain.Name == "issue_response") && fingerprint.ActuationStyle != "write_heavy" { @@ -387,19 +382,42 @@ func buildAgenticAssessments(processedRun ProcessedRun, metrics MetricsData, too Recommendation: "Try engine.model: gpt-4.1-mini or claude-haiku-4-5 in the workflow frontmatter.", }) } + return assessments +} - if awContext != nil { - assessments = append(assessments, AgenticAssessment{ - Kind: "delegated_context_present", - Severity: "info", - Summary: "The run preserved upstream dispatch context, which helps trace multi-workflow episodes.", - Evidence: fmt.Sprintf("workflow_call_id=%s event_type=%s", awContext.WorkflowCallID, awContext.EventType), - Recommendation: "Use this context when comparing downstream runs so follow-up workflows are evaluated as part of one task chain.", - }) - } +func buildAgenticAssessmentsReducible(metrics MetricsData, fingerprint *BehaviorFingerprint, assessments []AgenticAssessment) []AgenticAssessment { + // Partially reducible: the workflow has a low agentic fraction, meaning + // many turns are data-gathering that could be moved to deterministic steps: + // or post-steps: in the frontmatter. Only flag when there's substantive work + // (not lean/directed runs which overkill_for_agentic already covers). + if fingerprint.AgenticFraction <= 0 || fingerprint.AgenticFraction >= 0.6 || fingerprint.ResourceProfile == "lean" { + return assessments + } + severity := "low" + if fingerprint.AgenticFraction < 0.4 { + severity = "medium" + } + deterministicPct := int((1.0 - fingerprint.AgenticFraction) * 100) + return append(assessments, AgenticAssessment{ + Kind: "partially_reducible", + Severity: severity, + Summary: fmt.Sprintf("About %d%% of this run's turns appear to be data-gathering that could move to deterministic steps.", deterministicPct), + Evidence: fmt.Sprintf("agentic_fraction=%.2f turns=%d", fingerprint.AgenticFraction, metrics.Turns), + Recommendation: "Move data-fetching work to frontmatter steps: (pre-agent) writing to /tmp/gh-aw/agent/ or post-steps: (post-agent) to reduce inference cost. See the DeterministicOps guide.", + }) +} - auditAgenticLog.Printf("Built %d agentic assessments", len(assessments)) - return assessments +func buildAgenticAssessmentsDelegated(awContext *AwContext, assessments []AgenticAssessment) []AgenticAssessment { + if awContext == nil { + return assessments + } + return append(assessments, AgenticAssessment{ + Kind: "delegated_context_present", + Severity: "info", + Summary: "The run preserved upstream dispatch context, which helps trace multi-workflow episodes.", + Evidence: fmt.Sprintf("workflow_call_id=%s event_type=%s", awContext.WorkflowCallID, awContext.EventType), + Recommendation: "Use this context when comparing downstream runs so follow-up workflows are evaluated as part of one task chain.", + }) } func generateAgenticAssessmentFindings(assessments []AgenticAssessment) []Finding { diff --git a/pkg/cli/audit_comparison.go b/pkg/cli/audit_comparison.go index 0ee05986d40..957da4223a7 100644 --- a/pkg/cli/audit_comparison.go +++ b/pkg/cli/audit_comparison.go @@ -344,24 +344,53 @@ func buildAuditComparison(currentConclusion string, current auditComparisonSnaps reasonCodes := make([]string, 0, 4) currentConclusion = strings.TrimSpace(strings.ToLower(currentConclusion)) currentRunUnsuccessful := currentConclusion != "" && currentConclusion != "success" - delta := &AuditComparisonDelta{ - Turns: AuditComparisonIntDelta{ - Before: baseline.Turns, - After: current.Turns, - Changed: baseline.Turns != current.Turns, + delta := buildAuditComparisonDelta(current, baseline) + + reasonCodes = append(reasonCodes, buildAuditComparisonReasonCodes(current, baseline)...) + if currentRunUnsuccessful { + reasonCodes = append(reasonCodes, "run_unsuccessful") + } + + newMCPFailure, mcpFailuresResolved := buildAuditComparisonMCPFailureDelta(delta, current, baseline) + switch { + case newMCPFailure: + reasonCodes = append(reasonCodes, "new_mcp_failure") + case mcpFailuresResolved: + reasonCodes = append(reasonCodes, "mcp_failures_resolved") + } + + label := buildAuditComparisonLabel(currentRunUnsuccessful, newMCPFailure, mcpFailuresResolved, current, baseline, delta, reasonCodes) + + return &AuditComparisonData{ + BaselineFound: true, + Baseline: &AuditComparisonBaseline{ + RunID: baselineRun.DatabaseID, + WorkflowName: baselineRun.WorkflowName, + Conclusion: baselineRun.Conclusion, + CreatedAt: baselineRun.CreatedAt.Format("2006-01-02T15:04:05Z07:00"), + Selection: "latest_success", }, - Posture: AuditComparisonStringDelta{ - Before: baseline.Posture, - After: current.Posture, - Changed: baseline.Posture != current.Posture, + Delta: delta, + Classification: &AuditComparisonClassification{ + Label: label, + ReasonCodes: reasonCodes, }, - BlockedRequests: AuditComparisonIntDelta{ - Before: baseline.BlockedRequests, - After: current.BlockedRequests, - Changed: baseline.BlockedRequests != current.BlockedRequests, + Recommendation: &AuditComparisonRecommendation{ + Action: recommendAuditComparisonAction(label, currentConclusion, delta), }, } +} + +func buildAuditComparisonDelta(current auditComparisonSnapshot, baseline *auditComparisonSnapshot) *AuditComparisonDelta { + return &AuditComparisonDelta{ + Turns: AuditComparisonIntDelta{Before: baseline.Turns, After: current.Turns, Changed: baseline.Turns != current.Turns}, + Posture: AuditComparisonStringDelta{Before: baseline.Posture, After: current.Posture, Changed: baseline.Posture != current.Posture}, + BlockedRequests: AuditComparisonIntDelta{Before: baseline.BlockedRequests, After: current.BlockedRequests, Changed: baseline.BlockedRequests != current.BlockedRequests}, + } +} +func buildAuditComparisonReasonCodes(current auditComparisonSnapshot, baseline *auditComparisonSnapshot) []string { + var reasonCodes []string if current.Turns > baseline.Turns { reasonCodes = append(reasonCodes, "turns_increase") } else if current.Turns < baseline.Turns { @@ -375,10 +404,10 @@ func buildAuditComparison(currentConclusion string, current auditComparisonSnaps } else if current.BlockedRequests < baseline.BlockedRequests { reasonCodes = append(reasonCodes, "blocked_requests_decrease") } - if currentRunUnsuccessful { - reasonCodes = append(reasonCodes, "run_unsuccessful") - } + return reasonCodes +} +func buildAuditComparisonMCPFailureDelta(delta *AuditComparisonDelta, current auditComparisonSnapshot, baseline *auditComparisonSnapshot) (bool, bool) { newMCPFailure := len(baseline.MCPFailures) == 0 && len(current.MCPFailures) > 0 mcpFailuresResolved := len(baseline.MCPFailures) > 0 && len(current.MCPFailures) == 0 if newMCPFailure || len(baseline.MCPFailures) > 0 || len(current.MCPFailures) > 0 { @@ -388,49 +417,29 @@ func buildAuditComparison(currentConclusion string, current auditComparisonSnaps NewlyPresent: newMCPFailure, } } - if newMCPFailure { - reasonCodes = append(reasonCodes, "new_mcp_failure") - } else if mcpFailuresResolved { - reasonCodes = append(reasonCodes, "mcp_failures_resolved") - } + return newMCPFailure, mcpFailuresResolved +} - label := "stable" +func buildAuditComparisonLabel(currentRunUnsuccessful, newMCPFailure, mcpFailuresResolved bool, current auditComparisonSnapshot, baseline *auditComparisonSnapshot, delta *AuditComparisonDelta, reasonCodes []string) string { switch { case currentRunUnsuccessful: - label = "risky" + return "risky" case delta.Posture.Before == "read_only" && delta.Posture.After == "write_capable": - label = "risky" + return "risky" case newMCPFailure: - label = "risky" + return "risky" case current.BlockedRequests > baseline.BlockedRequests: - label = "risky" + return "risky" case delta.Posture.Before != "" && delta.Posture.After != "" && delta.Posture.Before != delta.Posture.After: - label = "changed" + return "changed" case mcpFailuresResolved: - label = "changed" + return "changed" case current.BlockedRequests < baseline.BlockedRequests: - label = "changed" + return "changed" case len(reasonCodes) > 0: - label = "changed" - } - - return &AuditComparisonData{ - BaselineFound: true, - Baseline: &AuditComparisonBaseline{ - RunID: baselineRun.DatabaseID, - WorkflowName: baselineRun.WorkflowName, - Conclusion: baselineRun.Conclusion, - CreatedAt: baselineRun.CreatedAt.Format("2006-01-02T15:04:05Z07:00"), - Selection: "latest_success", - }, - Delta: delta, - Classification: &AuditComparisonClassification{ - Label: label, - ReasonCodes: reasonCodes, - }, - Recommendation: &AuditComparisonRecommendation{ - Action: recommendAuditComparisonAction(label, currentConclusion, delta), - }, + return "changed" + default: + return "stable" } } diff --git a/pkg/cli/audit_diff.go b/pkg/cli/audit_diff.go index efb4d75df1d..5f9bd32c38f 100644 --- a/pkg/cli/audit_diff.go +++ b/pkg/cli/audit_diff.go @@ -66,23 +66,47 @@ func computeFirewallDiff(run1ID, run2ID int64, run1, run2 *FirewallAnalysis) *Fi Run2ID: run2ID, } - // Handle nil cases + run1Stats, run2Stats := computeFirewallDiffStats(run1, run2) + + // If both are nil/empty, return empty diff + if len(run1Stats) == 0 && len(run2Stats) == 0 { + return diff + } + + anomalyCount := 0 + for _, domain := range computeFirewallDiffSortedDomains(run1Stats, run2Stats) { + stats1, inRun1 := run1Stats[domain] + stats2, inRun2 := run2Stats[domain] + anomalyCount += computeFirewallDiffDomain(diff, domain, stats1, stats2, inRun1, inRun2) + } + + diff.Summary = FirewallDiffSummary{ + NewDomainCount: len(diff.NewDomains), + RemovedDomainCount: len(diff.RemovedDomains), + StatusChangeCount: len(diff.StatusChanges), + VolumeChangeCount: len(diff.VolumeChanges), + HasAnomalies: anomalyCount > 0, + AnomalyCount: anomalyCount, + } + + auditDiffLog.Printf("Firewall diff complete: new=%d, removed=%d, status_changes=%d, volume_changes=%d, anomalies=%d", + len(diff.NewDomains), len(diff.RemovedDomains), len(diff.StatusChanges), len(diff.VolumeChanges), anomalyCount) + return diff +} + +func computeFirewallDiffStats(run1, run2 *FirewallAnalysis) (map[string]DomainRequestStats, map[string]DomainRequestStats) { run1Stats := make(map[string]DomainRequestStats) run2Stats := make(map[string]DomainRequestStats) - if run1 != nil { run1Stats = run1.RequestsByDomain } if run2 != nil { run2Stats = run2.RequestsByDomain } + return run1Stats, run2Stats +} - // If both are nil/empty, return empty diff - if len(run1Stats) == 0 && len(run2Stats) == 0 { - return diff - } - - // Collect all domains +func computeFirewallDiffSortedDomains(run1Stats, run2Stats map[string]DomainRequestStats) []string { allDomains := make(map[string]struct{}) for domain := range run1Stats { allDomains[domain] = struct{}{} @@ -90,120 +114,110 @@ func computeFirewallDiff(run1ID, run2ID int64, run1, run2 *FirewallAnalysis) *Fi for domain := range run2Stats { allDomains[domain] = struct{}{} } + return sliceutil.SortedKeys(allDomains) +} - // Sorted domain list for deterministic output - sortedDomains := sliceutil.SortedKeys(allDomains) +func computeFirewallDiffDomain(diff *FirewallDiff, domain string, stats1, stats2 DomainRequestStats, inRun1, inRun2 bool) int { + switch { + case !inRun1 && inRun2: + entry := computeFirewallDiffNewDomain(domain, stats2) + diff.NewDomains = append(diff.NewDomains, entry) + if entry.IsAnomaly { + return 1 + } + case inRun1 && !inRun2: + entry := computeFirewallDiffRemovedDomain(domain, stats1) + diff.RemovedDomains = append(diff.RemovedDomains, entry) + if entry.IsAnomaly { + return 1 + } + default: + return computeFirewallDiffExistingDomain(diff, domain, stats1, stats2) + } + return 0 +} - anomalyCount := 0 +func computeFirewallDiffNewDomain(domain string, stats2 DomainRequestStats) DomainDiffEntry { + entry := DomainDiffEntry{ + Domain: domain, + Status: "new", + Run2Allowed: stats2.Allowed, + Run2Blocked: stats2.Blocked, + Run2Status: classifyFirewallDomainStatus(stats2), + } + if stats2.Blocked > 0 { + entry.IsAnomaly = true + entry.AnomalyNote = "new denied domain" + } + return entry +} - for _, domain := range sortedDomains { - stats1, inRun1 := run1Stats[domain] - stats2, inRun2 := run2Stats[domain] +func computeFirewallDiffRemovedDomain(domain string, stats1 DomainRequestStats) DomainDiffEntry { + entry := DomainDiffEntry{ + Domain: domain, + Status: "removed", + Run1Allowed: stats1.Allowed, + Run1Blocked: stats1.Blocked, + Run1Status: classifyFirewallDomainStatus(stats1), + } + // Anomaly: the removed domain was denied in the base run. This indicates a + // transient firewall block that prevented the agent from reaching an MCP server. + if stats1.Blocked > 0 { + entry.IsAnomaly = true + entry.AnomalyNote = "denied in base run — absent from comparison run" + } + return entry +} - if !inRun1 && inRun2 { - // New domain in run 2 - entry := DomainDiffEntry{ - Domain: domain, - Status: "new", - Run2Allowed: stats2.Allowed, - Run2Blocked: stats2.Blocked, - Run2Status: classifyFirewallDomainStatus(stats2), - } - // Anomaly: new denied domain - if stats2.Blocked > 0 { - entry.IsAnomaly = true - entry.AnomalyNote = "new denied domain" - anomalyCount++ - } - diff.NewDomains = append(diff.NewDomains, entry) - } else if inRun1 && !inRun2 { - // Removed domain - entry := DomainDiffEntry{ - Domain: domain, - Status: "removed", - Run1Allowed: stats1.Allowed, - Run1Blocked: stats1.Blocked, - Run1Status: classifyFirewallDomainStatus(stats1), - } - // Anomaly: the removed domain was denied in the base run. This indicates a - // transient firewall block that prevented the agent from reaching an MCP server - // (e.g. awmg-mcpg:8080) — even though the domain is absent from the comparison - // run (and therefore looks "normal"), its prior denial is worth surfacing so - // post-completion relaunch failures are detectable in audit diffs. - if stats1.Blocked > 0 { - entry.IsAnomaly = true - entry.AnomalyNote = "denied in base run — absent from comparison run" - anomalyCount++ - } - diff.RemovedDomains = append(diff.RemovedDomains, entry) - } else { - // Domain exists in both runs - check for changes - status1 := classifyFirewallDomainStatus(stats1) - status2 := classifyFirewallDomainStatus(stats2) - - if status1 != status2 { - // Status changed - entry := DomainDiffEntry{ - Domain: domain, - Status: "status_changed", - Run1Allowed: stats1.Allowed, - Run1Blocked: stats1.Blocked, - Run2Allowed: stats2.Allowed, - Run2Blocked: stats2.Blocked, - Run1Status: status1, - Run2Status: status2, - } - // Anomaly: previously denied, now allowed - if status1 == "denied" && status2 == "allowed" { - entry.IsAnomaly = true - entry.AnomalyNote = "previously denied, now allowed" - anomalyCount++ - } - // Anomaly: previously allowed, now denied - if status1 == "allowed" && status2 == "denied" { - entry.IsAnomaly = true - entry.AnomalyNote = "previously allowed, now denied" - anomalyCount++ - } - diff.StatusChanges = append(diff.StatusChanges, entry) - } else { - // Check for significant volume changes (>100% threshold) - total1 := stats1.Allowed + stats1.Blocked - total2 := stats2.Allowed + stats2.Blocked - - if total1 > 0 { - pctChange := (float64(total2-total1) / float64(total1)) * 100 - if math.Abs(pctChange) > volumeChangeThresholdPercent { - entry := DomainDiffEntry{ - Domain: domain, - Status: "volume_changed", - Run1Allowed: stats1.Allowed, - Run1Blocked: stats1.Blocked, - Run2Allowed: stats2.Allowed, - Run2Blocked: stats2.Blocked, - Run1Status: status1, - Run2Status: status2, - VolumeChange: formatVolumeChange(total1, total2), - } - diff.VolumeChanges = append(diff.VolumeChanges, entry) - } - } - } +func computeFirewallDiffExistingDomain(diff *FirewallDiff, domain string, stats1, stats2 DomainRequestStats) int { + status1 := classifyFirewallDomainStatus(stats1) + status2 := classifyFirewallDomainStatus(stats2) + if status1 != status2 { + entry := computeFirewallDiffStatusChange(domain, stats1, stats2, status1, status2) + diff.StatusChanges = append(diff.StatusChanges, entry) + if entry.IsAnomaly { + return 1 } + return 0 + } + if entry, ok := computeFirewallDiffVolumeChange(domain, stats1, stats2, status1, status2); ok { + diff.VolumeChanges = append(diff.VolumeChanges, entry) } + return 0 +} - diff.Summary = FirewallDiffSummary{ - NewDomainCount: len(diff.NewDomains), - RemovedDomainCount: len(diff.RemovedDomains), - StatusChangeCount: len(diff.StatusChanges), - VolumeChangeCount: len(diff.VolumeChanges), - HasAnomalies: anomalyCount > 0, - AnomalyCount: anomalyCount, +func computeFirewallDiffStatusChange(domain string, stats1, stats2 DomainRequestStats, status1, status2 string) DomainDiffEntry { + entry := DomainDiffEntry{ + Domain: domain, Status: "status_changed", Run1Allowed: stats1.Allowed, + Run1Blocked: stats1.Blocked, Run2Allowed: stats2.Allowed, Run2Blocked: stats2.Blocked, + Run1Status: status1, Run2Status: status2, + } + if status1 == "denied" && status2 == "allowed" { + entry.IsAnomaly = true + entry.AnomalyNote = "previously denied, now allowed" + } + if status1 == "allowed" && status2 == "denied" { + entry.IsAnomaly = true + entry.AnomalyNote = "previously allowed, now denied" } + return entry +} - auditDiffLog.Printf("Firewall diff complete: new=%d, removed=%d, status_changes=%d, volume_changes=%d, anomalies=%d", - len(diff.NewDomains), len(diff.RemovedDomains), len(diff.StatusChanges), len(diff.VolumeChanges), anomalyCount) - return diff +func computeFirewallDiffVolumeChange(domain string, stats1, stats2 DomainRequestStats, status1, status2 string) (DomainDiffEntry, bool) { + total1 := stats1.Allowed + stats1.Blocked + total2 := stats2.Allowed + stats2.Blocked + if total1 <= 0 { + return DomainDiffEntry{}, false + } + pctChange := (float64(total2-total1) / float64(total1)) * 100 + if math.Abs(pctChange) <= volumeChangeThresholdPercent { + return DomainDiffEntry{}, false + } + return DomainDiffEntry{ + Domain: domain, Status: "volume_changed", Run1Allowed: stats1.Allowed, + Run1Blocked: stats1.Blocked, Run2Allowed: stats2.Allowed, Run2Blocked: stats2.Blocked, + Run1Status: status1, Run2Status: status2, VolumeChange: formatVolumeChange(total1, total2), + }, true } // classifyFirewallDomainStatus returns "allowed", "denied", or "mixed" based on request stats @@ -418,9 +432,31 @@ func computeMCPToolsDiff(run1, run2 *MCPToolUsageData) *MCPToolsDiff { run2Count = len(run2.Summary) } auditDiffLog.Printf("Computing MCP tools diff: run1_tools=%d, run2_tools=%d", run1Count, run2Count) + run1Tools, run2Tools := computeMCPToolsDiffMaps(run1, run2) + + diff := &MCPToolsDiff{} + anomalyCount := 0 + + for _, key := range computeMCPToolsDiffSortedKeys(run1Tools, run2Tools) { + s1, inRun1 := run1Tools[key] + s2, inRun2 := run2Tools[key] + anomalyCount += computeMCPToolsDiffEntry(diff, s1, s2, inRun1, inRun2) + } + + diff.Summary = MCPToolsDiffSummary{ + NewToolCount: len(diff.NewTools), + RemovedToolCount: len(diff.RemovedTools), + ChangedToolCount: len(diff.ChangedTools), + HasAnomalies: anomalyCount > 0, + AnomalyCount: anomalyCount, + } + + return diff +} + +func computeMCPToolsDiffMaps(run1, run2 *MCPToolUsageData) (map[string]MCPToolSummary, map[string]MCPToolSummary) { run1Tools := make(map[string]MCPToolSummary) run2Tools := make(map[string]MCPToolSummary) - if run1 != nil { for _, s := range run1.Summary { run1Tools[mcpToolKey(s.ServerName, s.ToolName)] = s @@ -431,7 +467,10 @@ func computeMCPToolsDiff(run1, run2 *MCPToolUsageData) *MCPToolsDiff { run2Tools[mcpToolKey(s.ServerName, s.ToolName)] = s } } + return run1Tools, run2Tools +} +func computeMCPToolsDiffSortedKeys(run1Tools, run2Tools map[string]MCPToolSummary) []string { allKeys := make(map[string]struct{}) for k := range run1Tools { allKeys[k] = struct{}{} @@ -439,159 +478,131 @@ func computeMCPToolsDiff(run1, run2 *MCPToolUsageData) *MCPToolsDiff { for k := range run2Tools { allKeys[k] = struct{}{} } + return sliceutil.SortedKeys(allKeys) +} - sortedKeys := sliceutil.SortedKeys(allKeys) - - diff := &MCPToolsDiff{} - anomalyCount := 0 - - for _, key := range sortedKeys { - s1, inRun1 := run1Tools[key] - s2, inRun2 := run2Tools[key] - - if !inRun1 && inRun2 { - entry := MCPToolDiffEntry{ - ServerName: s2.ServerName, - ToolName: s2.ToolName, - Status: "new", - Run2CallCount: s2.CallCount, - Run2ErrorCount: s2.ErrorCount, - } - if s2.ErrorCount > 0 { - entry.IsAnomaly = true - entry.AnomalyNote = "new tool with errors" - anomalyCount++ - } +func computeMCPToolsDiffEntry(diff *MCPToolsDiff, s1, s2 MCPToolSummary, inRun1, inRun2 bool) int { + switch { + case !inRun1 && inRun2: + entry := MCPToolDiffEntry{ServerName: s2.ServerName, ToolName: s2.ToolName, Status: "new", Run2CallCount: s2.CallCount, Run2ErrorCount: s2.ErrorCount} + if s2.ErrorCount > 0 { + entry.IsAnomaly = true + entry.AnomalyNote = "new tool with errors" diff.NewTools = append(diff.NewTools, entry) - } else if inRun1 && !inRun2 { - diff.RemovedTools = append(diff.RemovedTools, MCPToolDiffEntry{ - ServerName: s1.ServerName, - ToolName: s1.ToolName, - Status: "removed", - Run1CallCount: s1.CallCount, - Run1ErrorCount: s1.ErrorCount, - }) - } else if s1.CallCount != s2.CallCount || s1.ErrorCount != s2.ErrorCount { - entry := MCPToolDiffEntry{ - ServerName: s1.ServerName, - ToolName: s1.ToolName, - Status: "changed", - Run1CallCount: s1.CallCount, - Run2CallCount: s2.CallCount, - Run1ErrorCount: s1.ErrorCount, - Run2ErrorCount: s2.ErrorCount, - CallCountChange: formatCountChange(s1.CallCount, s2.CallCount), - } - if s2.ErrorCount > s1.ErrorCount { - entry.IsAnomaly = true - entry.AnomalyNote = "error count increased" - anomalyCount++ - } + return 1 + } + diff.NewTools = append(diff.NewTools, entry) + case inRun1 && !inRun2: + diff.RemovedTools = append(diff.RemovedTools, MCPToolDiffEntry{ServerName: s1.ServerName, ToolName: s1.ToolName, Status: "removed", Run1CallCount: s1.CallCount, Run1ErrorCount: s1.ErrorCount}) + case s1.CallCount != s2.CallCount || s1.ErrorCount != s2.ErrorCount: + entry := MCPToolDiffEntry{ServerName: s1.ServerName, ToolName: s1.ToolName, Status: "changed", Run1CallCount: s1.CallCount, Run2CallCount: s2.CallCount, Run1ErrorCount: s1.ErrorCount, Run2ErrorCount: s2.ErrorCount, CallCountChange: formatCountChange(s1.CallCount, s2.CallCount)} + if s2.ErrorCount > s1.ErrorCount { + entry.IsAnomaly = true + entry.AnomalyNote = "error count increased" diff.ChangedTools = append(diff.ChangedTools, entry) + return 1 } + diff.ChangedTools = append(diff.ChangedTools, entry) } - - diff.Summary = MCPToolsDiffSummary{ - NewToolCount: len(diff.NewTools), - RemovedToolCount: len(diff.RemovedTools), - ChangedToolCount: len(diff.ChangedTools), - HasAnomalies: anomalyCount > 0, - AnomalyCount: anomalyCount, - } - - return diff + return 0 } // computeRunMetricsDiff computes the diff of run-level metrics between two runs. // Returns nil if no meaningful metrics data is available. func computeRunMetricsDiff(summary1, summary2 *RunSummary) *RunMetricsDiff { - var run1Tokens, run2Tokens int - var run1Duration, run2Duration time.Duration - var run1Turns, run2Turns int - var tu1, tu2 *TokenUsageSummary - var rl1, rl2 *GitHubRateLimitUsage - var m1, m2 *LogMetrics - - if summary1 != nil { - run1Tokens = summary1.Run.TokenUsage - run1Duration = summary1.Run.Duration - // Run.Turns may be zero on cached-summary paths; Metrics.Turns is authoritative. - run1Turns = summary1.Run.Turns - if run1Turns == 0 && summary1.Metrics.Turns > 0 { - run1Turns = summary1.Metrics.Turns - } - tu1 = summary1.TokenUsage - rl1 = summary1.GitHubRateLimitUsage - m1 = &summary1.Metrics - } - if summary2 != nil { - run2Tokens = summary2.Run.TokenUsage - run2Duration = summary2.Run.Duration - // Run.Turns may be zero on cached-summary paths; Metrics.Turns is authoritative. - run2Turns = summary2.Run.Turns - if run2Turns == 0 && summary2.Metrics.Turns > 0 { - run2Turns = summary2.Metrics.Turns - } - tu2 = summary2.TokenUsage - rl2 = summary2.GitHubRateLimitUsage - m2 = &summary2.Metrics - } + run1 := computeRunMetricsDiffValues(summary1) + run2 := computeRunMetricsDiffValues(summary2) // Skip if there is no meaningful data - hasTokenDetails := tu1 != nil || tu2 != nil - hasRateLimitDetails := rl1 != nil || rl2 != nil - if run1Tokens == 0 && run2Tokens == 0 && run1Duration == 0 && run2Duration == 0 && run1Turns == 0 && run2Turns == 0 && !hasTokenDetails && !hasRateLimitDetails { + hasTokenDetails := run1.tokenUsage != nil || run2.tokenUsage != nil + hasRateLimitDetails := run1.rateLimit != nil || run2.rateLimit != nil + if !computeRunMetricsDiffHasData(run1, run2, hasTokenDetails, hasRateLimitDetails) { return nil } diff := &RunMetricsDiff{ - Run1TokenUsage: run1Tokens, - Run2TokenUsage: run2Tokens, - Run1Turns: run1Turns, - Run2Turns: run2Turns, - TurnsChange: run2Turns - run1Turns, + Run1TokenUsage: run1.tokens, + Run2TokenUsage: run2.tokens, + Run1Turns: run1.turns, + Run2Turns: run2.turns, + TurnsChange: run2.turns - run1.turns, } - if run1Tokens > 0 || run2Tokens > 0 { - diff.TokenUsageChange = formatVolumeChange(run1Tokens, run2Tokens) - } - - if run1Duration > 0 { - diff.Run1Duration = run1Duration.Round(time.Second).String() - } - if run2Duration > 0 { - diff.Run2Duration = run2Duration.Round(time.Second).String() - } - if run1Duration > 0 && run2Duration > 0 { - delta := run2Duration - run1Duration - if delta >= 0 { - diff.DurationChange = "+" + delta.Round(time.Second).String() - } else { - diff.DurationChange = delta.Round(time.Second).String() - } - } + computeRunMetricsDiffPopulateTokenAndDuration(diff, run1, run2) // Compute tokens per turn using engine-level token usage. - run1PerTurn := run1Tokens - run2PerTurn := run2Tokens - if run1Turns > 0 { - diff.Run1TokensPerTurn = run1PerTurn / run1Turns + run1PerTurn := run1.tokens + run2PerTurn := run2.tokens + if run1.turns > 0 { + diff.Run1TokensPerTurn = run1PerTurn / run1.turns } - if run2Turns > 0 { - diff.Run2TokensPerTurn = run2PerTurn / run2Turns + if run2.turns > 0 { + diff.Run2TokensPerTurn = run2PerTurn / run2.turns } if diff.Run1TokensPerTurn > 0 || diff.Run2TokensPerTurn > 0 { diff.TokensPerTurnChange = formatVolumeChange(diff.Run1TokensPerTurn, diff.Run2TokensPerTurn) } - diff.TokenUsageDetails = computeTokenUsageDiff(tu1, tu2) - diff.GitHubRateLimitDetails = computeGitHubRateLimitDiff(rl1, rl2) - diff.ToolCallsDiff = computeToolCallsDiff(m1, m2) + diff.TokenUsageDetails = computeTokenUsageDiff(run1.tokenUsage, run2.tokenUsage) + diff.GitHubRateLimitDetails = computeGitHubRateLimitDiff(run1.rateLimit, run2.rateLimit) + diff.ToolCallsDiff = computeToolCallsDiff(run1.metrics, run2.metrics) - auditDiffLog.Printf("Run metrics diff: tokens %d->%d, turns %d->%d, has_token_details=%t, has_rate_limit_details=%t", run1Tokens, run2Tokens, run1Turns, run2Turns, hasTokenDetails, hasRateLimitDetails) + auditDiffLog.Printf("Run metrics diff: tokens %d->%d, turns %d->%d, has_token_details=%t, has_rate_limit_details=%t", run1.tokens, run2.tokens, run1.turns, run2.turns, hasTokenDetails, hasRateLimitDetails) return diff } +type computeRunMetricsDiffSummaryValues struct { + tokens int + duration time.Duration + turns int + tokenUsage *TokenUsageSummary + rateLimit *GitHubRateLimitUsage + metrics *LogMetrics +} + +func computeRunMetricsDiffValues(summary *RunSummary) computeRunMetricsDiffSummaryValues { + var values computeRunMetricsDiffSummaryValues + if summary == nil { + return values + } + values.tokens = summary.Run.TokenUsage + values.duration = summary.Run.Duration + // Run.Turns may be zero on cached-summary paths; Metrics.Turns is authoritative. + values.turns = summary.Run.Turns + if values.turns == 0 && summary.Metrics.Turns > 0 { + values.turns = summary.Metrics.Turns + } + values.tokenUsage = summary.TokenUsage + values.rateLimit = summary.GitHubRateLimitUsage + values.metrics = &summary.Metrics + return values +} + +func computeRunMetricsDiffHasData(run1, run2 computeRunMetricsDiffSummaryValues, hasTokenDetails, hasRateLimitDetails bool) bool { + return run1.tokens != 0 || run2.tokens != 0 || run1.duration != 0 || run2.duration != 0 || + run1.turns != 0 || run2.turns != 0 || hasTokenDetails || hasRateLimitDetails +} + +func computeRunMetricsDiffPopulateTokenAndDuration(diff *RunMetricsDiff, run1, run2 computeRunMetricsDiffSummaryValues) { + if run1.tokens > 0 || run2.tokens > 0 { + diff.TokenUsageChange = formatVolumeChange(run1.tokens, run2.tokens) + } + if run1.duration > 0 { + diff.Run1Duration = run1.duration.Round(time.Second).String() + } + if run2.duration > 0 { + diff.Run2Duration = run2.duration.Round(time.Second).String() + } + if run1.duration > 0 && run2.duration > 0 { + delta := run2.duration - run1.duration + if delta >= 0 { + diff.DurationChange = "+" + delta.Round(time.Second).String() + } else { + diff.DurationChange = delta.Round(time.Second).String() + } + } +} + // isBashTool returns true if the tool name represents a bash/shell invocation. // It matches the generic "bash" / "Bash" tool names used by most engines and the // per-command "bash_*" entries generated by the Codex log parser. @@ -603,122 +614,36 @@ func isBashTool(name string) bool { // computeToolCallsDiff diffs engine-level tool calls from two LogMetrics values. // Returns nil when both metrics have no tool call data. func computeToolCallsDiff(m1, m2 *LogMetrics) *ToolCallsDiff { - run1Tools := make(map[string]ToolCallInfo) - run2Tools := make(map[string]ToolCallInfo) - - // aggregateToolCall merges a tool call entry into the map, summing call counts and - // taking the max of size fields to handle duplicate entries across log files. - aggregateToolCall := func(tools map[string]ToolCallInfo, tc ToolCallInfo) { - if existing, ok := tools[tc.Name]; ok { - existing.CallCount += tc.CallCount - if tc.MaxInputSize > existing.MaxInputSize { - existing.MaxInputSize = tc.MaxInputSize - } - if tc.MaxOutputSize > existing.MaxOutputSize { - existing.MaxOutputSize = tc.MaxOutputSize - } - if tc.MaxDuration > existing.MaxDuration { - existing.MaxDuration = tc.MaxDuration - } - tools[tc.Name] = existing - return - } - tools[tc.Name] = tc - } - - if m1 != nil { - for _, tc := range m1.ToolCalls { - aggregateToolCall(run1Tools, tc) - } - } - if m2 != nil { - for _, tc := range m2.ToolCalls { - aggregateToolCall(run2Tools, tc) - } - } + run1Tools := computeToolCallsDiffMap(m1) + run2Tools := computeToolCallsDiffMap(m2) if len(run1Tools) == 0 && len(run2Tools) == 0 { return nil } - allNames := make(map[string]struct{}) - for k := range run1Tools { - allNames[k] = struct{}{} - } - for k := range run2Tools { - allNames[k] = struct{}{} - } - - sortedNames := sliceutil.SortedKeys(allNames) - diff := &ToolCallsDiff{} var run1Total, run2Total int // Collect bash tools during the main iteration to avoid a second traversal in computeBashCommandsDiff. bashRun1 := make(map[string]ToolCallInfo) bashRun2 := make(map[string]ToolCallInfo) - for _, name := range sortedNames { + for _, name := range computeToolCallsDiffSortedNames(run1Tools, run2Tools) { tc1, inRun1 := run1Tools[name] tc2, inRun2 := run2Tools[name] - if inRun1 { - run1Total += tc1.CallCount - if isBashTool(name) { - bashRun1[name] = tc1 - } - } - if inRun2 { - run2Total += tc2.CallCount - if isBashTool(name) { - bashRun2[name] = tc2 - } - } - - var entry ToolCallDiffEntry - switch { - case !inRun1 && inRun2: - entry = ToolCallDiffEntry{ - Name: name, - Status: "new", - Run2CallCount: tc2.CallCount, - Run2MaxInputSize: tc2.MaxInputSize, - Run2MaxOutputSize: tc2.MaxOutputSize, - } - diff.NewTools = append(diff.NewTools, entry) - case inRun1 && !inRun2: - entry = ToolCallDiffEntry{ - Name: name, - Status: "removed", - Run1CallCount: tc1.CallCount, - Run1MaxInputSize: tc1.MaxInputSize, - Run1MaxOutputSize: tc1.MaxOutputSize, - } - diff.RemovedTools = append(diff.RemovedTools, entry) - case tc1.CallCount != tc2.CallCount: - entry = ToolCallDiffEntry{ - Name: name, - Status: "changed", - Run1CallCount: tc1.CallCount, - Run2CallCount: tc2.CallCount, - CallCountChange: formatCountChange(tc1.CallCount, tc2.CallCount), - Run1MaxInputSize: tc1.MaxInputSize, - Run2MaxInputSize: tc2.MaxInputSize, - Run1MaxOutputSize: tc1.MaxOutputSize, - Run2MaxOutputSize: tc2.MaxOutputSize, - } - diff.ChangedTools = append(diff.ChangedTools, entry) - default: - entry = ToolCallDiffEntry{ - Name: name, - Status: "unchanged", - Run1CallCount: tc1.CallCount, - Run2CallCount: tc2.CallCount, - Run1MaxInputSize: tc1.MaxInputSize, - Run2MaxInputSize: tc2.MaxInputSize, - Run1MaxOutputSize: tc1.MaxOutputSize, - Run2MaxOutputSize: tc2.MaxOutputSize, - } - } + run1Total, run2Total = computeToolCallsDiffTrackBash(computeToolCallsDiffTrackBashParams{ + Name: name, + TC1: tc1, + TC2: tc2, + InRun1: inRun1, + InRun2: inRun2, + BashRun1: bashRun1, + BashRun2: bashRun2, + Run1Total: run1Total, + Run2Total: run2Total, + }) + entry := computeToolCallsDiffEntry(name, tc1, tc2, inRun1, inRun2) + computeToolCallsDiffAppend(diff, entry) diff.AllTools = append(diff.AllTools, entry) } @@ -736,6 +661,101 @@ func computeToolCallsDiff(m1, m2 *LogMetrics) *ToolCallsDiff { return diff } +func computeToolCallsDiffMap(metrics *LogMetrics) map[string]ToolCallInfo { + tools := make(map[string]ToolCallInfo) + if metrics == nil { + return tools + } + for _, tc := range metrics.ToolCalls { + computeToolCallsDiffAggregate(tools, tc) + } + return tools +} + +func computeToolCallsDiffAggregate(tools map[string]ToolCallInfo, tc ToolCallInfo) { + // Merges a tool call entry, summing call counts and taking max size fields. + if existing, ok := tools[tc.Name]; ok { + existing.CallCount += tc.CallCount + if tc.MaxInputSize > existing.MaxInputSize { + existing.MaxInputSize = tc.MaxInputSize + } + if tc.MaxOutputSize > existing.MaxOutputSize { + existing.MaxOutputSize = tc.MaxOutputSize + } + if tc.MaxDuration > existing.MaxDuration { + existing.MaxDuration = tc.MaxDuration + } + tools[tc.Name] = existing + return + } + tools[tc.Name] = tc +} + +func computeToolCallsDiffSortedNames(run1Tools, run2Tools map[string]ToolCallInfo) []string { + allNames := make(map[string]struct{}) + for k := range run1Tools { + allNames[k] = struct{}{} + } + for k := range run2Tools { + allNames[k] = struct{}{} + } + return sliceutil.SortedKeys(allNames) +} + +type computeToolCallsDiffTrackBashParams struct { + Name string + TC1 ToolCallInfo + TC2 ToolCallInfo + InRun1 bool + InRun2 bool + BashRun1 map[string]ToolCallInfo + BashRun2 map[string]ToolCallInfo + Run1Total int + Run2Total int +} + +func computeToolCallsDiffTrackBash(p computeToolCallsDiffTrackBashParams) (int, int) { + run1Total := p.Run1Total + run2Total := p.Run2Total + if p.InRun1 { + run1Total += p.TC1.CallCount + if isBashTool(p.Name) { + p.BashRun1[p.Name] = p.TC1 + } + } + if p.InRun2 { + run2Total += p.TC2.CallCount + if isBashTool(p.Name) { + p.BashRun2[p.Name] = p.TC2 + } + } + return run1Total, run2Total +} + +func computeToolCallsDiffEntry(name string, tc1, tc2 ToolCallInfo, inRun1, inRun2 bool) ToolCallDiffEntry { + switch { + case !inRun1 && inRun2: + return ToolCallDiffEntry{Name: name, Status: "new", Run2CallCount: tc2.CallCount, Run2MaxInputSize: tc2.MaxInputSize, Run2MaxOutputSize: tc2.MaxOutputSize} + case inRun1 && !inRun2: + return ToolCallDiffEntry{Name: name, Status: "removed", Run1CallCount: tc1.CallCount, Run1MaxInputSize: tc1.MaxInputSize, Run1MaxOutputSize: tc1.MaxOutputSize} + case tc1.CallCount != tc2.CallCount: + return ToolCallDiffEntry{Name: name, Status: "changed", Run1CallCount: tc1.CallCount, Run2CallCount: tc2.CallCount, CallCountChange: formatCountChange(tc1.CallCount, tc2.CallCount), Run1MaxInputSize: tc1.MaxInputSize, Run2MaxInputSize: tc2.MaxInputSize, Run1MaxOutputSize: tc1.MaxOutputSize, Run2MaxOutputSize: tc2.MaxOutputSize} + default: + return ToolCallDiffEntry{Name: name, Status: "unchanged", Run1CallCount: tc1.CallCount, Run2CallCount: tc2.CallCount, Run1MaxInputSize: tc1.MaxInputSize, Run2MaxInputSize: tc2.MaxInputSize, Run1MaxOutputSize: tc1.MaxOutputSize, Run2MaxOutputSize: tc2.MaxOutputSize} + } +} + +func computeToolCallsDiffAppend(diff *ToolCallsDiff, entry ToolCallDiffEntry) { + switch entry.Status { + case "new": + diff.NewTools = append(diff.NewTools, entry) + case "removed": + diff.RemovedTools = append(diff.RemovedTools, entry) + case "changed": + diff.ChangedTools = append(diff.ChangedTools, entry) + } +} + // computeBashCommandsDiff builds bash-specific analysis from pre-filtered bash tool call maps. // The maps should contain only bash-related entries (generic "bash"/"Bash" and per-command "bash_*"). // Returns nil when no bash tool calls are present in either map. @@ -851,75 +871,62 @@ func computeTokenUsageDiff(tu1, tu2 *TokenUsageSummary) *TokenUsageDiff { return nil } - var ( - run1Input, run2Input int - run1Output, run2Output int - run1CacheRead, run2CacheRead int - run1CacheWrite, run2CacheWrite int - run1AIC, run2AIC float64 - run1Requests, run2Requests int - run1CacheEff, run2CacheEff float64 - ) - - if tu1 != nil { - run1Input = tu1.TotalInputTokens - run1Output = tu1.TotalOutputTokens - run1CacheRead = tu1.TotalCacheReadTokens - run1CacheWrite = tu1.TotalCacheWriteTokens - run1AIC = tu1.TotalAIC - run1Requests = tu1.TotalRequests - run1CacheEff = tu1.CacheEfficiency - } - if tu2 != nil { - run2Input = tu2.TotalInputTokens - run2Output = tu2.TotalOutputTokens - run2CacheRead = tu2.TotalCacheReadTokens - run2CacheWrite = tu2.TotalCacheWriteTokens - run2AIC = tu2.TotalAIC - run2Requests = tu2.TotalRequests - run2CacheEff = tu2.CacheEfficiency - } + run1 := computeTokenUsageDiffValues(tu1) + run2 := computeTokenUsageDiffValues(tu2) diff := &TokenUsageDiff{ - Run1InputTokens: run1Input, - Run2InputTokens: run2Input, - Run1OutputTokens: run1Output, - Run2OutputTokens: run2Output, - Run1CacheReadTokens: run1CacheRead, - Run2CacheReadTokens: run2CacheRead, - Run1CacheWriteTokens: run1CacheWrite, - Run2CacheWriteTokens: run2CacheWrite, - Run1AIC: run1AIC, - Run2AIC: run2AIC, - Run1TotalRequests: run1Requests, - Run2TotalRequests: run2Requests, - Run1CacheEfficiency: run1CacheEff, - Run2CacheEfficiency: run2CacheEff, + Run1InputTokens: run1.input, Run2InputTokens: run2.input, + Run1OutputTokens: run1.output, Run2OutputTokens: run2.output, + Run1CacheReadTokens: run1.cacheRead, Run2CacheReadTokens: run2.cacheRead, + Run1CacheWriteTokens: run1.cacheWrite, Run2CacheWriteTokens: run2.cacheWrite, + Run1AIC: run1.aic, Run2AIC: run2.aic, + Run1TotalRequests: run1.requests, Run2TotalRequests: run2.requests, + Run1CacheEfficiency: run1.cacheEff, Run2CacheEfficiency: run2.cacheEff, + } + computeTokenUsageDiffChanges(diff, run1, run2) + + return diff +} + +type computeTokenUsageDiffSummaryValues struct { + input, output, cacheRead, cacheWrite int + aic, cacheEff float64 + requests int +} + +func computeTokenUsageDiffValues(tu *TokenUsageSummary) computeTokenUsageDiffSummaryValues { + if tu == nil { + return computeTokenUsageDiffSummaryValues{} + } + return computeTokenUsageDiffSummaryValues{ + input: tu.TotalInputTokens, output: tu.TotalOutputTokens, + cacheRead: tu.TotalCacheReadTokens, cacheWrite: tu.TotalCacheWriteTokens, + aic: tu.TotalAIC, requests: tu.TotalRequests, cacheEff: tu.CacheEfficiency, } +} - if run1Input > 0 || run2Input > 0 { - diff.InputTokensChange = formatVolumeChange(run1Input, run2Input) +func computeTokenUsageDiffChanges(diff *TokenUsageDiff, run1, run2 computeTokenUsageDiffSummaryValues) { + if run1.input > 0 || run2.input > 0 { + diff.InputTokensChange = formatVolumeChange(run1.input, run2.input) } - if run1Output > 0 || run2Output > 0 { - diff.OutputTokensChange = formatVolumeChange(run1Output, run2Output) + if run1.output > 0 || run2.output > 0 { + diff.OutputTokensChange = formatVolumeChange(run1.output, run2.output) } - if run1CacheRead > 0 || run2CacheRead > 0 { - diff.CacheReadTokensChange = formatVolumeChange(run1CacheRead, run2CacheRead) + if run1.cacheRead > 0 || run2.cacheRead > 0 { + diff.CacheReadTokensChange = formatVolumeChange(run1.cacheRead, run2.cacheRead) } - if run1CacheWrite > 0 || run2CacheWrite > 0 { - diff.CacheWriteTokensChange = formatVolumeChange(run1CacheWrite, run2CacheWrite) + if run1.cacheWrite > 0 || run2.cacheWrite > 0 { + diff.CacheWriteTokensChange = formatVolumeChange(run1.cacheWrite, run2.cacheWrite) } - if run1AIC > 0 || run2AIC > 0 { - diff.AICChange = formatFloatDelta(run1AIC, run2AIC) + if run1.aic > 0 || run2.aic > 0 { + diff.AICChange = formatFloatDelta(run1.aic, run2.aic) } - if run1Requests > 0 || run2Requests > 0 { - diff.RequestsDelta = formatCountChange(run1Requests, run2Requests) + if run1.requests > 0 || run2.requests > 0 { + diff.RequestsDelta = formatCountChange(run1.requests, run2.requests) } - if run1CacheEff > 0 || run2CacheEff > 0 { - diff.CacheEfficiencyChange = formatPercentagePointChange(run1CacheEff, run2CacheEff) + if run1.cacheEff > 0 || run2.cacheEff > 0 { + diff.CacheEfficiencyChange = formatPercentagePointChange(run1.cacheEff, run2.cacheEff) } - - return diff } // loadRunSummaryForDiff loads or builds a RunSummary for a given run for use in diffing. diff --git a/pkg/cli/audit_diff_command.go b/pkg/cli/audit_diff_command.go index c68a74cf833..33b9a1f3e4b 100644 --- a/pkg/cli/audit_diff_command.go +++ b/pkg/cli/audit_diff_command.go @@ -12,15 +12,7 @@ import ( "github.com/spf13/cobra" ) -// NewAuditDiffSubcommand creates the audit diff subcommand. -// Deprecated: pass multiple run IDs directly to `audit` instead (e.g. `gh aw audit `). -// This subcommand is hidden and kept for backward compatibility only. -func NewAuditDiffSubcommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "diff ...", - Short: "Compare behavior across workflow runs", - Hidden: true, - Long: `Deprecated: pass multiple run IDs directly to the audit command instead. +const auditDiffSubcommandLong = `Deprecated: pass multiple run IDs directly to the audit command instead. gh aw audit ... @@ -40,63 +32,27 @@ analyzes their data, and produces a diff showing: - Anomaly flags (new denied domains, previously-denied now allowed) - MCP tool invocation changes (new/removed tools, call count and error count diffs) - Run metrics comparison (token usage, duration, turns) when cached data is available -- Detailed token usage breakdown (input/output/cache + AI Credits) from firewall proxy`, - Example: ` ` + string(constants.CLIExtensionPrefix) + ` audit diff 12345 12346 # Compare two runs +- Detailed token usage breakdown (input/output/cache + AI Credits) from firewall proxy` + +const auditDiffSubcommandExample = ` ` + string(constants.CLIExtensionPrefix) + ` audit diff 12345 12346 # Compare two runs ` + string(constants.CLIExtensionPrefix) + ` audit diff 12345 12346 12347 12348 # Compare base against 3 runs ` + string(constants.CLIExtensionPrefix) + ` audit diff 12345 12346 --format markdown # Markdown output for PR comments ` + string(constants.CLIExtensionPrefix) + ` audit diff 12345 12346 --json # JSON for CI integration - ` + string(constants.CLIExtensionPrefix) + ` audit diff 12345 12346 --repo owner/repo # Specify repository`, - Args: cobra.MinimumNArgs(2), + ` + string(constants.CLIExtensionPrefix) + ` audit diff 12345 12346 --repo owner/repo # Specify repository` + +// NewAuditDiffSubcommand creates the audit diff subcommand. +// Deprecated: pass multiple run IDs directly to `audit` instead (e.g. `gh aw audit `). +// This subcommand is hidden and kept for backward compatibility only. +func NewAuditDiffSubcommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "diff ...", + Short: "Compare behavior across workflow runs", + Hidden: true, + Long: auditDiffSubcommandLong, + Example: auditDiffSubcommandExample, + Args: cobra.MinimumNArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - baseRunID, err := strconv.ParseInt(args[0], 10, 64) - if err != nil { - return fmt.Errorf("invalid base run ID %q: must be a numeric run ID", args[0]) - } - - compareRunIDs := make([]int64, 0, len(args)-1) - seen := make(map[int64]bool) - for _, arg := range args[1:] { - id, err := strconv.ParseInt(arg, 10, 64) - if err != nil { - return fmt.Errorf("invalid run ID %q: must be a numeric run ID", arg) - } - if id == baseRunID { - return fmt.Errorf("comparison run ID %d is the same as the base run ID: cannot diff a run against itself", id) - } - if seen[id] { - return fmt.Errorf("duplicate comparison run ID %d: each run ID must appear only once", id) - } - seen[id] = true - compareRunIDs = append(compareRunIDs, id) - } - - outputDir, _ := cmd.Flags().GetString("output") - verbose, _ := cmd.Flags().GetBool("verbose") - jsonOutput, _ := cmd.Flags().GetBool("json") - format, _ := cmd.Flags().GetString("format") - repoFlag, _ := cmd.Flags().GetString("repo") - artifacts, _ := cmd.Flags().GetStringSlice("artifacts") - - var owner, repo, hostname string - if repoFlag != "" { - parts := strings.SplitN(repoFlag, "/", 2) - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return fmt.Errorf("invalid repository format '%s': expected 'owner/repo'", repoFlag) - } - owner = parts[0] - repo = parts[1] - } - - return RunAuditDiff(cmd.Context(), baseRunID, compareRunIDs, AuditOptions{ - Owner: owner, - Repo: repo, - Hostname: hostname, - OutputDir: outputDir, - Verbose: verbose, - JSONOutput: jsonOutput, - Format: format, - ArtifactSets: artifacts, - }) + return newAuditDiffSubcommandRunE(cmd, args) }, } @@ -109,6 +65,69 @@ analyzes their data, and produces a diff showing: return cmd } +func newAuditDiffSubcommandRunE(cmd *cobra.Command, args []string) error { + baseRunID, compareRunIDs, err := newAuditDiffSubcommandParseRunIDs(args) + if err != nil { + return err + } + opts, err := newAuditDiffSubcommandOptions(cmd) + if err != nil { + return err + } + return RunAuditDiff(cmd.Context(), baseRunID, compareRunIDs, opts) +} + +func newAuditDiffSubcommandParseRunIDs(args []string) (int64, []int64, error) { + baseRunID, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + return 0, nil, fmt.Errorf("invalid base run ID %q: must be a numeric run ID", args[0]) + } + + compareRunIDs := make([]int64, 0, len(args)-1) + seen := make(map[int64]bool) + for _, arg := range args[1:] { + id, err := strconv.ParseInt(arg, 10, 64) + if err != nil { + return 0, nil, fmt.Errorf("invalid run ID %q: must be a numeric run ID", arg) + } + if id == baseRunID { + return 0, nil, fmt.Errorf("comparison run ID %d is the same as the base run ID: cannot diff a run against itself", id) + } + if seen[id] { + return 0, nil, fmt.Errorf("duplicate comparison run ID %d: each run ID must appear only once", id) + } + seen[id] = true + compareRunIDs = append(compareRunIDs, id) + } + return baseRunID, compareRunIDs, nil +} + +func newAuditDiffSubcommandOptions(cmd *cobra.Command) (AuditOptions, error) { + outputDir, _ := cmd.Flags().GetString("output") + verbose, _ := cmd.Flags().GetBool("verbose") + jsonOutput, _ := cmd.Flags().GetBool("json") + format, _ := cmd.Flags().GetString("format") + repoFlag, _ := cmd.Flags().GetString("repo") + artifacts, _ := cmd.Flags().GetStringSlice("artifacts") + + owner, repo, err := newAuditDiffSubcommandRepo(repoFlag) + if err != nil { + return AuditOptions{}, err + } + return AuditOptions{Owner: owner, Repo: repo, OutputDir: outputDir, Verbose: verbose, JSONOutput: jsonOutput, Format: format, ArtifactSets: artifacts}, nil +} + +func newAuditDiffSubcommandRepo(repoFlag string) (string, string, error) { + if repoFlag == "" { + return "", "", nil + } + parts := strings.SplitN(repoFlag, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid repository format '%s': expected 'owner/repo'", repoFlag) + } + return parts[0], parts[1], nil +} + // RunAuditDiff compares behavior between a base workflow run and one or more comparison runs. // The base run is the reference point; each comparison run is diffed against it independently. func RunAuditDiff(ctx context.Context, baseRunID int64, compareRunIDs []int64, opts AuditOptions) error { @@ -122,9 +141,53 @@ func RunAuditDiff(ctx context.Context, baseRunID int64, compareRunIDs []int64, o auditDiffLog.Printf("Starting audit diff: base=%d, compare=%v", baseRunID, compareRunIDs) + artifactFilter, err := runAuditDiffArtifactFilter(artifactSets, verbose) + if err != nil { + return err + } + + hostname = runAuditDiffHostname(hostname) + + if err := runAuditDiffCheckContext(ctx); err != nil { + return err + } + + if len(compareRunIDs) == 1 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Comparing workflow runs: Run #%d → Run #%d", baseRunID, compareRunIDs[0]))) + } else { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Comparing workflow runs: Run #%d (base) vs %d comparison runs", baseRunID, len(compareRunIDs)))) + } + + // Load base run summary once (shared across all comparisons) + fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Loading data for base run %d...", baseRunID))) + baseSummary, err := loadRunSummaryForDiff(ctx, baseRunID, outputDir, owner, repo, hostname, verbose, artifactFilter) + if err != nil { + return fmt.Errorf("failed to load data for base run %d: %w", baseRunID, err) + } + + diffs, err := runAuditDiffComparisons(runAuditDiffComparisonsParams{ + Ctx: ctx, + BaseRunID: baseRunID, + CompareRunIDs: compareRunIDs, + BaseSummary: baseSummary, + OutputDir: outputDir, + Owner: owner, + Repo: repo, + Hostname: hostname, + Verbose: verbose, + ArtifactFilter: artifactFilter, + }) + if err != nil { + return err + } + + return runAuditDiffRender(diffs, opts, format) +} + +func runAuditDiffArtifactFilter(artifactSets []string, verbose bool) ([]string, error) { // Validate and resolve artifact sets into a concrete filter. if err := ValidateArtifactSets(artifactSets); err != nil { - return err + return nil, err } artifactFilter := ResolveArtifactFilter(artifactSets) if len(artifactFilter) > 0 { @@ -133,82 +196,84 @@ func RunAuditDiff(ctx context.Context, baseRunID int64, compareRunIDs []int64, o fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Artifact filter: downloading only "+strings.Join(artifactFilter, ", "))) } } + return artifactFilter, nil +} +func runAuditDiffHostname(hostname string) string { // Auto-detect GHES host from git remote if hostname is not provided - if hostname == "" { - hostname = getHostFromOriginRemote() - if hostname != "github.com" { - auditDiffLog.Printf("Auto-detected GHES host from git remote: %s", hostname) - } + if hostname != "" { + return hostname + } + hostname = getHostFromOriginRemote() + if hostname != "github.com" { + auditDiffLog.Printf("Auto-detected GHES host from git remote: %s", hostname) } + return hostname +} - // Check context cancellation +func runAuditDiffCheckContext(ctx context.Context) error { select { case <-ctx.Done(): fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) return ctx.Err() default: + return nil } +} - if len(compareRunIDs) == 1 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Comparing workflow runs: Run #%d → Run #%d", baseRunID, compareRunIDs[0]))) - } else { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Comparing workflow runs: Run #%d (base) vs %d comparison runs", baseRunID, len(compareRunIDs)))) - } - - // Load base run summary once (shared across all comparisons) - fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Loading data for base run %d...", baseRunID))) - baseSummary, err := loadRunSummaryForDiff(ctx, baseRunID, outputDir, owner, repo, hostname, verbose, artifactFilter) - if err != nil { - return fmt.Errorf("failed to load data for base run %d: %w", baseRunID, err) - } - - diffs := make([]*AuditDiff, 0, len(compareRunIDs)) +type runAuditDiffComparisonsParams struct { + Ctx context.Context + BaseRunID int64 + CompareRunIDs []int64 + BaseSummary *RunSummary + OutputDir string + Owner string + Repo string + Hostname string + Verbose bool + ArtifactFilter []string +} - for _, compareRunID := range compareRunIDs { - // Check context cancellation between downloads - select { - case <-ctx.Done(): - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) - return ctx.Err() - default: +func runAuditDiffComparisons(p runAuditDiffComparisonsParams) ([]*AuditDiff, error) { + diffs := make([]*AuditDiff, 0, len(p.CompareRunIDs)) + for _, compareRunID := range p.CompareRunIDs { + if err := runAuditDiffCheckContext(p.Ctx); err != nil { + return nil, err } - fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Loading data for run %d...", compareRunID))) - compareSummary, err := loadRunSummaryForDiff(ctx, compareRunID, outputDir, owner, repo, hostname, verbose, artifactFilter) + compareSummary, err := loadRunSummaryForDiff(p.Ctx, compareRunID, p.OutputDir, p.Owner, p.Repo, p.Hostname, p.Verbose, p.ArtifactFilter) if err != nil { - return fmt.Errorf("failed to load data for run %d: %w", compareRunID, err) - } - - // Warn if no firewall data found for this pair - fw1 := baseSummary.FirewallAnalysis - fw2 := compareSummary.FirewallAnalysis - if fw1 == nil && fw2 == nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No firewall data found for run pair %d→%d. Both runs may predate firewall logging.", baseRunID, compareRunID))) - } else { - if fw1 == nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No firewall data found for base run %d (older run may lack firewall logs)", baseRunID))) - } - if fw2 == nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No firewall data found for run %d", compareRunID))) - } + return nil, fmt.Errorf("failed to load data for run %d: %w", compareRunID, err) } + runAuditDiffWarnFirewall(p.BaseRunID, compareRunID, p.BaseSummary, compareSummary) + diffs = append(diffs, computeAuditDiff(p.BaseRunID, compareRunID, p.BaseSummary, compareSummary)) + } + return diffs, nil +} - diff := computeAuditDiff(baseRunID, compareRunID, baseSummary, compareSummary) - diffs = append(diffs, diff) +func runAuditDiffWarnFirewall(baseRunID, compareRunID int64, baseSummary, compareSummary *RunSummary) { + fw1 := baseSummary.FirewallAnalysis + fw2 := compareSummary.FirewallAnalysis + if fw1 == nil && fw2 == nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No firewall data found for run pair %d→%d. Both runs may predate firewall logging.", baseRunID, compareRunID))) + return + } + if fw1 == nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No firewall data found for base run %d (older run may lack firewall logs)", baseRunID))) + } + if fw2 == nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No firewall data found for run %d", compareRunID))) } +} - // Render output +func runAuditDiffRender(diffs []*AuditDiff, opts AuditOptions, format string) error { if opts.JSONOutput || format == "json" { return renderAuditDiffJSON(diffs) } - if format == "markdown" { renderAuditDiffMarkdown(diffs) return nil } - - // Default: pretty console output renderAuditDiffPretty(diffs) return nil } diff --git a/pkg/cli/audit_diff_render.go b/pkg/cli/audit_diff_render.go index ce55627060e..b54811de7fb 100644 --- a/pkg/cli/audit_diff_render.go +++ b/pkg/cli/audit_diff_render.go @@ -82,10 +82,24 @@ func renderSingleAuditDiffPretty(diff *AuditDiff) { return } - // Collect top-level summary across all sections + summaryParts, anomalyCount := renderSingleAuditDiffPrettySummary(diff) + + if len(summaryParts) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Changes: "+strings.Join(summaryParts, " | "))) + } + if anomalyCount > 0 { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("⚠️ %d anomalies detected", anomalyCount))) + } + fmt.Fprintln(os.Stderr) + + renderFirewallDiffPrettySection(diff.FirewallDiff) + renderMCPToolsDiffPrettySection(diff.MCPToolsDiff) + renderRunMetricsDiffPrettySection(diff.Run1ID, diff.Run2ID, diff.RunMetricsDiff) +} + +func renderSingleAuditDiffPrettySummary(diff *AuditDiff) ([]string, int) { var summaryParts []string anomalyCount := 0 - if diff.FirewallDiff != nil && !isEmptyFirewallDiff(diff.FirewallDiff) { fwParts := []string{} if len(diff.FirewallDiff.NewDomains) > 0 { @@ -105,35 +119,28 @@ func renderSingleAuditDiffPretty(diff *AuditDiff) { } anomalyCount += diff.FirewallDiff.Summary.AnomalyCount } + return renderSingleAuditDiffPrettyMCPSummary(diff, summaryParts, anomalyCount) +} - if diff.MCPToolsDiff != nil && !isEmptyMCPToolsDiff(diff.MCPToolsDiff) { - mcpParts := []string{} - if diff.MCPToolsDiff.Summary.NewToolCount > 0 { - mcpParts = append(mcpParts, fmt.Sprintf("%d new tools", diff.MCPToolsDiff.Summary.NewToolCount)) - } - if diff.MCPToolsDiff.Summary.RemovedToolCount > 0 { - mcpParts = append(mcpParts, fmt.Sprintf("%d removed tools", diff.MCPToolsDiff.Summary.RemovedToolCount)) - } - if diff.MCPToolsDiff.Summary.ChangedToolCount > 0 { - mcpParts = append(mcpParts, fmt.Sprintf("%d changed tools", diff.MCPToolsDiff.Summary.ChangedToolCount)) - } - if len(mcpParts) > 0 { - summaryParts = append(summaryParts, "MCP tools: "+strings.Join(mcpParts, ", ")) - } - anomalyCount += diff.MCPToolsDiff.Summary.AnomalyCount +func renderSingleAuditDiffPrettyMCPSummary(diff *AuditDiff, summaryParts []string, anomalyCount int) ([]string, int) { + if diff.MCPToolsDiff == nil || isEmptyMCPToolsDiff(diff.MCPToolsDiff) { + return summaryParts, anomalyCount } - - if len(summaryParts) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Changes: "+strings.Join(summaryParts, " | "))) + mcpParts := []string{} + if diff.MCPToolsDiff.Summary.NewToolCount > 0 { + mcpParts = append(mcpParts, fmt.Sprintf("%d new tools", diff.MCPToolsDiff.Summary.NewToolCount)) } - if anomalyCount > 0 { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("⚠️ %d anomalies detected", anomalyCount))) + if diff.MCPToolsDiff.Summary.RemovedToolCount > 0 { + mcpParts = append(mcpParts, fmt.Sprintf("%d removed tools", diff.MCPToolsDiff.Summary.RemovedToolCount)) } - fmt.Fprintln(os.Stderr) - - renderFirewallDiffPrettySection(diff.FirewallDiff) - renderMCPToolsDiffPrettySection(diff.MCPToolsDiff) - renderRunMetricsDiffPrettySection(diff.Run1ID, diff.Run2ID, diff.RunMetricsDiff) + if diff.MCPToolsDiff.Summary.ChangedToolCount > 0 { + mcpParts = append(mcpParts, fmt.Sprintf("%d changed tools", diff.MCPToolsDiff.Summary.ChangedToolCount)) + } + if len(mcpParts) > 0 { + summaryParts = append(summaryParts, "MCP tools: "+strings.Join(mcpParts, ", ")) + } + anomalyCount += diff.MCPToolsDiff.Summary.AnomalyCount + return summaryParts, anomalyCount } // renderFirewallDiffMarkdownSection renders the firewall diff sub-section as markdown @@ -307,78 +314,62 @@ func renderFirewallDiffPrettySection(diff *FirewallDiff) { fmt.Fprintln(os.Stderr, console.FormatSectionHeader("Firewall Changes")) fmt.Fprintln(os.Stderr) - if len(diff.NewDomains) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("New Domains (%d)", len(diff.NewDomains)))) - config := console.TableConfig{ - Headers: []string{"Domain", "Status", "Requests", "Anomaly"}, - Rows: make([][]string, 0, len(diff.NewDomains)), - } - for _, entry := range diff.NewDomains { - total := entry.Run2Allowed + entry.Run2Blocked - anomalyNote := formatAnomalyNote(entry.IsAnomaly, entry.AnomalyNote) - config.Rows = append(config.Rows, []string{ - entry.Domain, - firewallStatusEmoji(entry.Run2Status) + " " + entry.Run2Status, - strconv.Itoa(total), - anomalyNote, - }) - } - fmt.Fprint(os.Stderr, console.RenderTable(config)) + renderFirewallDiffPrettyNewDomains(diff.NewDomains) + renderFirewallDiffPrettyRemovedDomains(diff.RemovedDomains) + renderFirewallDiffPrettyStatusChanges(diff.StatusChanges) + renderFirewallDiffPrettyVolumeChanges(diff.VolumeChanges) +} + +func renderFirewallDiffPrettyNewDomains(entries []DomainDiffEntry) { + if len(entries) == 0 { + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("New Domains (%d)", len(entries)))) + config := console.TableConfig{Headers: []string{"Domain", "Status", "Requests", "Anomaly"}, Rows: make([][]string, 0, len(entries))} + for _, entry := range entries { + total := entry.Run2Allowed + entry.Run2Blocked + config.Rows = append(config.Rows, []string{entry.Domain, firewallStatusEmoji(entry.Run2Status) + " " + entry.Run2Status, strconv.Itoa(total), formatAnomalyNote(entry.IsAnomaly, entry.AnomalyNote)}) } + fmt.Fprint(os.Stderr, console.RenderTable(config)) +} - if len(diff.RemovedDomains) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Removed Domains (%d)", len(diff.RemovedDomains)))) - config := console.TableConfig{ - Headers: []string{"Domain", "Previous Status", "Previous Requests"}, - Rows: make([][]string, 0, len(diff.RemovedDomains)), - } - for _, entry := range diff.RemovedDomains { - total := entry.Run1Allowed + entry.Run1Blocked - config.Rows = append(config.Rows, []string{ - entry.Domain, - firewallStatusEmoji(entry.Run1Status) + " " + entry.Run1Status, - strconv.Itoa(total), - }) - } - fmt.Fprint(os.Stderr, console.RenderTable(config)) +func renderFirewallDiffPrettyRemovedDomains(entries []DomainDiffEntry) { + if len(entries) == 0 { + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Removed Domains (%d)", len(entries)))) + config := console.TableConfig{Headers: []string{"Domain", "Previous Status", "Previous Requests"}, Rows: make([][]string, 0, len(entries))} + for _, entry := range entries { + total := entry.Run1Allowed + entry.Run1Blocked + config.Rows = append(config.Rows, []string{entry.Domain, firewallStatusEmoji(entry.Run1Status) + " " + entry.Run1Status, strconv.Itoa(total)}) } + fmt.Fprint(os.Stderr, console.RenderTable(config)) +} - if len(diff.StatusChanges) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Status Changes (%d)", len(diff.StatusChanges)))) - config := console.TableConfig{ - Headers: []string{"Domain", "Before", "After", "Anomaly"}, - Rows: make([][]string, 0, len(diff.StatusChanges)), - } - for _, entry := range diff.StatusChanges { - anomalyNote := formatAnomalyNote(entry.IsAnomaly, entry.AnomalyNote) - config.Rows = append(config.Rows, []string{ - entry.Domain, - firewallStatusEmoji(entry.Run1Status) + " " + entry.Run1Status, - firewallStatusEmoji(entry.Run2Status) + " " + entry.Run2Status, - anomalyNote, - }) - } - fmt.Fprint(os.Stderr, console.RenderTable(config)) +func renderFirewallDiffPrettyStatusChanges(entries []DomainDiffEntry) { + if len(entries) == 0 { + return } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Status Changes (%d)", len(entries)))) + config := console.TableConfig{Headers: []string{"Domain", "Before", "After", "Anomaly"}, Rows: make([][]string, 0, len(entries))} + for _, entry := range entries { + config.Rows = append(config.Rows, []string{entry.Domain, firewallStatusEmoji(entry.Run1Status) + " " + entry.Run1Status, firewallStatusEmoji(entry.Run2Status) + " " + entry.Run2Status, formatAnomalyNote(entry.IsAnomaly, entry.AnomalyNote)}) + } + fmt.Fprint(os.Stderr, console.RenderTable(config)) +} - if len(diff.VolumeChanges) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Volume Changes")) - config := console.TableConfig{ - Headers: []string{"Domain", "Requests (before)", "Requests (after)", "Change"}, - Rows: make([][]string, 0, len(diff.VolumeChanges)), - } - for _, entry := range diff.VolumeChanges { - total1 := entry.Run1Allowed + entry.Run1Blocked - total2 := entry.Run2Allowed + entry.Run2Blocked - config.Rows = append(config.Rows, []string{ - entry.Domain, - strconv.Itoa(total1), - strconv.Itoa(total2), - entry.VolumeChange, - }) - } - fmt.Fprint(os.Stderr, console.RenderTable(config)) +func renderFirewallDiffPrettyVolumeChanges(entries []DomainDiffEntry) { + if len(entries) == 0 { + return } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Volume Changes")) + config := console.TableConfig{Headers: []string{"Domain", "Requests (before)", "Requests (after)", "Change"}, Rows: make([][]string, 0, len(entries))} + for _, entry := range entries { + total1 := entry.Run1Allowed + entry.Run1Blocked + total2 := entry.Run2Allowed + entry.Run2Blocked + config.Rows = append(config.Rows, []string{entry.Domain, strconv.Itoa(total1), strconv.Itoa(total2), entry.VolumeChange}) + } + fmt.Fprint(os.Stderr, console.RenderTable(config)) } // renderMCPToolsDiffPrettySection renders the MCP tools diff as a pretty console sub-section @@ -390,61 +381,45 @@ func renderMCPToolsDiffPrettySection(diff *MCPToolsDiff) { fmt.Fprintln(os.Stderr, console.FormatSectionHeader("MCP Tool Changes")) fmt.Fprintln(os.Stderr) - if len(diff.NewTools) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("New Tools (%d)", len(diff.NewTools)))) - config := console.TableConfig{ - Headers: []string{"Server", "Tool", "Calls", "Anomaly"}, - Rows: make([][]string, 0, len(diff.NewTools)), - } - for _, entry := range diff.NewTools { - anomalyNote := formatAnomalyNote(entry.IsAnomaly, entry.AnomalyNote) - config.Rows = append(config.Rows, []string{ - entry.ServerName, - entry.ToolName, - strconv.Itoa(entry.Run2CallCount), - anomalyNote, - }) - } - fmt.Fprint(os.Stderr, console.RenderTable(config)) + renderMCPToolsDiffPrettyNewTools(diff.NewTools) + renderMCPToolsDiffPrettyRemovedTools(diff.RemovedTools) + renderMCPToolsDiffPrettyChangedTools(diff.ChangedTools) +} + +func renderMCPToolsDiffPrettyNewTools(entries []MCPToolDiffEntry) { + if len(entries) == 0 { + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("New Tools (%d)", len(entries)))) + config := console.TableConfig{Headers: []string{"Server", "Tool", "Calls", "Anomaly"}, Rows: make([][]string, 0, len(entries))} + for _, entry := range entries { + config.Rows = append(config.Rows, []string{entry.ServerName, entry.ToolName, strconv.Itoa(entry.Run2CallCount), formatAnomalyNote(entry.IsAnomaly, entry.AnomalyNote)}) } + fmt.Fprint(os.Stderr, console.RenderTable(config)) +} - if len(diff.RemovedTools) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Removed Tools (%d)", len(diff.RemovedTools)))) - config := console.TableConfig{ - Headers: []string{"Server", "Tool", "Previous Calls"}, - Rows: make([][]string, 0, len(diff.RemovedTools)), - } - for _, entry := range diff.RemovedTools { - config.Rows = append(config.Rows, []string{ - entry.ServerName, - entry.ToolName, - strconv.Itoa(entry.Run1CallCount), - }) - } - fmt.Fprint(os.Stderr, console.RenderTable(config)) +func renderMCPToolsDiffPrettyRemovedTools(entries []MCPToolDiffEntry) { + if len(entries) == 0 { + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Removed Tools (%d)", len(entries)))) + config := console.TableConfig{Headers: []string{"Server", "Tool", "Previous Calls"}, Rows: make([][]string, 0, len(entries))} + for _, entry := range entries { + config.Rows = append(config.Rows, []string{entry.ServerName, entry.ToolName, strconv.Itoa(entry.Run1CallCount)}) } + fmt.Fprint(os.Stderr, console.RenderTable(config)) +} - if len(diff.ChangedTools) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Changed Tools (%d)", len(diff.ChangedTools)))) - config := console.TableConfig{ - Headers: []string{"Server", "Tool", "Calls (before)", "Calls (after)", "Change", "Errors (before)", "Errors (after)", "Anomaly"}, - Rows: make([][]string, 0, len(diff.ChangedTools)), - } - for _, entry := range diff.ChangedTools { - anomalyNote := formatAnomalyNote(entry.IsAnomaly, entry.AnomalyNote) - config.Rows = append(config.Rows, []string{ - entry.ServerName, - entry.ToolName, - strconv.Itoa(entry.Run1CallCount), - strconv.Itoa(entry.Run2CallCount), - entry.CallCountChange, - strconv.Itoa(entry.Run1ErrorCount), - strconv.Itoa(entry.Run2ErrorCount), - anomalyNote, - }) - } - fmt.Fprint(os.Stderr, console.RenderTable(config)) +func renderMCPToolsDiffPrettyChangedTools(entries []MCPToolDiffEntry) { + if len(entries) == 0 { + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Changed Tools (%d)", len(entries)))) + config := console.TableConfig{Headers: []string{"Server", "Tool", "Calls (before)", "Calls (after)", "Change", "Errors (before)", "Errors (after)", "Anomaly"}, Rows: make([][]string, 0, len(entries))} + for _, entry := range entries { + config.Rows = append(config.Rows, []string{entry.ServerName, entry.ToolName, strconv.Itoa(entry.Run1CallCount), strconv.Itoa(entry.Run2CallCount), entry.CallCountChange, strconv.Itoa(entry.Run1ErrorCount), strconv.Itoa(entry.Run2ErrorCount), formatAnomalyNote(entry.IsAnomaly, entry.AnomalyNote)}) } + fmt.Fprint(os.Stderr, console.RenderTable(config)) } // renderRunMetricsDiffPrettySection renders the run metrics diff as a pretty console sub-section @@ -461,38 +436,7 @@ func renderRunMetricsDiffPrettySection(run1ID, run2ID int64, diff *RunMetricsDif Rows: make([][]string, 0), } - if diff.Run1TokenUsage > 0 || diff.Run2TokenUsage > 0 { - config.Rows = append(config.Rows, []string{ - "Token usage", - strconv.Itoa(diff.Run1TokenUsage), - strconv.Itoa(diff.Run2TokenUsage), - diff.TokenUsageChange, - }) - } - if diff.Run1Duration != "" || diff.Run2Duration != "" { - config.Rows = append(config.Rows, []string{ - "Duration", - diff.Run1Duration, - diff.Run2Duration, - diff.DurationChange, - }) - } - if diff.Run1Turns > 0 || diff.Run2Turns > 0 { - config.Rows = append(config.Rows, []string{ - "Turns", - strconv.Itoa(diff.Run1Turns), - strconv.Itoa(diff.Run2Turns), - fmt.Sprintf("%+d", diff.TurnsChange), - }) - } - if diff.Run1TokensPerTurn > 0 || diff.Run2TokensPerTurn > 0 { - config.Rows = append(config.Rows, []string{ - "Tokens / turn", - strconv.Itoa(diff.Run1TokensPerTurn), - strconv.Itoa(diff.Run2TokensPerTurn), - diff.TokensPerTurnChange, - }) - } + renderRunMetricsDiffPrettyRows(&config, diff) if len(config.Rows) > 0 { fmt.Fprint(os.Stderr, console.RenderTable(config)) @@ -512,6 +456,21 @@ func renderRunMetricsDiffPrettySection(run1ID, run2ID int64, diff *RunMetricsDif } } +func renderRunMetricsDiffPrettyRows(config *console.TableConfig, diff *RunMetricsDiff) { + if diff.Run1TokenUsage > 0 || diff.Run2TokenUsage > 0 { + config.Rows = append(config.Rows, []string{"Token usage", strconv.Itoa(diff.Run1TokenUsage), strconv.Itoa(diff.Run2TokenUsage), diff.TokenUsageChange}) + } + if diff.Run1Duration != "" || diff.Run2Duration != "" { + config.Rows = append(config.Rows, []string{"Duration", diff.Run1Duration, diff.Run2Duration, diff.DurationChange}) + } + if diff.Run1Turns > 0 || diff.Run2Turns > 0 { + config.Rows = append(config.Rows, []string{"Turns", strconv.Itoa(diff.Run1Turns), strconv.Itoa(diff.Run2Turns), fmt.Sprintf("%+d", diff.TurnsChange)}) + } + if diff.Run1TokensPerTurn > 0 || diff.Run2TokensPerTurn > 0 { + config.Rows = append(config.Rows, []string{"Tokens / turn", strconv.Itoa(diff.Run1TokensPerTurn), strconv.Itoa(diff.Run2TokensPerTurn), diff.TokensPerTurnChange}) + } +} + // renderTokenUsageDiffPrettySection renders detailed token usage as a pretty console sub-section func renderTokenUsageDiffPrettySection(run1ID, run2ID int64, diff *TokenUsageDiff) { fmt.Fprintln(os.Stderr, console.FormatSectionHeader("Token Usage Details")) @@ -522,65 +481,34 @@ func renderTokenUsageDiffPrettySection(run1ID, run2ID int64, diff *TokenUsageDif Rows: make([][]string, 0), } + renderTokenUsageDiffPrettyRows(&config, diff) + + if len(config.Rows) > 0 { + fmt.Fprint(os.Stderr, console.RenderTable(config)) + } +} + +func renderTokenUsageDiffPrettyRows(config *console.TableConfig, diff *TokenUsageDiff) { if diff.Run1InputTokens > 0 || diff.Run2InputTokens > 0 { - config.Rows = append(config.Rows, []string{ - "Input", - strconv.Itoa(diff.Run1InputTokens), - strconv.Itoa(diff.Run2InputTokens), - diff.InputTokensChange, - }) + config.Rows = append(config.Rows, []string{"Input", strconv.Itoa(diff.Run1InputTokens), strconv.Itoa(diff.Run2InputTokens), diff.InputTokensChange}) } if diff.Run1OutputTokens > 0 || diff.Run2OutputTokens > 0 { - config.Rows = append(config.Rows, []string{ - "Output", - strconv.Itoa(diff.Run1OutputTokens), - strconv.Itoa(diff.Run2OutputTokens), - diff.OutputTokensChange, - }) + config.Rows = append(config.Rows, []string{"Output", strconv.Itoa(diff.Run1OutputTokens), strconv.Itoa(diff.Run2OutputTokens), diff.OutputTokensChange}) } if diff.Run1CacheReadTokens > 0 || diff.Run2CacheReadTokens > 0 { - config.Rows = append(config.Rows, []string{ - "Cache read", - strconv.Itoa(diff.Run1CacheReadTokens), - strconv.Itoa(diff.Run2CacheReadTokens), - diff.CacheReadTokensChange, - }) + config.Rows = append(config.Rows, []string{"Cache read", strconv.Itoa(diff.Run1CacheReadTokens), strconv.Itoa(diff.Run2CacheReadTokens), diff.CacheReadTokensChange}) } if diff.Run1CacheWriteTokens > 0 || diff.Run2CacheWriteTokens > 0 { - config.Rows = append(config.Rows, []string{ - "Cache write", - strconv.Itoa(diff.Run1CacheWriteTokens), - strconv.Itoa(diff.Run2CacheWriteTokens), - diff.CacheWriteTokensChange, - }) + config.Rows = append(config.Rows, []string{"Cache write", strconv.Itoa(diff.Run1CacheWriteTokens), strconv.Itoa(diff.Run2CacheWriteTokens), diff.CacheWriteTokensChange}) } if diff.Run1AIC > 0 || diff.Run2AIC > 0 { - config.Rows = append(config.Rows, []string{ - "AI Credits", - fmt.Sprintf("%.3f", diff.Run1AIC), - fmt.Sprintf("%.3f", diff.Run2AIC), - diff.AICChange, - }) + config.Rows = append(config.Rows, []string{"AI Credits", fmt.Sprintf("%.3f", diff.Run1AIC), fmt.Sprintf("%.3f", diff.Run2AIC), diff.AICChange}) } if diff.Run1TotalRequests > 0 || diff.Run2TotalRequests > 0 { - config.Rows = append(config.Rows, []string{ - "API requests", - strconv.Itoa(diff.Run1TotalRequests), - strconv.Itoa(diff.Run2TotalRequests), - diff.RequestsDelta, - }) + config.Rows = append(config.Rows, []string{"API requests", strconv.Itoa(diff.Run1TotalRequests), strconv.Itoa(diff.Run2TotalRequests), diff.RequestsDelta}) } if diff.Run1CacheEfficiency > 0 || diff.Run2CacheEfficiency > 0 { - config.Rows = append(config.Rows, []string{ - "Cache efficiency", - fmt.Sprintf("%.1f%%", diff.Run1CacheEfficiency*100), - fmt.Sprintf("%.1f%%", diff.Run2CacheEfficiency*100), - diff.CacheEfficiencyChange, - }) - } - - if len(config.Rows) > 0 { - fmt.Fprint(os.Stderr, console.RenderTable(config)) + config.Rows = append(config.Rows, []string{"Cache efficiency", fmt.Sprintf("%.1f%%", diff.Run1CacheEfficiency*100), fmt.Sprintf("%.1f%%", diff.Run2CacheEfficiency*100), diff.CacheEfficiencyChange}) } } diff --git a/pkg/cli/audit_expanded.go b/pkg/cli/audit_expanded.go index 42d0fac19a6..a4da97b8441 100644 --- a/pkg/cli/audit_expanded.go +++ b/pkg/cli/audit_expanded.go @@ -304,47 +304,53 @@ func buildSessionAnalysis(processedRun ProcessedRun, metrics LogMetrics) *Sessio session.WallTime = timeutil.FormatDuration(run.Duration) } + buildSessionAnalysisTiming(session, run, metrics) + buildSessionAnalysisTokenRate(session, run, metrics) + buildSessionAnalysisTimeout(session, processedRun) + + auditExpandedLog.Printf("Built session analysis: turns=%d, wall_time=%s, avg_tbt=%s, max_tbt=%s, timeout=%v", + session.TurnCount, session.WallTime, session.AvgTimeBetweenTurns, session.MaxTimeBetweenTurns, session.TimeoutDetected) + return session +} + +func buildSessionAnalysisTiming(session *SessionAnalysis, run WorkflowRun, metrics LogMetrics) { // Average turn duration if metrics.Turns > 0 && run.Duration > 0 { avgTurnDuration := run.Duration / (time.Duration(metrics.Turns)) session.AvgTurnDuration = timeutil.FormatDuration(avgTurnDuration) } + buildSessionAnalysisTimeBetweenTurns(session, run, metrics) +} +func buildSessionAnalysisTimeBetweenTurns(session *SessionAnalysis, run WorkflowRun, metrics LogMetrics) { // Time Between Turns (TBT): prefer precise per-turn timestamps from log metrics; // fall back to wall-time / turns when timestamps are unavailable. - // TBT measures the gap between consecutive LLM API calls (tool execution overhead). - // Anthropic's prompt cache TTL is 5 minutes — if TBT exceeds this, cache entries - // expire and every turn incurs full prompt re-processing costs. const anthropicCacheTTL = 5 * time.Minute if metrics.AvgTimeBetweenTurns > 0 { session.AvgTimeBetweenTurns = timeutil.FormatDuration(metrics.AvgTimeBetweenTurns) if metrics.MaxTimeBetweenTurns > 0 { session.MaxTimeBetweenTurns = timeutil.FormatDuration(metrics.MaxTimeBetweenTurns) } - // Warn when the maximum observed TBT exceeds the Anthropic cache TTL. - if metrics.MaxTimeBetweenTurns > anthropicCacheTTL { - session.CacheWarning = fmt.Sprintf( - "Max TBT (%s) exceeds Anthropic 5-min cache TTL — prompt cache will expire between turns, increasing cost", - timeutil.FormatDuration(metrics.MaxTimeBetweenTurns), - ) - } else if metrics.AvgTimeBetweenTurns > anthropicCacheTTL { - session.CacheWarning = fmt.Sprintf( - "Avg TBT (%s) exceeds Anthropic 5-min cache TTL — prompt cache likely expiring between turns", - timeutil.FormatDuration(metrics.AvgTimeBetweenTurns), - ) - } + buildSessionAnalysisCacheWarning(session, metrics.AvgTimeBetweenTurns, metrics.MaxTimeBetweenTurns, anthropicCacheTTL) } else if metrics.Turns > 1 && run.Duration > 0 { - // Fallback: estimate TBT from wall time over turns-1 intervals. avgTBT := run.Duration / time.Duration(metrics.Turns-1) session.AvgTimeBetweenTurns = timeutil.FormatDuration(avgTBT) + " (estimated)" if avgTBT > anthropicCacheTTL { - session.CacheWarning = fmt.Sprintf( - "Estimated avg TBT (%s) exceeds Anthropic 5-min cache TTL — prompt cache likely expiring between turns", - timeutil.FormatDuration(avgTBT), - ) + session.CacheWarning = fmt.Sprintf("Estimated avg TBT (%s) exceeds Anthropic 5-min cache TTL — prompt cache likely expiring between turns", timeutil.FormatDuration(avgTBT)) } } +} + +func buildSessionAnalysisCacheWarning(session *SessionAnalysis, avgTBT, maxTBT, cacheTTL time.Duration) { + // Warn when the maximum observed TBT exceeds the Anthropic cache TTL. + if maxTBT > cacheTTL { + session.CacheWarning = fmt.Sprintf("Max TBT (%s) exceeds Anthropic 5-min cache TTL — prompt cache will expire between turns, increasing cost", timeutil.FormatDuration(maxTBT)) + } else if avgTBT > cacheTTL { + session.CacheWarning = fmt.Sprintf("Avg TBT (%s) exceeds Anthropic 5-min cache TTL — prompt cache likely expiring between turns", timeutil.FormatDuration(avgTBT)) + } +} +func buildSessionAnalysisTokenRate(session *SessionAnalysis, run WorkflowRun, metrics LogMetrics) { // Tokens per minute if metrics.TokenUsage > 0 && run.Duration > 0 { minutes := run.Duration.Minutes() @@ -352,9 +358,11 @@ func buildSessionAnalysis(processedRun ProcessedRun, metrics LogMetrics) *Sessio session.TokensPerMinute = float64(metrics.TokenUsage) / minutes } } +} +func buildSessionAnalysisTimeout(session *SessionAnalysis, processedRun ProcessedRun) { // Timeout detection: check if the run was cancelled (typically indicates timeout) - if run.Conclusion == "cancelled" || run.Conclusion == "timed_out" { + if processedRun.Run.Conclusion == "cancelled" || processedRun.Run.Conclusion == "timed_out" { session.TimeoutDetected = true } @@ -365,10 +373,6 @@ func buildSessionAnalysis(processedRun ProcessedRun, metrics LogMetrics) *Sessio break } } - - auditExpandedLog.Printf("Built session analysis: turns=%d, wall_time=%s, avg_tbt=%s, max_tbt=%s, timeout=%v", - session.TurnCount, session.WallTime, session.AvgTimeBetweenTurns, session.MaxTimeBetweenTurns, session.TimeoutDetected) - return session } // buildSafeOutputSummary creates a summary of safe output items by type @@ -473,47 +477,54 @@ func buildMCPServerHealth(mcpToolUsage *MCPToolUsageData, mcpFailures []MCPFailu health := &MCPServerHealth{} - // Track failed servers from MCPFailures - failedServers := make(map[string]struct { - }) - for _, failure := range mcpFailures { - failedServers[failure.ServerName] = struct { - }{} - } + failedServers := buildMCPServerHealthFailedServers(mcpFailures) health.FailedSvrs = len(failedServers) // Process server statistics from mcpToolUsage if mcpToolUsage != nil { - for _, server := range mcpToolUsage.Servers { - health.TotalRequests += server.RequestCount - health.TotalErrors += server.ErrorCount + buildMCPServerHealthStats(health, mcpToolUsage, failedServers) + } - errorRate := safePercent(server.ErrorCount, server.RequestCount) + buildMCPServerHealthMissingFailures(health, failedServers) - status := "✅ healthy" - if _, isFailed := failedServers[server.ServerName]; isFailed { - status = "❌ failed" - } else if errorRate > 10 { - status = "⚠️ degraded" - } + health.TotalServers = len(health.Servers) - health.Servers = append(health.Servers, MCPServerHealthDetail{ - ServerName: server.ServerName, - RequestCount: server.RequestCount, - ToolCalls: server.ToolCallCount, - ErrorCount: server.ErrorCount, - ErrorRate: errorRate, - ErrorRateStr: fmt.Sprintf("%.1f%%", errorRate), - AvgLatency: server.AvgDuration, - Status: status, - }) - } + buildMCPServerHealthFinalize(health) + + auditExpandedLog.Printf("Built MCP server health: %s, total_requests=%d, error_rate=%.1f%%", + health.Summary, health.TotalRequests, health.ErrorRate) + return health +} - // Build slowest tool calls from individual call records (top 5) - health.SlowestCalls = buildSlowestToolCalls(mcpToolUsage.ToolCalls, 5) +func buildMCPServerHealthFailedServers(mcpFailures []MCPFailureReport) map[string]struct{} { + failedServers := make(map[string]struct{}) + for _, failure := range mcpFailures { + failedServers[failure.ServerName] = struct{}{} } + return failedServers +} - // Add failed servers that don't appear in stats +func buildMCPServerHealthStats(health *MCPServerHealth, mcpToolUsage *MCPToolUsageData, failedServers map[string]struct{}) { + for _, server := range mcpToolUsage.Servers { + health.TotalRequests += server.RequestCount + health.TotalErrors += server.ErrorCount + errorRate := safePercent(server.ErrorCount, server.RequestCount) + status := "✅ healthy" + if _, isFailed := failedServers[server.ServerName]; isFailed { + status = "❌ failed" + } else if errorRate > 10 { + status = "⚠️ degraded" + } + health.Servers = append(health.Servers, MCPServerHealthDetail{ + ServerName: server.ServerName, RequestCount: server.RequestCount, ToolCalls: server.ToolCallCount, + ErrorCount: server.ErrorCount, ErrorRate: errorRate, ErrorRateStr: fmt.Sprintf("%.1f%%", errorRate), + AvgLatency: server.AvgDuration, Status: status, + }) + } + health.SlowestCalls = buildSlowestToolCalls(mcpToolUsage.ToolCalls, 5) +} + +func buildMCPServerHealthMissingFailures(health *MCPServerHealth, failedServers map[string]struct{}) { for serverName := range failedServers { found := false for _, s := range health.Servers { @@ -523,16 +534,12 @@ func buildMCPServerHealth(mcpToolUsage *MCPToolUsageData, mcpFailures []MCPFailu } } if !found { - health.Servers = append(health.Servers, MCPServerHealthDetail{ - ServerName: serverName, - Status: "❌ failed", - }) + health.Servers = append(health.Servers, MCPServerHealthDetail{ServerName: serverName, Status: "❌ failed"}) } } +} - health.TotalServers = len(health.Servers) - - // Count servers by status for accurate summary +func buildMCPServerHealthFinalize(health *MCPServerHealth) { degradedCount := 0 for _, s := range health.Servers { if strings.Contains(s.Status, "degraded") { @@ -541,11 +548,7 @@ func buildMCPServerHealth(mcpToolUsage *MCPToolUsageData, mcpFailures []MCPFailu } health.DegradedSvrs = degradedCount health.HealthySvrs = health.TotalServers - health.FailedSvrs - health.DegradedSvrs - - // Calculate overall error rate health.ErrorRate = safePercent(health.TotalErrors, health.TotalRequests) - - // Sort servers by request count (highest first) slices.SortFunc(health.Servers, func(a, b MCPServerHealthDetail) int { if a.RequestCount > b.RequestCount { return -1 @@ -555,14 +558,7 @@ func buildMCPServerHealth(mcpToolUsage *MCPToolUsageData, mcpFailures []MCPFailu } return 0 }) - - // Build summary string - health.Summary = fmt.Sprintf("%d server(s), %d healthy, %d degraded, %d failed", - health.TotalServers, health.HealthySvrs, health.DegradedSvrs, health.FailedSvrs) - - auditExpandedLog.Printf("Built MCP server health: %s, total_requests=%d, error_rate=%.1f%%", - health.Summary, health.TotalRequests, health.ErrorRate) - return health + health.Summary = fmt.Sprintf("%d server(s), %d healthy, %d degraded, %d failed", health.TotalServers, health.HealthySvrs, health.DegradedSvrs, health.FailedSvrs) } // buildSlowestToolCalls extracts the N slowest tool calls from the call records diff --git a/pkg/cli/audit_report.go b/pkg/cli/audit_report.go index 4b4cb5b7d3b..fdf7272421d 100644 --- a/pkg/cli/audit_report.go +++ b/pkg/cli/audit_report.go @@ -271,47 +271,123 @@ func buildAuditData(processedRun ProcessedRun, metrics LogMetrics, mcpToolUsage // to populate the Experiments section of the report. expData := extractExperimentData(run.LogsPath) - // Build overview - overview := OverviewData{ - RunID: run.DatabaseID, - WorkflowName: run.WorkflowName, - Status: run.Status, - Conclusion: run.Conclusion, - CreatedAt: run.CreatedAt, - StartedAt: run.StartedAt, - UpdatedAt: run.UpdatedAt, - Event: run.Event, - Branch: run.HeadBranch, - URL: run.URL, - Experiment: formatExperimentLabel(expData), - } + overview := buildAuditDataOverview(run, expData) + metricsData, inferredEngineID := buildAuditDataMetrics(processedRun, metrics) + jobs := buildAuditDataJobs(processedRun.JobDetails) - if run.LogsPath != "" { - overview.LogsPath = run.LogsPath + errors := buildAuditDataErrors(run) + + toolUsage := buildToolUsageInfo(metrics) + toolUsage = mergeMCPToolUsageInfo(toolUsage, mcpToolUsage) + + analysis := buildAuditDataAnalysis(processedRun, metricsData, toolUsage, errors, overview.AwContext) + + expanded := buildAuditDataExpanded(processedRun, metrics, metricsData, toolUsage, analysis.createdItems, mcpToolUsage, inferredEngineID) + + if auditReportLog.Enabled() { + auditReportLog.Printf("Built audit data: %d jobs, %d errors, %d tool types, %d findings, %d recommendations", + len(jobs), len(errors), len(toolUsage), len(analysis.findings), len(analysis.recommendations)) + } + + auditData := buildAuditDataAssemble(buildAuditDataAssembleParams{ + ProcessedRun: processedRun, + Overview: overview, + MetricsData: metricsData, + Jobs: jobs, + Errors: errors, + ToolUsage: toolUsage, + MCPToolUsage: mcpToolUsage, + ExpData: expData, + Analysis: analysis, + Expanded: expanded, + }) + + // Evaluate outcomes for created items if any exist + buildAuditDataOutcomes(&auditData, analysis.createdItems) + + return auditData +} + +type buildAuditDataExpandedData struct { + downloadedFiles []FileInfo + performanceMetrics *PerformanceMetrics + engineConfig *AuditEngineConfig + promptAnalysis *PromptAnalysis + sessionAnalysis *SessionAnalysis + safeOutputSummary *SafeOutputSummary + mcpServerHealth *MCPServerHealth +} + +func buildAuditDataExpanded(processedRun ProcessedRun, metrics LogMetrics, metricsData MetricsData, toolUsage []ToolUsageInfo, createdItems []CreatedItemReport, mcpToolUsage *MCPToolUsageData, inferredEngineID string) buildAuditDataExpandedData { + run := processedRun.Run + chainMetrics := buildSafeOutputChainMetrics(run.LogsPath) + return buildAuditDataExpandedData{ + downloadedFiles: extractDownloadedFiles(run.LogsPath), + performanceMetrics: generatePerformanceMetrics(processedRun, metricsData, toolUsage), + engineConfig: extractEngineConfigWithInferredEngine(run.LogsPath, inferredEngineID), + promptAnalysis: extractPromptAnalysis(run.LogsPath), + sessionAnalysis: buildSessionAnalysis(processedRun, metrics), + safeOutputSummary: buildSafeOutputSummary(createdItems, chainMetrics), + mcpServerHealth: buildMCPServerHealth(mcpToolUsage, processedRun.MCPFailures), } +} - if run.Duration > 0 { - overview.Duration = timeutil.FormatDuration(run.Duration) +type buildAuditDataAssembleParams struct { + ProcessedRun ProcessedRun + Overview OverviewData + MetricsData MetricsData + Jobs []JobData + Errors []ErrorInfo + ToolUsage []ToolUsageInfo + MCPToolUsage *MCPToolUsageData + ExpData *ExperimentData + Analysis buildAuditDataAnalysisResult + Expanded buildAuditDataExpandedData +} + +func buildAuditDataAssemble(p buildAuditDataAssembleParams) AuditData { + return AuditData{ + Overview: p.Overview, TaskDomain: p.Analysis.taskDomain, BehaviorFingerprint: p.Analysis.behaviorFingerprint, + AgenticAssessments: p.Analysis.agenticAssessments, Metrics: p.MetricsData, KeyFindings: p.Analysis.findings, + Recommendations: p.Analysis.recommendations, ObservabilityInsights: p.Analysis.observabilityInsights, + PerformanceMetrics: p.Expanded.performanceMetrics, EngineConfig: p.Expanded.engineConfig, + PromptAnalysis: p.Expanded.promptAnalysis, SessionAnalysis: p.Expanded.sessionAnalysis, + SafeOutputSummary: p.Expanded.safeOutputSummary, MCPServerHealth: p.Expanded.mcpServerHealth, + Jobs: p.Jobs, DownloadedFiles: p.Expanded.downloadedFiles, MissingTools: p.ProcessedRun.MissingTools, + MissingData: p.ProcessedRun.MissingData, Noops: p.ProcessedRun.Noops, MCPFailures: p.ProcessedRun.MCPFailures, + FirewallTokenUsage: p.ProcessedRun.TokenUsage, GitHubRateLimitUsage: p.ProcessedRun.GitHubRateLimitUsage, + FirewallAnalysis: p.ProcessedRun.FirewallAnalysis, PolicyAnalysis: p.ProcessedRun.PolicyAnalysis, + RedactedDomainsAnalysis: p.ProcessedRun.RedactedDomainsAnalysis, Errors: p.Errors, ToolUsage: p.ToolUsage, + MCPToolUsage: p.MCPToolUsage, CreatedItems: p.Analysis.createdItems, Experiments: p.ExpData, } +} +func buildAuditDataOverview(run WorkflowRun, expData *ExperimentData) OverviewData { + overview := OverviewData{ + RunID: run.DatabaseID, WorkflowName: run.WorkflowName, Status: run.Status, + Conclusion: run.Conclusion, CreatedAt: run.CreatedAt, StartedAt: run.StartedAt, + UpdatedAt: run.UpdatedAt, Event: run.Event, Branch: run.HeadBranch, URL: run.URL, + Experiment: formatExperimentLabel(expData), + } if run.LogsPath != "" { + overview.LogsPath = run.LogsPath awInfoPath := filepath.Join(run.LogsPath, "aw_info.json") if awInfo, err := parseAwInfo(awInfoPath, false); err == nil && awInfo != nil { overview.AwContext = awInfo.Context } } - - // Build metrics - metricsData := MetricsData{ - TokenUsage: run.TokenUsage, - Turns: run.Turns, - ErrorCount: run.ErrorCount, - WarningCount: run.WarningCount, + if run.Duration > 0 { + overview.Duration = timeutil.FormatDuration(run.Duration) } + return overview +} + +func buildAuditDataMetrics(processedRun ProcessedRun, metrics LogMetrics) (MetricsData, string) { + run := processedRun.Run + metricsData := MetricsData{TokenUsage: run.TokenUsage, Turns: run.Turns, ErrorCount: run.ErrorCount, WarningCount: run.WarningCount} if run.Conclusion == "failure" && metricsData.ErrorCount == 0 { metricsData.ErrorCount = 1 } - needsFallbackMetrics := metricsData.TokenUsage == 0 || metricsData.Turns == 0 needsFallbackEngineConfig := run.LogsPath != "" && findAwInfoPath(run.LogsPath) == "" var fallbackMetrics LogMetrics @@ -319,7 +395,11 @@ func buildAuditData(processedRun ProcessedRun, metrics LogMetrics, mcpToolUsage if run.LogsPath != "" && (needsFallbackMetrics || needsFallbackEngineConfig) { fallbackMetrics, inferredEngineID = inferFallbackLogMetrics(run.LogsPath) } + buildAuditDataApplyMetricFallbacks(&metricsData, processedRun, metrics, fallbackMetrics) + return metricsData, inferredEngineID +} +func buildAuditDataApplyMetricFallbacks(metricsData *MetricsData, processedRun ProcessedRun, metrics, fallbackMetrics LogMetrics) { // Fallback token usage: when the run-level metric is missing/zero for older // runs, use aggregated input+output tokens from agent_usage/token usage artifacts. if metricsData.TokenUsage == 0 && processedRun.TokenUsage != nil { @@ -337,129 +417,83 @@ func buildAuditData(processedRun ProcessedRun, metrics LogMetrics, mcpToolUsage if metricsData.Turns == 0 && fallbackMetrics.Turns > 0 { metricsData.Turns = fallbackMetrics.Turns } + buildAuditDataApplyTokenDetails(metricsData, processedRun) +} +func buildAuditDataApplyTokenDetails(metricsData *MetricsData, processedRun ProcessedRun) { if processedRun.TokenUsage != nil && processedRun.TokenUsage.TotalAIC > 0 { metricsData.AIC = processedRun.TokenUsage.TotalAIC } if processedRun.TokenUsage != nil && processedRun.TokenUsage.AmbientContext != nil { metricsData.AmbientContext = processedRun.TokenUsage.AmbientContext } - // Populate ActionMinutes from run duration so it is always visible even // when token/turn metrics are zero (e.g. Codex runs that exit early). // Use math.Ceil to match the billable-minute rounding used elsewhere. - if run.ActionMinutes > 0 { - metricsData.ActionMinutes = run.ActionMinutes - } else if run.Duration > 0 { - metricsData.ActionMinutes = math.Ceil(run.Duration.Minutes()) + if processedRun.Run.ActionMinutes > 0 { + metricsData.ActionMinutes = processedRun.Run.ActionMinutes + } else if processedRun.Run.Duration > 0 { + metricsData.ActionMinutes = math.Ceil(processedRun.Run.Duration.Minutes()) } +} - // Build job data - jobs := sliceutil.Map(processedRun.JobDetails, func(jobDetail JobInfoWithDuration) JobData { +func buildAuditDataJobs(jobDetails []JobInfoWithDuration) []JobData { + return sliceutil.Map(jobDetails, func(jobDetail JobInfoWithDuration) JobData { job := JobData{ - Name: jobDetail.Name, - Status: jobDetail.Status, - Conclusion: jobDetail.Conclusion, - Steps: sliceutil.Map(jobDetail.Steps, func(step JobStep) JobStepData { - return JobStepData(step) - }), + Name: jobDetail.Name, Status: jobDetail.Status, Conclusion: jobDetail.Conclusion, + Steps: sliceutil.Map(jobDetail.Steps, func(step JobStep) JobStepData { return JobStepData(step) }), } if jobDetail.Duration > 0 { job.Duration = timeutil.FormatDuration(jobDetail.Duration) } return job }) +} - // Build downloaded files list - downloadedFiles := extractDownloadedFiles(run.LogsPath) - +func buildAuditDataErrors(run WorkflowRun) []ErrorInfo { // For failed workflows where the agent never ran (no agent-stdio.log), // extract errors from step log files to surface the actual failure reason. - var errors []ErrorInfo if run.Conclusion == "failure" && run.LogsPath != "" { if stepErrors := extractPreAgentStepErrors(run.LogsPath); len(stepErrors) > 0 { - errors = stepErrors + return stepErrors } } + return nil +} - toolUsage := buildToolUsageInfo(metrics) - toolUsage = mergeMCPToolUsageInfo(toolUsage, mcpToolUsage) - - createdItems := extractCreatedItemsFromManifest(run.LogsPath) - taskDomain := detectTaskDomain(processedRun, createdItems, toolUsage, overview.AwContext) - behaviorFingerprint := buildBehaviorFingerprint(processedRun, metricsData, toolUsage, createdItems, overview.AwContext) - agenticAssessments := buildAgenticAssessments(processedRun, metricsData, toolUsage, createdItems, taskDomain, behaviorFingerprint, overview.AwContext) +type buildAuditDataAnalysisResult struct { + createdItems []CreatedItemReport + taskDomain *TaskDomainInfo + behaviorFingerprint *BehaviorFingerprint + agenticAssessments []AgenticAssessment + findings []Finding + recommendations []Recommendation + observabilityInsights []ObservabilityInsight +} - // Generate key findings +func buildAuditDataAnalysis(processedRun ProcessedRun, metricsData MetricsData, toolUsage []ToolUsageInfo, errors []ErrorInfo, awContext *AwContext) buildAuditDataAnalysisResult { + createdItems := extractCreatedItemsFromManifest(processedRun.Run.LogsPath) + taskDomain := detectTaskDomain(processedRun, createdItems, toolUsage, awContext) + behaviorFingerprint := buildBehaviorFingerprint(processedRun, metricsData, toolUsage, createdItems, awContext) + agenticAssessments := buildAgenticAssessments(processedRun, metricsData, toolUsage, createdItems, taskDomain, behaviorFingerprint, awContext) findings := generateFindings(processedRun, metricsData, errors) findings = append(findings, generateAgenticAssessmentFindings(agenticAssessments)...) - - // Generate recommendations recommendations := generateRecommendations(processedRun, metricsData, findings) recommendations = append(recommendations, generateAgenticAssessmentRecommendations(agenticAssessments)...) - observabilityInsights := buildAuditObservabilityInsights(processedRun, metricsData, toolUsage, createdItems) observabilityInsights = append(observabilityInsights, buildDrain3Insights(processedRun, metricsData, toolUsage)...) + return buildAuditDataAnalysisResult{createdItems, taskDomain, behaviorFingerprint, agenticAssessments, findings, recommendations, observabilityInsights} +} - // Generate performance metrics - performanceMetrics := generatePerformanceMetrics(processedRun, metricsData, toolUsage) - chainMetrics := buildSafeOutputChainMetrics(run.LogsPath) - - // Extract expanded audit data - engineConfig := extractEngineConfigWithInferredEngine(run.LogsPath, inferredEngineID) - promptAnalysis := extractPromptAnalysis(run.LogsPath) - sessionAnalysis := buildSessionAnalysis(processedRun, metrics) - safeOutputSummary := buildSafeOutputSummary(createdItems, chainMetrics) - mcpServerHealth := buildMCPServerHealth(mcpToolUsage, processedRun.MCPFailures) - - if auditReportLog.Enabled() { - auditReportLog.Printf("Built audit data: %d jobs, %d errors, %d tool types, %d findings, %d recommendations", - len(jobs), len(errors), len(toolUsage), len(findings), len(recommendations)) - } - - auditData := AuditData{ - Overview: overview, - TaskDomain: taskDomain, - BehaviorFingerprint: behaviorFingerprint, - AgenticAssessments: agenticAssessments, - Metrics: metricsData, - KeyFindings: findings, - Recommendations: recommendations, - ObservabilityInsights: observabilityInsights, - PerformanceMetrics: performanceMetrics, - EngineConfig: engineConfig, - PromptAnalysis: promptAnalysis, - SessionAnalysis: sessionAnalysis, - SafeOutputSummary: safeOutputSummary, - MCPServerHealth: mcpServerHealth, - Jobs: jobs, - DownloadedFiles: downloadedFiles, - MissingTools: processedRun.MissingTools, - MissingData: processedRun.MissingData, - Noops: processedRun.Noops, - MCPFailures: processedRun.MCPFailures, - FirewallTokenUsage: processedRun.TokenUsage, - GitHubRateLimitUsage: processedRun.GitHubRateLimitUsage, - FirewallAnalysis: processedRun.FirewallAnalysis, - PolicyAnalysis: processedRun.PolicyAnalysis, - RedactedDomainsAnalysis: processedRun.RedactedDomainsAnalysis, - Errors: errors, - ToolUsage: toolUsage, - MCPToolUsage: mcpToolUsage, - CreatedItems: createdItems, - Experiments: expData, - } - - // Evaluate outcomes for created items if any exist - if len(createdItems) > 0 { - mapping := github.LoadObjectiveMappingFromConfig() - outcomeReports := EvaluateOutcomes(createdItems, "", mapping) - auditData.Outcomes = outcomeReports - outcomeSummary := ComputeOutcomeSummary(outcomeReports, mapping) - auditData.OutcomeSummary = &outcomeSummary +func buildAuditDataOutcomes(auditData *AuditData, createdItems []CreatedItemReport) { + if len(createdItems) == 0 { + return } - - return auditData + mapping := github.LoadObjectiveMappingFromConfig() + outcomeReports := EvaluateOutcomes(createdItems, "", mapping) + auditData.Outcomes = outcomeReports + outcomeSummary := ComputeOutcomeSummary(outcomeReports, mapping) + auditData.OutcomeSummary = &outcomeSummary } // extractDownloadedFiles scans the logs directory recursively and returns file information. @@ -644,161 +678,135 @@ func extractPreAgentStepErrors(logsPath string) []ErrorInfo { if _, err := os.Stat(workflowLogsDir); err != nil { auditReportLog.Printf("workflow-logs directory not found, skipping step log extraction") // Even without workflow-logs, fall back to agent-stdio.log if the agent ran. - if agentRan { - if agentExcerpt := extractAgentStdioFailureExcerpt(agentStdioPath, maxMessageLen); agentExcerpt != "" { - return []ErrorInfo{{ - Type: "agent_failure", - File: "agent-stdio.log", - Message: agentExcerpt, - }} - } - } - return nil + return extractPreAgentStepErrorsAgentFallback(agentRan, agentStdioPath, maxMessageLen) } - // Scan all job step log files in a single pass, collecting both ##[error] annotations - // and tracking the last step for fallback use. - // GitHub Actions log zip structure: {job_name}/{step_num}_{step_name}.txt - type stepLog struct { - path string - num int - stepKey string // job/step_name for display + errorAnnotations, lastStep := extractPreAgentStepErrorsScanWorkflowLogs(workflowLogsDir, maxMessageLen) + + // Prefer ##[error] annotations over generic last-step content + if len(errorAnnotations) > 0 { + return errorAnnotations } - var lastStep *stepLog - var errorAnnotations []ErrorInfo + // If the agent executed but no ##[error] annotations were present in workflow-logs, + // extract a concise excerpt from agent-stdio.log instead of returning a generic message. + if agentRan { + return extractPreAgentStepErrorsAgentFallback(agentRan, agentStdioPath, maxMessageLen) + } - jobDirs, err := os.ReadDir(workflowLogsDir) - if err != nil { + return extractPreAgentStepErrorsLastStepFallback(lastStep, workflowLogsDir, maxMessageLen) +} + +type extractPreAgentStepErrorsStepLog struct { + path string + num int + stepKey string // job/step_name for display +} + +func extractPreAgentStepErrorsAgentFallback(agentRan bool, agentStdioPath string, maxMessageLen int) []ErrorInfo { + if !agentRan { return nil } + if agentExcerpt := extractAgentStdioFailureExcerpt(agentStdioPath, maxMessageLen); agentExcerpt != "" { + return []ErrorInfo{{Type: "agent_failure", File: "agent-stdio.log", Message: agentExcerpt}} + } + return nil +} +func extractPreAgentStepErrorsScanWorkflowLogs(workflowLogsDir string, maxMessageLen int) ([]ErrorInfo, *extractPreAgentStepErrorsStepLog) { + var lastStep *extractPreAgentStepErrorsStepLog + var errorAnnotations []ErrorInfo + jobDirs, err := os.ReadDir(workflowLogsDir) + if err != nil { + return nil, nil + } for _, jobEntry := range jobDirs { if !jobEntry.IsDir() { - // Handle flat job-level log files (e.g., 3_activation.txt). - // GitHub Actions log zips may place per-job log files directly at the root of - // the zip (flat format) rather than in per-job subdirectories (hierarchical format). - if !strings.HasSuffix(jobEntry.Name(), ".txt") { - continue - } - num, jobName := parseStepFilename(jobEntry.Name()) - if num <= 0 { - continue - } - flatFilePath := filepath.Join(workflowLogsDir, jobEntry.Name()) - - // Track the last flat job log (highest number) for fallback - if lastStep == nil || num > lastStep.num { - lastStep = &stepLog{ - path: flatFilePath, - num: num, - stepKey: jobName, - } - } - - // Scan this flat job log for ##[error] annotations - content, err := os.ReadFile(flatFilePath) - if err != nil { - auditReportLog.Printf("Failed to read job log %s: %v", flatFilePath, err) - continue - } + annotations, step := extractPreAgentStepErrorsFlatLog(workflowLogsDir, jobEntry, maxMessageLen) + errorAnnotations = append(errorAnnotations, annotations...) + lastStep = extractPreAgentStepErrorsLaterStep(lastStep, step) + continue + } + annotations, step := extractPreAgentStepErrorsJobDir(workflowLogsDir, jobEntry, maxMessageLen) + errorAnnotations = append(errorAnnotations, annotations...) + lastStep = extractPreAgentStepErrorsLaterStep(lastStep, step) + } + return errorAnnotations, lastStep +} - var errorLines []string - for line := range strings.SplitSeq(string(content), "\n") { - if strings.Contains(line, "##[error]") { - stripped := stripGHALogTimestamps(line) - if stripped != "" { - errorLines = append(errorLines, stripped) - } - } - } +func extractPreAgentStepErrorsFlatLog(workflowLogsDir string, jobEntry os.DirEntry, maxMessageLen int) ([]ErrorInfo, *extractPreAgentStepErrorsStepLog) { + // Handle flat job-level log files (e.g., 3_activation.txt). + if !strings.HasSuffix(jobEntry.Name(), ".txt") { + return nil, nil + } + num, jobName := parseStepFilename(jobEntry.Name()) + if num <= 0 { + return nil, nil + } + flatFilePath := filepath.Join(workflowLogsDir, jobEntry.Name()) + step := &extractPreAgentStepErrorsStepLog{path: flatFilePath, num: num, stepKey: jobName} + annotations := extractPreAgentStepErrorsAnnotations(flatFilePath, jobName, num, maxMessageLen) + return annotations, step +} - if len(errorLines) > 0 { - message := strings.Join(errorLines, "\n") - message = stringutil.Truncate(message, maxMessageLen) - auditReportLog.Printf("Extracted ##[error] annotations from flat job log %s (job %d)", jobName, num) - errorAnnotations = append(errorAnnotations, ErrorInfo{ - Type: "step_failure", - File: jobName, - Message: message, - }) - } +func extractPreAgentStepErrorsJobDir(workflowLogsDir string, jobEntry os.DirEntry, maxMessageLen int) ([]ErrorInfo, *extractPreAgentStepErrorsStepLog) { + jobDir := filepath.Join(workflowLogsDir, jobEntry.Name()) + stepFiles, err := os.ReadDir(jobDir) + if err != nil { + return nil, nil + } + var lastStep *extractPreAgentStepErrorsStepLog + var annotations []ErrorInfo + for _, stepFile := range stepFiles { + if stepFile.IsDir() || !strings.HasSuffix(stepFile.Name(), ".txt") { continue } - jobDir := filepath.Join(workflowLogsDir, jobEntry.Name()) - stepFiles, err := os.ReadDir(jobDir) - if err != nil { + num, stepName := parseStepFilename(stepFile.Name()) + if num <= 0 { continue } - for _, stepFile := range stepFiles { - if stepFile.IsDir() || !strings.HasSuffix(stepFile.Name(), ".txt") { - continue - } - num, stepName := parseStepFilename(stepFile.Name()) - if num <= 0 { - continue - } - stepFilePath := filepath.Join(jobDir, stepFile.Name()) - stepKey := jobEntry.Name() + "/" + stepName - - // Track the last step (highest step number) for fallback - if lastStep == nil || num > lastStep.num { - lastStep = &stepLog{ - path: stepFilePath, - num: num, - stepKey: stepKey, - } - } - - // Scan this step for ##[error] annotations - content, err := os.ReadFile(stepFilePath) - if err != nil { - auditReportLog.Printf("Failed to read step log %s: %v", stepFilePath, err) - continue - } - - var errorLines []string - for line := range strings.SplitSeq(string(content), "\n") { - if strings.Contains(line, "##[error]") { - stripped := stripGHALogTimestamps(line) - if stripped != "" { - errorLines = append(errorLines, stripped) - } - } - } + stepFilePath := filepath.Join(jobDir, stepFile.Name()) + stepKey := jobEntry.Name() + "/" + stepName + lastStep = extractPreAgentStepErrorsLaterStep(lastStep, &extractPreAgentStepErrorsStepLog{path: stepFilePath, num: num, stepKey: stepKey}) + annotations = append(annotations, extractPreAgentStepErrorsAnnotations(stepFilePath, stepKey, num, maxMessageLen)...) + } + return annotations, lastStep +} - if len(errorLines) > 0 { - message := strings.Join(errorLines, "\n") - message = stringutil.Truncate(message, maxMessageLen) - auditReportLog.Printf("Extracted ##[error] annotations from %s (step %d)", stepKey, num) - errorAnnotations = append(errorAnnotations, ErrorInfo{ - Type: "step_failure", - File: stepKey, - Message: message, - }) +func extractPreAgentStepErrorsAnnotations(filePath, stepKey string, num, maxMessageLen int) []ErrorInfo { + content, err := os.ReadFile(filePath) + if err != nil { + auditReportLog.Printf("Failed to read step log %s: %v", filePath, err) + return nil + } + var errorLines []string + for line := range strings.SplitSeq(string(content), "\n") { + if strings.Contains(line, "##[error]") { + stripped := stripGHALogTimestamps(line) + if stripped != "" { + errorLines = append(errorLines, stripped) } } } - - // Prefer ##[error] annotations over generic last-step content - if len(errorAnnotations) > 0 { - return errorAnnotations + if len(errorLines) == 0 { + return nil } + message := stringutil.Truncate(strings.Join(errorLines, "\n"), maxMessageLen) + auditReportLog.Printf("Extracted ##[error] annotations from %s (step %d)", stepKey, num) + return []ErrorInfo{{Type: "step_failure", File: stepKey, Message: message}} +} - // If the agent executed but no ##[error] annotations were present in workflow-logs, - // extract a concise excerpt from agent-stdio.log instead of returning a generic message. - if agentRan { - if agentExcerpt := extractAgentStdioFailureExcerpt(agentStdioPath, maxMessageLen); agentExcerpt != "" { - return []ErrorInfo{{ - Type: "agent_failure", - File: "agent-stdio.log", - Message: agentExcerpt, - }} - } - return nil +func extractPreAgentStepErrorsLaterStep(current, candidate *extractPreAgentStepErrorsStepLog) *extractPreAgentStepErrorsStepLog { + if candidate == nil { + return current } + if current == nil || candidate.num > current.num { + return candidate + } + return current +} - // Fallback: return the content of the last step that ran +func extractPreAgentStepErrorsLastStepFallback(lastStep *extractPreAgentStepErrorsStepLog, workflowLogsDir string, maxMessageLen int) []ErrorInfo { if lastStep == nil { auditReportLog.Printf("No step log files found in %s", workflowLogsDir) return nil @@ -833,24 +841,8 @@ func extractAgentStdioFailureExcerpt(agentStdioPath string, maxMessageLen int) s } defer f.Close() - // Read only the tail of the file to bound memory use for large logs. - const maxReadBytes int64 = 64 * 1024 // 64 KB tail window - - fi, err := f.Stat() + tail, err := extractAgentStdioFailureExcerptTail(f, agentStdioPath) if err != nil { - auditReportLog.Printf("Failed to stat %s: %v", agentStdioPath, err) - return "" - } - if fi.Size() > maxReadBytes { - if _, err := f.Seek(-maxReadBytes, io.SeekEnd); err != nil { - auditReportLog.Printf("Failed to seek in %s: %v", agentStdioPath, err) - return "" - } - } - - tail, err := io.ReadAll(f) - if err != nil { - auditReportLog.Printf("Failed to read %s: %v", agentStdioPath, err) return "" } @@ -889,6 +881,28 @@ func extractAgentStdioFailureExcerpt(agentStdioPath string, maxMessageLen int) s return stringutil.Truncate(strings.Join(nonEmpty[start:], "\n"), maxMessageLen) } +func extractAgentStdioFailureExcerptTail(f *os.File, agentStdioPath string) ([]byte, error) { + // Read only the tail of the file to bound memory use for large logs. + const maxReadBytes int64 = 64 * 1024 // 64 KB tail window + fi, err := f.Stat() + if err != nil { + auditReportLog.Printf("Failed to stat %s: %v", agentStdioPath, err) + return nil, err + } + if fi.Size() > maxReadBytes { + if _, err := f.Seek(-maxReadBytes, io.SeekEnd); err != nil { + auditReportLog.Printf("Failed to seek in %s: %v", agentStdioPath, err) + return nil, err + } + } + tail, err := io.ReadAll(f) + if err != nil { + auditReportLog.Printf("Failed to read %s: %v", agentStdioPath, err) + return nil, err + } + return tail, nil +} + // parseStepFilename extracts the step number and name from a GitHub Actions step log // filename in the format "{step_num}_{step_name}.txt" (e.g. "12_Validate lockdown mode.txt"). // Returns (0, filename) if the filename does not match the expected format. diff --git a/pkg/cli/audit_report_analysis.go b/pkg/cli/audit_report_analysis.go index d65b0c2030e..bba8746bf0c 100644 --- a/pkg/cli/audit_report_analysis.go +++ b/pkg/cli/audit_report_analysis.go @@ -31,159 +31,110 @@ func generateFindings(processedRun ProcessedRun, metrics MetricsData, errors []E var findings []Finding run := processedRun.Run - // Failure findings - if run.Conclusion == "failure" { - var desc string - if metrics.ErrorCount == 0 && len(errors) == 0 { - if agentJob, ok := findFailedAgentJob(processedRun.JobDetails); ok { - // The agent job ran and failed, but telemetry/error artifacts were not exported. - // Surface this explicitly rather than misclassifying as pre-activation failure. - desc = fmt.Sprintf("Workflow '%s' failed after agent activation — agent job ran for %s before failing and no agent telemetry was available to analyze", run.WorkflowName, timeutil.FormatDuration(agentJob.Duration)) - } else { - // No log data available — run likely failed before agent activation (e.g. cancelled, - // infrastructure failure, or no downloadable artifacts). Saying "failed with 0 error(s)" - // is logically contradictory, so surface a clearer message instead. - desc = fmt.Sprintf("Workflow '%s' failed before agent activation — no error logs were available to analyze", run.WorkflowName) - } - } else { - // Prefer the length of the actual errors slice as the count, since it is the - // ground-truth list that is included in the audit output. Fall back to - // metrics.ErrorCount only when no individual error details were collected - // (e.g. the log was unavailable but a count was extracted from a summary). - errorCount := len(errors) - if errorCount == 0 { - errorCount = metrics.ErrorCount - } + findings = generateFindingsFailures(findings, processedRun, metrics, errors) + findings = generateFindingsPerformance(findings, run, metrics, errors) + findings = generateFindingsTooling(findings, processedRun) + findings = generateFindingsFirewall(findings, processedRun) + findings = generateFindingsSuccess(findings, run, metrics, errors) - desc = fmt.Sprintf("Workflow '%s' failed with %d error(s)", run.WorkflowName, errorCount) - if len(errors) > 0 { - // Append a truncated first error message to help quickly identify the root cause. - // Keep descriptions short enough to be useful in a key findings summary. - const maxErrMsgLen = 200 - msg := stringutil.Truncate(errors[0].Message, maxErrMsgLen) - desc += ": " + msg - } + return findings +} +func generateFindingsFailures(findings []Finding, processedRun ProcessedRun, metrics MetricsData, errors []ErrorInfo) []Finding { + run := processedRun.Run + if run.Conclusion != "failure" { + return findings + } + desc := generateFindingsFailureDescription(processedRun, metrics, errors) + return append(findings, Finding{ + Category: "error", + Severity: "critical", + Title: "Workflow Failed", + Description: desc, + Impact: "Workflow did not complete successfully and may need intervention", + }) +} + +func generateFindingsFailureDescription(processedRun ProcessedRun, metrics MetricsData, errors []ErrorInfo) string { + run := processedRun.Run + if metrics.ErrorCount == 0 && len(errors) == 0 { + if agentJob, ok := findFailedAgentJob(processedRun.JobDetails); ok { + // The agent job ran and failed, but telemetry/error artifacts were not exported. + return fmt.Sprintf("Workflow '%s' failed after agent activation — agent job ran for %s before failing and no agent telemetry was available to analyze", run.WorkflowName, timeutil.FormatDuration(agentJob.Duration)) } - findings = append(findings, Finding{ - Category: "error", - Severity: "critical", - Title: "Workflow Failed", - Description: desc, - Impact: "Workflow did not complete successfully and may need intervention", - }) + // No log data available — run likely failed before agent activation. + return fmt.Sprintf("Workflow '%s' failed before agent activation — no error logs were available to analyze", run.WorkflowName) } + errorCount := len(errors) + if errorCount == 0 { + errorCount = metrics.ErrorCount + } + desc := fmt.Sprintf("Workflow '%s' failed with %d error(s)", run.WorkflowName, errorCount) + if len(errors) > 0 { + const maxErrMsgLen = 200 + desc += ": " + stringutil.Truncate(errors[0].Message, maxErrMsgLen) + } + return desc +} +func generateFindingsPerformance(findings []Finding, run WorkflowRun, metrics MetricsData, errors []ErrorInfo) []Finding { if run.Conclusion == "timed_out" { - findings = append(findings, Finding{ - Category: "performance", - Severity: "high", - Title: "Workflow Timeout", - Description: "Workflow exceeded time limit and was terminated", - Impact: "Tasks may be incomplete, consider optimizing workflow or increasing timeout", - }) + findings = append(findings, Finding{Category: "performance", Severity: "high", Title: "Workflow Timeout", Description: "Workflow exceeded time limit and was terminated", Impact: "Tasks may be incomplete, consider optimizing workflow or increasing timeout"}) } - - // Token usage findings if metrics.TokenUsage > 50000 { - findings = append(findings, Finding{ - Category: "performance", - Severity: "medium", - Title: "High Token Usage", - Description: fmt.Sprintf("Used %s tokens", console.FormatNumber(metrics.TokenUsage)), - Impact: "High token usage may indicate verbose outputs or inefficient prompts", - }) + findings = append(findings, Finding{Category: "performance", Severity: "medium", Title: "High Token Usage", Description: fmt.Sprintf("Used %s tokens", console.FormatNumber(metrics.TokenUsage)), Impact: "High token usage may indicate verbose outputs or inefficient prompts"}) } - - // Turn count findings if metrics.Turns > 10 { - findings = append(findings, Finding{ - Category: "performance", - Severity: "medium", - Title: "Many Iterations", - Description: fmt.Sprintf("Workflow took %d turns to complete", metrics.Turns), - Impact: "Many turns may indicate task complexity or unclear instructions", - }) + findings = append(findings, Finding{Category: "performance", Severity: "medium", Title: "Many Iterations", Description: fmt.Sprintf("Workflow took %d turns to complete", metrics.Turns), Impact: "Many turns may indicate task complexity or unclear instructions"}) } - - // Error findings if len(errors) > 5 { - findings = append(findings, Finding{ - Category: "error", - Severity: "high", - Title: "Multiple Errors", - Description: fmt.Sprintf("Encountered %d errors during execution", len(errors)), - Impact: "Multiple errors may indicate systemic issues requiring attention", - }) + findings = append(findings, Finding{Category: "error", Severity: "high", Title: "Multiple Errors", Description: fmt.Sprintf("Encountered %d errors during execution", len(errors)), Impact: "Multiple errors may indicate systemic issues requiring attention"}) } + return findings +} - // MCP failure findings +func generateFindingsTooling(findings []Finding, processedRun ProcessedRun) []Finding { if len(processedRun.MCPFailures) > 0 { - serverNames := sliceutil.Map(processedRun.MCPFailures, func(failure MCPFailureReport) string { - return failure.ServerName - }) - findings = append(findings, Finding{ - Category: "tooling", - Severity: "high", - Title: "MCP Server Failures", - Description: "Failed MCP servers: " + strings.Join(serverNames, ", "), - Impact: "Missing tools may limit workflow capabilities", - }) + serverNames := sliceutil.Map(processedRun.MCPFailures, func(failure MCPFailureReport) string { return failure.ServerName }) + findings = append(findings, Finding{Category: "tooling", Severity: "high", Title: "MCP Server Failures", Description: "Failed MCP servers: " + strings.Join(serverNames, ", "), Impact: "Missing tools may limit workflow capabilities"}) } - - // Missing tool findings if len(processedRun.MissingTools) > 0 { - toolNames := sliceutil.Map( - processedRun.MissingTools[:min(3, len(processedRun.MissingTools))], - func(t MissingToolReport) string { return t.Tool }, - ) + toolNames := sliceutil.Map(processedRun.MissingTools[:min(3, len(processedRun.MissingTools))], func(t MissingToolReport) string { return t.Tool }) desc := "Missing tools: " + strings.Join(toolNames, ", ") if len(processedRun.MissingTools) > 3 { desc += fmt.Sprintf(" (and %d more)", len(processedRun.MissingTools)-3) } - findings = append(findings, Finding{ - Category: "tooling", - Severity: "medium", - Title: "Tools Not Available", - Description: desc, - Impact: "Agent requested tools that were not configured or available", - }) + findings = append(findings, Finding{Category: "tooling", Severity: "medium", Title: "Tools Not Available", Description: desc, Impact: "Agent requested tools that were not configured or available"}) } + return findings +} - // Firewall findings - if processedRun.FirewallAnalysis != nil && processedRun.FirewallAnalysis.BlockedRequests > 0 { - blockedDomains := filterActionableDomains(processedRun.FirewallAnalysis.GetBlockedDomains()) - var desc string - switch { - case len(blockedDomains) == 1: - desc = "Agent attempted to access blocked domain: " + blockedDomains[0] - case len(blockedDomains) > 1 && len(blockedDomains) <= 3: - desc = "Agent attempted to access blocked domains: " + strings.Join(blockedDomains, ", ") - case len(blockedDomains) > 3: - desc = fmt.Sprintf("Agent attempted to access %d blocked domains, including: %s", len(blockedDomains), strings.Join(blockedDomains[:3], ", ")) - default: - desc = fmt.Sprintf("%d network request(s) were blocked by firewall", processedRun.FirewallAnalysis.BlockedRequests) - } - findings = append(findings, Finding{ - Category: "network", - Severity: "medium", - Title: "Blocked Network Requests", - Description: desc, - Impact: "Blocked requests may indicate missing network permissions or unexpected behavior", - }) +func generateFindingsFirewall(findings []Finding, processedRun ProcessedRun) []Finding { + if processedRun.FirewallAnalysis == nil || processedRun.FirewallAnalysis.BlockedRequests == 0 { + return findings } + blockedDomains := filterActionableDomains(processedRun.FirewallAnalysis.GetBlockedDomains()) + desc := generateFindingsFirewallDescription(blockedDomains, processedRun.FirewallAnalysis.BlockedRequests) + return append(findings, Finding{Category: "network", Severity: "medium", Title: "Blocked Network Requests", Description: desc, Impact: "Blocked requests may indicate missing network permissions or unexpected behavior"}) +} - // Success findings - if run.Conclusion == "success" && len(errors) == 0 { - findings = append(findings, Finding{ - Category: "success", - Severity: "info", - Title: "Workflow Completed Successfully", - Description: fmt.Sprintf("Completed in %d turns with no errors", metrics.Turns), - Impact: "No action needed", - }) +func generateFindingsFirewallDescription(blockedDomains []string, blockedRequests int) string { + switch { + case len(blockedDomains) == 1: + return "Agent attempted to access blocked domain: " + blockedDomains[0] + case len(blockedDomains) > 1 && len(blockedDomains) <= 3: + return "Agent attempted to access blocked domains: " + strings.Join(blockedDomains, ", ") + case len(blockedDomains) > 3: + return fmt.Sprintf("Agent attempted to access %d blocked domains, including: %s", len(blockedDomains), strings.Join(blockedDomains[:3], ", ")) + default: + return fmt.Sprintf("%d network request(s) were blocked by firewall", blockedRequests) } +} +func generateFindingsSuccess(findings []Finding, run WorkflowRun, metrics MetricsData, errors []ErrorInfo) []Finding { + if run.Conclusion == "success" && len(errors) == 0 { + findings = append(findings, Finding{Category: "success", Severity: "info", Title: "Workflow Completed Successfully", Description: fmt.Sprintf("Completed in %d turns with no errors", metrics.Turns), Impact: "No action needed"}) + } return findings } @@ -203,104 +154,81 @@ func generateRecommendations(processedRun ProcessedRun, metrics MetricsData, fin var recommendations []Recommendation run := processedRun.Run - // Check for high-severity findings - hasCriticalFindings := false - hasHighCostFindings := false - hasManyTurns := false + flags := generateRecommendationsFindingFlags(findings) + recommendations = generateRecommendationsFromFindings(recommendations, run, flags) + recommendations = generateRecommendationsTooling(recommendations, processedRun) + recommendations = generateRecommendationsFirewall(recommendations, processedRun) + + // General best practices + if len(recommendations) == 0 && run.Conclusion == "success" { + recommendations = append(recommendations, Recommendation{ + Priority: "low", + Action: "Monitor workflow performance over time", + Reason: "Tracking metrics helps identify trends and optimization opportunities", + Example: "Run 'gh aw logs' periodically to review cost and performance trends", + }) + } + + return recommendations +} + +type generateRecommendationsFlags struct { + hasCriticalFindings bool + hasHighCostFindings bool + hasManyTurns bool +} + +func generateRecommendationsFindingFlags(findings []Finding) generateRecommendationsFlags { + var flags generateRecommendationsFlags for _, finding := range findings { if finding.Severity == "critical" { - hasCriticalFindings = true + flags.hasCriticalFindings = true } if finding.Category == "cost" && (finding.Severity == "high" || finding.Severity == "medium") { - hasHighCostFindings = true + flags.hasHighCostFindings = true } if finding.Category == "performance" && strings.Contains(finding.Title, "Iterations") { - hasManyTurns = true + flags.hasManyTurns = true } } + return flags +} - // Recommendations for failures - if run.Conclusion == "failure" || hasCriticalFindings { - recommendations = append(recommendations, Recommendation{ - Priority: "high", - Action: "Review error logs to identify root cause of failure", - Reason: "Understanding failure causes helps prevent recurrence", - Example: "Check the errors field for specific error messages, or inspect the log files in logs_path", - }) +func generateRecommendationsFromFindings(recommendations []Recommendation, run WorkflowRun, flags generateRecommendationsFlags) []Recommendation { + if run.Conclusion == "failure" || flags.hasCriticalFindings { + recommendations = append(recommendations, Recommendation{Priority: "high", Action: "Review error logs to identify root cause of failure", Reason: "Understanding failure causes helps prevent recurrence", Example: "Check the errors field for specific error messages, or inspect the log files in logs_path"}) } - - // Recommendations for cost optimization - if hasHighCostFindings { - recommendations = append(recommendations, Recommendation{ - Priority: "medium", - Action: "Optimize prompt size and reduce verbose outputs", - Reason: "High token usage increases costs and may slow execution", - Example: "Use concise prompts, limit output verbosity, and consider caching repeated data", - }) + if flags.hasHighCostFindings { + recommendations = append(recommendations, Recommendation{Priority: "medium", Action: "Optimize prompt size and reduce verbose outputs", Reason: "High token usage increases costs and may slow execution", Example: "Use concise prompts, limit output verbosity, and consider caching repeated data"}) } - - // Recommendations for many turns - if hasManyTurns { - recommendations = append(recommendations, Recommendation{ - Priority: "medium", - Action: "Clarify workflow instructions or break into smaller tasks", - Reason: "Many iterations may indicate unclear objectives or overly complex tasks", - Example: "Split complex workflows into discrete steps with clear success criteria", - }) + if flags.hasManyTurns { + recommendations = append(recommendations, Recommendation{Priority: "medium", Action: "Clarify workflow instructions or break into smaller tasks", Reason: "Many iterations may indicate unclear objectives or overly complex tasks", Example: "Split complex workflows into discrete steps with clear success criteria"}) } + return recommendations +} - // Recommendations for missing tools +func generateRecommendationsTooling(recommendations []Recommendation, processedRun ProcessedRun) []Recommendation { if len(processedRun.MissingTools) > 0 { - recommendations = append(recommendations, Recommendation{ - Priority: "medium", - Action: "Add missing tools to workflow configuration", - Reason: "Missing tools limit agent capabilities and may cause failures", - Example: "Add tools configuration for: " + processedRun.MissingTools[0].Tool, - }) + recommendations = append(recommendations, Recommendation{Priority: "medium", Action: "Add missing tools to workflow configuration", Reason: "Missing tools limit agent capabilities and may cause failures", Example: "Add tools configuration for: " + processedRun.MissingTools[0].Tool}) } - - // Recommendations for MCP failures if len(processedRun.MCPFailures) > 0 { - recommendations = append(recommendations, Recommendation{ - Priority: "high", - Action: "Fix MCP server configuration or dependencies", - Reason: "MCP server failures prevent agent from accessing required tools", - Example: "Check server logs and verify MCP server is properly configured and accessible", - }) + recommendations = append(recommendations, Recommendation{Priority: "high", Action: "Fix MCP server configuration or dependencies", Reason: "MCP server failures prevent agent from accessing required tools", Example: "Check server logs and verify MCP server is properly configured and accessible"}) } + return recommendations +} +func generateRecommendationsFirewall(recommendations []Recommendation, processedRun ProcessedRun) []Recommendation { // Recommendations for firewall blocks – trigger on any block so even a single // domain denial (e.g. Codex CLI reporting one blocked domain) surfaces an action. - if processedRun.FirewallAnalysis != nil && processedRun.FirewallAnalysis.BlockedRequests > 0 { - blockedDomains := filterActionableDomains(processedRun.FirewallAnalysis.GetBlockedDomains()) - var example string - if len(blockedDomains) > 0 { - example = fmt.Sprintf( - "Add the blocked domain(s) to your workflow frontmatter:\n\n```yaml\nnetwork:\n allowed:\n - %s\n```", - strings.Join(blockedDomains, "\n - "), - ) - } else { - example = "Add allowed domains to network configuration or review firewall rules" - } - recommendations = append(recommendations, Recommendation{ - Priority: "medium", - Action: "Add blocked domains to the workflow network allow-list", - Reason: "Firewall-blocked domains prevent the agent from completing its tasks", - Example: example, - }) + if processedRun.FirewallAnalysis == nil || processedRun.FirewallAnalysis.BlockedRequests == 0 { + return recommendations } - - // General best practices - if len(recommendations) == 0 && run.Conclusion == "success" { - recommendations = append(recommendations, Recommendation{ - Priority: "low", - Action: "Monitor workflow performance over time", - Reason: "Tracking metrics helps identify trends and optimization opportunities", - Example: "Run 'gh aw logs' periodically to review cost and performance trends", - }) + blockedDomains := filterActionableDomains(processedRun.FirewallAnalysis.GetBlockedDomains()) + example := "Add allowed domains to network configuration or review firewall rules" + if len(blockedDomains) > 0 { + example = fmt.Sprintf("Add the blocked domain(s) to your workflow frontmatter:\n\n```yaml\nnetwork:\n allowed:\n - %s\n```", strings.Join(blockedDomains, "\n - ")) } - - return recommendations + return append(recommendations, Recommendation{Priority: "medium", Action: "Add blocked domains to the workflow network allow-list", Reason: "Firewall-blocked domains prevent the agent from completing its tasks", Example: example}) } // generatePerformanceMetrics calculates aggregated performance statistics diff --git a/pkg/cli/audit_report_render.go b/pkg/cli/audit_report_render.go index 94050857c59..da6917940ba 100644 --- a/pkg/cli/audit_report_render.go +++ b/pkg/cli/audit_report_render.go @@ -24,6 +24,17 @@ func renderJSON(data AuditData) error { func renderConsole(data AuditData, logsPath string) { auditReportLog.Print("Rendering compact audit report to console") + renderConsoleOverview(data) + renderConsoleMetrics(data) + renderConsoleJobs(data) + renderConsoleActionableSections(data) + renderConsoleDiagnostics(data) + renderConsoleOutputs(data) + renderConsoleTools(data) + renderConsoleNetwork(data, logsPath) +} + +func renderConsoleOverview(data AuditData) { // Line 1: Identity + outcome statusIcon := "✅" switch data.Overview.Conclusion { @@ -75,7 +86,9 @@ func renderConsole(data AuditData, logsPath string) { data.BehaviorFingerprint.ResourceProfile, data.BehaviorFingerprint.DispatchMode) } +} +func renderConsoleMetrics(data AuditData) { // Line 5: Metrics (always present) metricsLine := fmt.Sprintf(" metrics: errors=%d warnings=%d", data.Metrics.ErrorCount, data.Metrics.WarningCount) @@ -131,7 +144,9 @@ func renderConsole(data AuditData, logsPath string) { console.FormatNumber(data.GitHubRateLimitUsage.CoreConsumed), console.FormatNumber(data.GitHubRateLimitUsage.CoreLimit)) } +} +func renderConsoleJobs(data AuditData) { // Jobs (compact: one line if all pass, table if failures) if len(data.Jobs) > 0 { allPassed := true @@ -169,7 +184,9 @@ func renderConsole(data AuditData, logsPath string) { } fmt.Fprintln(os.Stderr, promptLine) } +} +func renderConsoleActionableSections(data AuditData) { // --- Actionable sections below (only rendered when non-trivial) --- // Key Findings: only show non-success findings in compact form @@ -214,7 +231,9 @@ func renderConsole(data AuditData, logsPath string) { fmt.Fprintln(os.Stderr, line) } } +} +func renderConsoleDiagnostics(data AuditData) { // Errors and Warnings (always show if present) if len(data.Errors) > 0 { fmt.Fprintln(os.Stderr, " errors:") @@ -257,7 +276,9 @@ func renderConsole(data AuditData, logsPath string) { if data.MCPServerHealth != nil { renderCompactMCPHealth(data.MCPServerHealth) } +} +func renderConsoleOutputs(data AuditData) { // Safe Output Summary (compact) if data.SafeOutputSummary != nil && data.SafeOutputSummary.TotalItems > 0 { fmt.Fprintf(os.Stderr, " safe_outputs: %d items — %s\n", @@ -277,7 +298,9 @@ func renderConsole(data AuditData, logsPath string) { fmt.Fprintln(os.Stderr, line) } } +} +func renderConsoleTools(data AuditData) { // Tool Usage (compact table only when tools were used) if len(data.ToolUsage) > 0 { fmt.Fprintln(os.Stderr, " tools:") @@ -308,7 +331,9 @@ func renderConsole(data AuditData, logsPath string) { fmt.Fprintf(os.Stderr, " guard_blocked: %d\n", data.MCPToolUsage.GuardPolicySummary.TotalBlocked) } } +} +func renderConsoleNetwork(data AuditData, logsPath string) { // Firewall Analysis (compact) if data.FirewallAnalysis != nil && data.FirewallAnalysis.TotalRequests > 0 { renderCompactFirewall(data.FirewallAnalysis) diff --git a/pkg/cli/bootstrap_profile_manifest.go b/pkg/cli/bootstrap_profile_manifest.go index 747ea5c62df..51fec609080 100644 --- a/pkg/cli/bootstrap_profile_manifest.go +++ b/pkg/cli/bootstrap_profile_manifest.go @@ -82,6 +82,23 @@ func extractManifestConfig(value any, manifestPath string) (*repositoryPackageBo func parseManifestBootstrapAction(actionType string, actionMap map[string]any, manifestPath string, index int) (repositoryPackageBootstrapAction, error) { action := repositoryPackageBootstrapAction{Type: actionType} + if err := parseManifestBootstrapActionFields(&action, actionMap, manifestPath, index); err != nil { + return repositoryPackageBootstrapAction{}, err + } + if err := parseManifestBootstrapActionValidate(&action, actionType, manifestPath, index); err != nil { + return repositoryPackageBootstrapAction{}, err + } + + return action, nil +} + +func parseManifestBootstrapActionFields(action *repositoryPackageBootstrapAction, actionMap map[string]any, manifestPath string, index int) error { + parseManifestBootstrapActionStringFields(action, actionMap) + parseManifestBootstrapActionBoolFields(action, actionMap) + return parseManifestBootstrapActionComplexFields(action, actionMap, manifestPath, index) +} + +func parseManifestBootstrapActionStringFields(action *repositoryPackageBootstrapAction, actionMap map[string]any) { if owner, ok := stringValue(actionMap["owner"]); ok { action.Owner = strings.TrimSpace(owner) } @@ -124,80 +141,67 @@ func parseManifestBootstrapAction(actionType string, actionMap map[string]any, m if homepageURL, ok := stringValue(actionMap["homepage-url"]); ok { action.HomepageURL = strings.TrimSpace(homepageURL) } +} + +func parseManifestBootstrapActionBoolFields(action *repositoryPackageBootstrapAction, actionMap map[string]any) { if optional, ok := actionMap["optional"].(bool); ok { action.Optional = optional } if existingOnly, ok := actionMap["existing-only"].(bool); ok { action.ExistingOnly = existingOnly } +} + +func parseManifestBootstrapActionComplexFields(action *repositoryPackageBootstrapAction, actionMap map[string]any, manifestPath string, index int) error { if enumValues, ok, err := stringListValue(actionMap["enum"]); err != nil { - return repositoryPackageBootstrapAction{}, manifestBootstrapFieldError(manifestPath, index, "enum", err) + return manifestBootstrapFieldError(manifestPath, index, "enum", err) } else if ok { action.Enum = enumValues } if events, ok, err := stringListValue(actionMap["events"]); err != nil { - return repositoryPackageBootstrapAction{}, manifestBootstrapFieldError(manifestPath, index, "events", err) + return manifestBootstrapFieldError(manifestPath, index, "events", err) } else if ok { action.Events = events } if _, exists := actionMap["when"]; exists { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].when is not supported yet. Example: remove the when field and keep only supported keys such as type, name, and prompt", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].when is not supported yet. Example: remove the when field and keep only supported keys such as type, name, and prompt", manifestPath, index) } if permissionsValue, exists := actionMap["permissions"]; exists { permissions, err := stringMapValue(permissionsValue) if err != nil { - return repositoryPackageBootstrapAction{}, manifestBootstrapFieldError(manifestPath, index, "permissions", err) + return manifestBootstrapFieldError(manifestPath, index, "permissions", err) } action.Permissions = permissions } + return nil +} + +func parseManifestBootstrapActionValidate(action *repositoryPackageBootstrapAction, actionType string, manifestPath string, index int) error { switch actionType { case "require-owner-type": if action.Owner != "" && action.Owner != "repo" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].owner must be 'repo' when type=require-owner-type. Example: { type: require-owner-type, owner: repo, value: org }", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].owner must be 'repo' when type=require-owner-type. Example: { type: require-owner-type, owner: repo, value: org }", manifestPath, index) } if action.Value != "any" && action.Value != "org" && action.Value != "user" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].value must be one of: any, org, user. Example: { type: require-owner-type, value: org }", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].value must be one of: any, org, user. Example: { type: require-owner-type, value: org }", manifestPath, index) } case "repo-variable": if action.Name == "" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].name is required when type=repo-variable. Example: { type: repo-variable, name: EXAMPLE, prompt: Enter a value }", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].name is required when type=repo-variable. Example: { type: repo-variable, name: EXAMPLE, prompt: Enter a value }", manifestPath, index) } if action.Prompt == "" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].prompt is required when type=repo-variable. Example: { type: repo-variable, name: EXAMPLE, prompt: Enter a value }", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].prompt is required when type=repo-variable. Example: { type: repo-variable, name: EXAMPLE, prompt: Enter a value }", manifestPath, index) } case "repo-secret": if action.Name == "" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].name is required when type=repo-secret. Example: { type: repo-secret, name: EXAMPLE_SECRET, prompt: Enter a secret }", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].name is required when type=repo-secret. Example: { type: repo-secret, name: EXAMPLE_SECRET, prompt: Enter a secret }", manifestPath, index) } if action.Prompt == "" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].prompt is required when type=repo-secret. Example: { type: repo-secret, name: EXAMPLE_SECRET, prompt: Enter a secret }", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].prompt is required when type=repo-secret. Example: { type: repo-secret, name: EXAMPLE_SECRET, prompt: Enter a secret }", manifestPath, index) } case "github-app": - if action.AppName == "" && action.Name != "" { - action.AppName = action.Name - } - if action.ExistingOnly && action.Mode != "" && action.Mode != "existing" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].existing-only requires mode to be 'existing' or unset. Remove mode=%q or set it to 'existing'", manifestPath, index, action.Mode) - } - if action.ExistingOnly && action.Mode == "" { - action.Mode = "existing" - } - if action.Mode == "" { - action.Mode = "create-or-existing" - } - if action.Mode != "create-or-existing" && action.Mode != "existing" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].mode must be one of: create-or-existing, existing. Example: { type: github-app, mode: existing, app-id-variable: APP_ID, private-key-secret: APP_PRIVATE_KEY }", manifestPath, index) - } - if action.Owner != "" && action.Owner != "repo" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].owner must be 'repo' when type=github-app. Example: { type: github-app, owner: repo, app-id-variable: APP_ID, private-key-secret: APP_PRIVATE_KEY }", manifestPath, index) - } - if action.AppIDVariable == "" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].app-id-variable is required when type=github-app. Example: { type: github-app, app-id-variable: APP_ID, private-key-secret: APP_PRIVATE_KEY }", manifestPath, index) - } - if action.PrivateKeySecret == "" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].private-key-secret is required when type=github-app. Example: { type: github-app, app-id-variable: APP_ID, private-key-secret: APP_PRIVATE_KEY }", manifestPath, index) - } + return parseManifestBootstrapActionValidateGitHubApp(action, manifestPath, index) case "copilot-auth": if action.Secret == "" { action.Secret = "COPILOT_GITHUB_TOKEN" @@ -206,21 +210,49 @@ func parseManifestBootstrapAction(actionType string, actionMap map[string]any, m action.Strategy = "prompt-if-actions-auth-unavailable" } if action.Strategy != "prompt-if-actions-auth-unavailable" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].strategy must be 'prompt-if-actions-auth-unavailable'. Example: { type: copilot-auth, strategy: prompt-if-actions-auth-unavailable }", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].strategy must be 'prompt-if-actions-auth-unavailable'. Example: { type: copilot-auth, strategy: prompt-if-actions-auth-unavailable }", manifestPath, index) } case "commit-and-push": if action.Message == "" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].message is required when type=commit-and-push. Example: { type: commit-and-push, message: Bootstrap repository changes }", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].message is required when type=commit-and-push. Example: { type: commit-and-push, message: Bootstrap repository changes }", manifestPath, index) } case "handoff": if action.Message == "" { - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].message is required when type=handoff. Example: { type: handoff, message: Continue with repository-specific setup. }", manifestPath, index) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].message is required when type=handoff. Example: { type: handoff, message: Continue with repository-specific setup. }", manifestPath, index) } default: - return repositoryPackageBootstrapAction{}, fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].type %q is not supported. Example: use one of %s", manifestPath, index, actionType, bootstrapActionTypeExample) + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].type %q is not supported. Example: use one of %s", manifestPath, index, actionType, bootstrapActionTypeExample) } - return action, nil + return nil +} + +func parseManifestBootstrapActionValidateGitHubApp(action *repositoryPackageBootstrapAction, manifestPath string, index int) error { + if action.AppName == "" && action.Name != "" { + action.AppName = action.Name + } + if action.ExistingOnly && action.Mode != "" && action.Mode != "existing" { + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].existing-only requires mode to be 'existing' or unset. Remove mode=%q or set it to 'existing'", manifestPath, index, action.Mode) + } + if action.ExistingOnly && action.Mode == "" { + action.Mode = "existing" + } + if action.Mode == "" { + action.Mode = "create-or-existing" + } + if action.Mode != "create-or-existing" && action.Mode != "existing" { + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].mode must be one of: create-or-existing, existing. Example: { type: github-app, mode: existing, app-id-variable: APP_ID, private-key-secret: APP_PRIVATE_KEY }", manifestPath, index) + } + if action.Owner != "" && action.Owner != "repo" { + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].owner must be 'repo' when type=github-app. Example: { type: github-app, owner: repo, app-id-variable: APP_ID, private-key-secret: APP_PRIVATE_KEY }", manifestPath, index) + } + if action.AppIDVariable == "" { + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].app-id-variable is required when type=github-app. Example: { type: github-app, app-id-variable: APP_ID, private-key-secret: APP_PRIVATE_KEY }", manifestPath, index) + } + if action.PrivateKeySecret == "" { + return fmt.Errorf("invalid Agentic Workflow manifest %q: config[%d].private-key-secret is required when type=github-app. Example: { type: github-app, app-id-variable: APP_ID, private-key-secret: APP_PRIVATE_KEY }", manifestPath, index) + } + return nil } func stringListValue(value any) ([]string, bool, error) { diff --git a/pkg/cli/bootstrap_shared.go b/pkg/cli/bootstrap_shared.go index 23c71bfcdc3..92b74cbe3ad 100644 --- a/pkg/cli/bootstrap_shared.go +++ b/pkg/cli/bootstrap_shared.go @@ -55,42 +55,11 @@ func isBootstrapInitMarkerSatisfied(baseDir string, marker string) (bool, error) switch marker { case ".gitattributes": - content, err := os.ReadFile(markerPath) - if err != nil { - return false, fmt.Errorf("failed to inspect %s: %w", marker, err) - } - return strings.Contains(string(content), constants.WorkflowsLockYmlGitAttributesEntry), nil + return isBootstrapInitMarkerSatisfiedGitAttributes(markerPath, marker) case bootstrapMCPConfigPath: - content, err := os.ReadFile(markerPath) - if err != nil { - return false, fmt.Errorf("failed to inspect %s: %w", marker, err) - } - var config MCPConfig - if err := json.Unmarshal(content, &config); err != nil { - return false, nil - } - servers := config.MCPServers - if len(servers) == 0 { - servers = config.Servers - } - server, ok := servers["github-agentic-workflows"] - if !ok { - return false, nil - } - if strings.TrimSpace(server.Command) != "gh" { - return false, nil - } - return len(server.Args) >= 2 && server.Args[0] == "aw" && server.Args[1] == "mcp-server", nil + return isBootstrapInitMarkerSatisfiedMCPConfig(markerPath, marker) case bootstrapCopilotSetupPath: - content, err := os.ReadFile(markerPath) - if err != nil { - return false, fmt.Errorf("failed to inspect %s: %w", marker, err) - } - steps := string(content) - hasLegacyInstall := strings.Contains(steps, "install-gh-aw.sh") || - (strings.Contains(steps, "Install gh-aw extension") && strings.Contains(steps, "curl -fsSL")) - hasActionInstall := strings.Contains(steps, "actions/setup-cli") - return hasLegacyInstall || hasActionInstall, nil + return isBootstrapInitMarkerSatisfiedCopilotSetup(markerPath, marker) case bootstrapAgenticSkillPath: expected, err := buildAgenticWorkflowsSkillContent() if err != nil { @@ -116,6 +85,49 @@ func isBootstrapInitMarkerSatisfied(baseDir string, marker string) (bool, error) } } +func isBootstrapInitMarkerSatisfiedGitAttributes(markerPath string, marker string) (bool, error) { + content, err := os.ReadFile(markerPath) + if err != nil { + return false, fmt.Errorf("failed to inspect %s: %w", marker, err) + } + return strings.Contains(string(content), constants.WorkflowsLockYmlGitAttributesEntry), nil +} + +func isBootstrapInitMarkerSatisfiedMCPConfig(markerPath string, marker string) (bool, error) { + content, err := os.ReadFile(markerPath) + if err != nil { + return false, fmt.Errorf("failed to inspect %s: %w", marker, err) + } + var config MCPConfig + if err := json.Unmarshal(content, &config); err != nil { + return false, nil + } + servers := config.MCPServers + if len(servers) == 0 { + servers = config.Servers + } + server, ok := servers["github-agentic-workflows"] + if !ok { + return false, nil + } + if strings.TrimSpace(server.Command) != "gh" { + return false, nil + } + return len(server.Args) >= 2 && server.Args[0] == "aw" && server.Args[1] == "mcp-server", nil +} + +func isBootstrapInitMarkerSatisfiedCopilotSetup(markerPath string, marker string) (bool, error) { + content, err := os.ReadFile(markerPath) + if err != nil { + return false, fmt.Errorf("failed to inspect %s: %w", marker, err) + } + steps := string(content) + hasLegacyInstall := strings.Contains(steps, "install-gh-aw.sh") || + (strings.Contains(steps, "Install gh-aw extension") && strings.Contains(steps, "curl -fsSL")) + hasActionInstall := strings.Contains(steps, "actions/setup-cli") + return hasLegacyInstall || hasActionInstall, nil +} + func expectedBootstrapInitMarkers(engineOverride string) []string { markers := []string{ ".gitattributes", diff --git a/pkg/cli/codemod_agent_session.go b/pkg/cli/codemod_agent_session.go index 23ffddfe553..ce7f34f9e77 100644 --- a/pkg/cli/codemod_agent_session.go +++ b/pkg/cli/codemod_agent_session.go @@ -16,73 +16,77 @@ func getAgentTaskToAgentSessionCodemod() Codemod { Description: "Replaces deprecated 'safe-outputs.create-agent-task' field with 'safe-outputs.create-agent-session'", IntroducedIn: "0.4.0", Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Check if safe-outputs.create-agent-task exists - safeOutputsValue, hasSafeOutputs := frontmatter["safe-outputs"] - if !hasSafeOutputs { - return content, false, nil - } + return getAgentTaskToAgentSessionCodemodApply(content, frontmatter) + }, + } +} - safeOutputsMap, ok := safeOutputsValue.(map[string]any) - if !ok { - return content, false, nil - } +func getAgentTaskToAgentSessionCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + // Check if safe-outputs.create-agent-task exists + safeOutputsValue, hasSafeOutputs := frontmatter["safe-outputs"] + if !hasSafeOutputs { + return content, false, nil + } - // Check if create-agent-task field exists in safe-outputs (deprecated) - _, hasAgentTask := safeOutputsMap["create-agent-task"] - if !hasAgentTask { - return content, false, nil - } + safeOutputsMap, ok := safeOutputsValue.(map[string]any) + if !ok { + return content, false, nil + } - // Check if create-agent-session already exists - if so, don't migrate to avoid data loss - _, hasAgentSession := safeOutputsMap["create-agent-session"] - if hasAgentSession { - agentSessionCodemodLog.Print("Skipping migration: create-agent-session already exists") - return content, false, nil - } + // Check if create-agent-task field exists in safe-outputs (deprecated) + _, hasAgentTask := safeOutputsMap["create-agent-task"] + if !hasAgentTask { + return content, false, nil + } + + // Check if create-agent-session already exists - if so, don't migrate to avoid data loss + _, hasAgentSession := safeOutputsMap["create-agent-session"] + if hasAgentSession { + agentSessionCodemodLog.Print("Skipping migration: create-agent-session already exists") + return content, false, nil + } + + newContent, applied, err := applyFrontmatterLineTransform(content, getAgentTaskToAgentSessionCodemodTransformLines) + if applied { + agentSessionCodemodLog.Print("Applied create-agent-task to create-agent-session migration") + } + return newContent, applied, err +} - newContent, applied, err := applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - var modified bool - var inSafeOutputsBlock bool - var safeOutputsIndent string - result := make([]string, len(lines)) - for i, line := range lines { - trimmedLine := strings.TrimSpace(line) +func getAgentTaskToAgentSessionCodemodTransformLines(lines []string) ([]string, bool) { + var modified bool + var inSafeOutputsBlock bool + var safeOutputsIndent string + result := make([]string, len(lines)) + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) - // Track if we're in the safe-outputs block - if strings.HasPrefix(trimmedLine, "safe-outputs:") { - inSafeOutputsBlock = true - safeOutputsIndent = getIndentation(line) - result[i] = line - continue - } + // Track if we're in the safe-outputs block + if strings.HasPrefix(trimmedLine, "safe-outputs:") { + inSafeOutputsBlock = true + safeOutputsIndent = getIndentation(line) + result[i] = line + continue + } - // Check if we've left the safe-outputs block - if inSafeOutputsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputsBlock = false - } - } + // Check if we've left the safe-outputs block + if inSafeOutputsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + if hasExitedBlock(line, safeOutputsIndent) { + inSafeOutputsBlock = false + } + } - // Replace create-agent-task with create-agent-session if in safe-outputs block - if inSafeOutputsBlock && strings.HasPrefix(trimmedLine, "create-agent-task:") { - replacedLine, didReplace := findAndReplaceInLine(line, "create-agent-task", "create-agent-session") - if didReplace { - result[i] = replacedLine - modified = true - agentSessionCodemodLog.Printf("Replaced safe-outputs.create-agent-task with safe-outputs.create-agent-session on line %d", i+1) - } else { - result[i] = line - } - } else { - result[i] = line - } - } - return result, modified - }) - if applied { - agentSessionCodemodLog.Print("Applied create-agent-task to create-agent-session migration") + // Replace create-agent-task with create-agent-session if in safe-outputs block + if inSafeOutputsBlock && strings.HasPrefix(trimmedLine, "create-agent-task:") { + replacedLine, didReplace := findAndReplaceInLine(line, "create-agent-task", "create-agent-session") + if didReplace { + result[i] = replacedLine + modified = true + agentSessionCodemodLog.Printf("Replaced safe-outputs.create-agent-task with safe-outputs.create-agent-session on line %d", i+1) + continue } - return newContent, applied, err - }, + } + result[i] = line } + return result, modified } diff --git a/pkg/cli/codemod_assign_to_agent.go b/pkg/cli/codemod_assign_to_agent.go index 69427760c0a..bda693a4176 100644 --- a/pkg/cli/codemod_assign_to_agent.go +++ b/pkg/cli/codemod_assign_to_agent.go @@ -17,101 +17,96 @@ func getAssignToAgentDefaultAgentCodemod() Codemod { Description: "Renames the deprecated 'default-agent' field to 'name' inside 'safe-outputs.assign-to-agent'", IntroducedIn: "0.12.0", Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Check if safe-outputs.assign-to-agent.default-agent exists - safeOutputsValue, hasSafeOutputs := frontmatter["safe-outputs"] - if !hasSafeOutputs { - return content, false, nil - } - - safeOutputsMap, ok := safeOutputsValue.(map[string]any) - if !ok { - return content, false, nil - } - - assignToAgentValue, hasAssignToAgent := safeOutputsMap["assign-to-agent"] - if !hasAssignToAgent { - return content, false, nil - } + return getAssignToAgentDefaultAgentCodemodApply(content, frontmatter) + }, + } +} - assignToAgentMap, ok := assignToAgentValue.(map[string]any) - if !ok { - return content, false, nil - } +func getAssignToAgentDefaultAgentCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + // Check if safe-outputs.assign-to-agent.default-agent exists + safeOutputsValue, hasSafeOutputs := frontmatter["safe-outputs"] + if !hasSafeOutputs { + return content, false, nil + } - // Check if deprecated 'default-agent' key exists - _, hasDefaultAgent := assignToAgentMap["default-agent"] - if !hasDefaultAgent { - return content, false, nil - } + safeOutputsMap, ok := safeOutputsValue.(map[string]any) + if !ok { + return content, false, nil + } - // Don't migrate if 'name' already exists to avoid overwriting it - _, hasName := assignToAgentMap["name"] - if hasName { - assignToAgentCodemodLog.Print("Skipping migration: 'name' already exists in assign-to-agent config") - return content, false, nil - } + assignToAgentValue, hasAssignToAgent := safeOutputsMap["assign-to-agent"] + if !hasAssignToAgent { + return content, false, nil + } - newContent, applied, err := applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - var modified bool - var inSafeOutputsBlock bool - var safeOutputsIndent string - var inAssignToAgentBlock bool - var assignToAgentIndent string - result := make([]string, len(lines)) - for i, line := range lines { - trimmedLine := strings.TrimSpace(line) + assignToAgentMap, ok := assignToAgentValue.(map[string]any) + if !ok { + return content, false, nil + } - // Track if we're in the safe-outputs block - if strings.HasPrefix(trimmedLine, "safe-outputs:") { - inSafeOutputsBlock = true - safeOutputsIndent = getIndentation(line) - result[i] = line - continue - } + // Check if deprecated 'default-agent' key exists + _, hasDefaultAgent := assignToAgentMap["default-agent"] + if !hasDefaultAgent { + return content, false, nil + } - // Check if we've left the safe-outputs block - if inSafeOutputsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputsBlock = false - inAssignToAgentBlock = false - } - } + // Don't migrate if 'name' already exists to avoid overwriting it + _, hasName := assignToAgentMap["name"] + if hasName { + assignToAgentCodemodLog.Print("Skipping migration: 'name' already exists in assign-to-agent config") + return content, false, nil + } - // Track if we're in the assign-to-agent block within safe-outputs - if inSafeOutputsBlock && strings.HasPrefix(trimmedLine, "assign-to-agent:") { - inAssignToAgentBlock = true - assignToAgentIndent = getIndentation(line) - result[i] = line - continue - } + newContent, applied, err := applyFrontmatterLineTransform(content, getAssignToAgentDefaultAgentCodemodTransformLines) + if applied { + assignToAgentCodemodLog.Print("Applied assign-to-agent default-agent to name migration") + } + return newContent, applied, err +} - // Check if we've left the assign-to-agent block - if inAssignToAgentBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, assignToAgentIndent) { - inAssignToAgentBlock = false - } - } +func getAssignToAgentDefaultAgentCodemodTransformLines(lines []string) ([]string, bool) { + var modified bool + var inSafeOutputsBlock bool + var safeOutputsIndent string + var inAssignToAgentBlock bool + var assignToAgentIndent string + result := make([]string, len(lines)) + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) - // Replace default-agent with name if in assign-to-agent block - if inAssignToAgentBlock && strings.HasPrefix(trimmedLine, "default-agent:") { - replacedLine, didReplace := findAndReplaceInLine(line, "default-agent", "name") - if didReplace { - result[i] = replacedLine - modified = true - assignToAgentCodemodLog.Printf("Replaced safe-outputs.assign-to-agent.default-agent with safe-outputs.assign-to-agent.name on line %d", i+1) - } else { - result[i] = line - } - } else { - result[i] = line - } - } - return result, modified - }) - if applied { - assignToAgentCodemodLog.Print("Applied assign-to-agent default-agent to name migration") + if strings.HasPrefix(trimmedLine, "safe-outputs:") { + inSafeOutputsBlock = true + safeOutputsIndent = getIndentation(line) + result[i] = line + continue + } + if inSafeOutputsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + if hasExitedBlock(line, safeOutputsIndent) { + inSafeOutputsBlock = false + inAssignToAgentBlock = false } - return newContent, applied, err - }, + } + if inSafeOutputsBlock && strings.HasPrefix(trimmedLine, "assign-to-agent:") { + inAssignToAgentBlock = true + assignToAgentIndent = getIndentation(line) + result[i] = line + continue + } + if inAssignToAgentBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + if hasExitedBlock(line, assignToAgentIndent) { + inAssignToAgentBlock = false + } + } + if inAssignToAgentBlock && strings.HasPrefix(trimmedLine, "default-agent:") { + replacedLine, didReplace := findAndReplaceInLine(line, "default-agent", "name") + if didReplace { + result[i] = replacedLine + modified = true + assignToAgentCodemodLog.Printf("Replaced safe-outputs.assign-to-agent.default-agent with safe-outputs.assign-to-agent.name on line %d", i+1) + continue + } + } + result[i] = line } + return result, modified } diff --git a/pkg/cli/codemod_bash_single_quoted_args.go b/pkg/cli/codemod_bash_single_quoted_args.go index 5040c58c987..70a5212bfbd 100644 --- a/pkg/cli/codemod_bash_single_quoted_args.go +++ b/pkg/cli/codemod_bash_single_quoted_args.go @@ -22,73 +22,82 @@ func getBashSingleQuotedArgsCodemod() Codemod { Description: "Rewrites tools.bash entries like grep -n 'foo' to grep -n \"foo\" when safe, reducing Copilot shell() truncation warnings.", IntroducedIn: "0.39.0", Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - toolsValue, hasTools := frontmatter["tools"] - if !hasTools { - return content, false, nil - } + return getBashSingleQuotedArgsCodemodApply(content, frontmatter) + }, + } +} - toolsMap, ok := toolsValue.(map[string]any) - if !ok { - return content, false, nil - } +func getBashSingleQuotedArgsCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + toolsValue, hasTools := frontmatter["tools"] + if !hasTools { + return content, false, nil + } - bashValue, hasBash := toolsMap["bash"] - if !hasBash { - return content, false, nil - } + toolsMap, ok := toolsValue.(map[string]any) + if !ok { + return content, false, nil + } - bashCommands, ok := bashValue.([]any) - if !ok { - return content, false, nil - } + bashValue, hasBash := toolsMap["bash"] + if !hasBash { + return content, false, nil + } - updated := make([]any, len(bashCommands)) - copy(updated, bashCommands) - - changed := false - var unsafeCommands []string - for i, cmd := range bashCommands { - cmdStr, ok := cmd.(string) - if !ok { - continue - } - - rewritten, safe, rewrittenChanged := rewriteSingleQuotedBashArgs(cmdStr) - if !safe { - unsafeCommands = append(unsafeCommands, cmdStr) - continue - } - if rewrittenChanged { - updated[i] = rewritten - changed = true - } - } + bashCommands, ok := bashValue.([]any) + if !ok { + return content, false, nil + } - for _, cmd := range unsafeCommands { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage( - fmt.Sprintf("tools.bash entry %q contains an unclosed single-quoted segment and could not be safely rewritten; left unchanged", cmd))) - } + updated, changed := getBashSingleQuotedArgsCodemodRewriteCommands(bashCommands) + if !changed { + return content, false, nil + } - if !changed { - return content, false, nil - } + toolsMap["bash"] = updated + frontmatter["tools"] = toolsMap - toolsMap["bash"] = updated - frontmatter["tools"] = toolsMap + result, err := parser.ExtractFrontmatterFromContent(content) + if err != nil { + return content, false, fmt.Errorf("failed to parse frontmatter for rewrite: %w", err) + } - result, err := parser.ExtractFrontmatterFromContent(content) - if err != nil { - return content, false, fmt.Errorf("failed to parse frontmatter for rewrite: %w", err) - } + bashSingleQuotedArgsCodemodLog.Print("Rewrote single-quoted tools.bash arguments to safe double-quoted forms") + updatedContent, err := reconstructWorkflowFileFromMap(frontmatter, result.Markdown) + if err != nil { + return content, false, fmt.Errorf("failed to reconstruct workflow content after rewrite: %w", err) + } + return updatedContent, true, nil +} - bashSingleQuotedArgsCodemodLog.Print("Rewrote single-quoted tools.bash arguments to safe double-quoted forms") - updatedContent, err := reconstructWorkflowFileFromMap(frontmatter, result.Markdown) - if err != nil { - return content, false, fmt.Errorf("failed to reconstruct workflow content after rewrite: %w", err) - } - return updatedContent, true, nil - }, +func getBashSingleQuotedArgsCodemodRewriteCommands(bashCommands []any) ([]any, bool) { + updated := make([]any, len(bashCommands)) + copy(updated, bashCommands) + + changed := false + var unsafeCommands []string + for i, cmd := range bashCommands { + cmdStr, ok := cmd.(string) + if !ok { + continue + } + + rewritten, safe, rewrittenChanged := rewriteSingleQuotedBashArgs(cmdStr) + if !safe { + unsafeCommands = append(unsafeCommands, cmdStr) + continue + } + if rewrittenChanged { + updated[i] = rewritten + changed = true + } } + + for _, cmd := range unsafeCommands { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage( + fmt.Sprintf("tools.bash entry %q contains an unclosed single-quoted segment and could not be safely rewritten; left unchanged", cmd))) + } + + return updated, changed } // rewriteSingleQuotedBashArgs rewrites single-quoted shell segments to @@ -108,18 +117,7 @@ func rewriteSingleQuotedBashArgs(cmd string) (string, bool, bool) { for i := 0; i < len(cmd); i++ { ch := cmd[i] if inDoubleQuotes { - b.WriteByte(ch) - if escaped { - escaped = false - continue - } - if ch == '\\' { - escaped = true - continue - } - if ch == '"' { - inDoubleQuotes = false - } + rewriteSingleQuotedBashArgsWriteDoubleQuotedByte(&b, ch, &escaped, &inDoubleQuotes) continue } @@ -139,26 +137,12 @@ func rewriteSingleQuotedBashArgs(cmd string) (string, bool, bool) { inDoubleQuotes = true continue case '\'': - j := i + 1 - for j < len(cmd) && cmd[j] != '\'' { - j++ - } - if j >= len(cmd) { + j, ok := rewriteSingleQuotedBashArgsFindClosingQuote(cmd, i+1) + if !ok { return cmd, false, false } - content := cmd[i+1 : j] - b.WriteByte('"') - for k := range len(content) { - contentCh := content[k] - switch contentCh { - case '\\', '"', '$', '`': - b.WriteByte('\\') - } - b.WriteByte(contentCh) - } - b.WriteByte('"') - + rewriteSingleQuotedBashArgsWriteDoubleQuotedLiteral(&b, cmd[i+1:j]) changed = true i = j continue @@ -173,3 +157,40 @@ func rewriteSingleQuotedBashArgs(cmd string) (string, bool, bool) { } return rewritten, true, true } + +func rewriteSingleQuotedBashArgsWriteDoubleQuotedByte(b *strings.Builder, ch byte, escaped *bool, inDoubleQuotes *bool) { + b.WriteByte(ch) + if *escaped { + *escaped = false + return + } + if ch == '\\' { + *escaped = true + return + } + if ch == '"' { + *inDoubleQuotes = false + } +} + +func rewriteSingleQuotedBashArgsFindClosingQuote(cmd string, start int) (int, bool) { + for i := start; i < len(cmd); i++ { + if cmd[i] == '\'' { + return i, true + } + } + return 0, false +} + +func rewriteSingleQuotedBashArgsWriteDoubleQuotedLiteral(b *strings.Builder, content string) { + b.WriteByte('"') + for k := range len(content) { + contentCh := content[k] + switch contentCh { + case '\\', '"', '$', '`': + b.WriteByte('\\') + } + b.WriteByte(contentCh) + } + b.WriteByte('"') +} diff --git a/pkg/cli/codemod_checkout_persist_credentials_false.go b/pkg/cli/codemod_checkout_persist_credentials_false.go index 44e7cfe14ca..39f9c76b909 100644 --- a/pkg/cli/codemod_checkout_persist_credentials_false.go +++ b/pkg/cli/codemod_checkout_persist_credentials_false.go @@ -8,6 +8,21 @@ import ( var checkoutPersistCredentialsFalseCodemodLog = logger.New("cli:codemod_checkout_persist_credentials_false") +type checkoutPersistCredentialsBlock struct { + start int + end int + indent string +} + +type checkoutPersistCredentialsStepScan struct { + usesIdx int + usesIndent string + withStart int + withEnd int + withIndent string + persistIdx int +} + // getCheckoutPersistCredentialsFalseCodemod ensures checkout steps set with.persist-credentials: false. func getCheckoutPersistCredentialsFalseCodemod() Codemod { return Codemod{ @@ -118,86 +133,90 @@ func transformSectionCheckoutPersistCredentials(lines []string, sectionName stri } func transformAgentJobCheckoutPersistCredentials(lines []string, sectionNames []string) ([]string, bool) { - jobsStart := -1 - jobsIndent := "" + jobsBlock, ok := transformAgentJobCheckoutPersistCredentialsFindJobs(lines) + if !ok { + return lines, false + } + + jobsLines := lines[jobsBlock.start : jobsBlock.end+1] + agentBlock, ok := transformAgentJobCheckoutPersistCredentialsFindAgent(jobsLines, jobsBlock.indent) + if !ok { + return lines, false + } + + agentLines := append([]string(nil), jobsLines[agentBlock.start:agentBlock.end+1]...) + agentLines, modified := transformAgentJobCheckoutPersistCredentialsSections(agentLines, sectionNames, agentBlock.indent) + if !modified { + return lines, false + } + + updatedJobsLines := make([]string, 0, len(jobsLines)) + updatedJobsLines = append(updatedJobsLines, jobsLines[:agentBlock.start]...) + updatedJobsLines = append(updatedJobsLines, agentLines...) + updatedJobsLines = append(updatedJobsLines, jobsLines[agentBlock.end+1:]...) + + result := make([]string, 0, len(lines)) + result = append(result, lines[:jobsBlock.start]...) + result = append(result, updatedJobsLines...) + result = append(result, lines[jobsBlock.end+1:]...) + return result, true +} + +func transformAgentJobCheckoutPersistCredentialsFindJobs(lines []string) (checkoutPersistCredentialsBlock, bool) { for i, line := range lines { trimmed := strings.TrimSpace(line) if isTopLevelKey(line) && strings.HasPrefix(trimmed, "jobs:") { - jobsStart = i - jobsIndent = getIndentation(line) - break + return checkoutPersistCredentialsBlock{ + start: i, + end: transformAgentJobCheckoutPersistCredentialsBlockEnd(lines, i, getIndentation(line)), + indent: getIndentation(line), + }, true } } - if jobsStart == -1 { - return lines, false - } + return checkoutPersistCredentialsBlock{}, false +} - jobsEnd := len(lines) - 1 - for i := jobsStart + 1; i < len(lines); i++ { +func transformAgentJobCheckoutPersistCredentialsBlockEnd(lines []string, start int, indent string) int { + end := len(lines) - 1 + for i := start + 1; i < len(lines); i++ { trimmed := strings.TrimSpace(lines[i]) if trimmed == "" || strings.HasPrefix(trimmed, "#") { continue } - if len(getIndentation(lines[i])) <= len(jobsIndent) { - jobsEnd = i - 1 - break + if len(getIndentation(lines[i])) <= len(indent) { + return i - 1 } } + return end +} - jobsLines := lines[jobsStart : jobsEnd+1] +func transformAgentJobCheckoutPersistCredentialsFindAgent(jobsLines []string, jobsIndent string) (checkoutPersistCredentialsBlock, bool) { jobsChildIndentLen, hasJobsChild := findDirectChildIndentLen(jobsLines, 0, len(jobsIndent)) if !hasJobsChild { - return lines, false + return checkoutPersistCredentialsBlock{}, false } - - agentStart := -1 - agentIndent := "" for i, line := range jobsLines { trimmed := strings.TrimSpace(line) indent := getIndentation(line) if len(indent) == jobsChildIndentLen && parseYAMLMapKey(trimmed) == "agent" { - agentStart = i - agentIndent = indent - break - } - } - if agentStart == -1 { - return lines, false - } - - agentEnd := len(jobsLines) - 1 - for i := agentStart + 1; i < len(jobsLines); i++ { - trimmed := strings.TrimSpace(jobsLines[i]) - if trimmed == "" || strings.HasPrefix(trimmed, "#") { - continue - } - if len(getIndentation(jobsLines[i])) <= len(agentIndent) { - agentEnd = i - 1 - break + return checkoutPersistCredentialsBlock{ + start: i, + end: transformAgentJobCheckoutPersistCredentialsBlockEnd(jobsLines, i, indent), + indent: indent, + }, true } } + return checkoutPersistCredentialsBlock{}, false +} - agentLines := append([]string(nil), jobsLines[agentStart:agentEnd+1]...) +func transformAgentJobCheckoutPersistCredentialsSections(agentLines []string, sectionNames []string, agentIndent string) ([]string, bool) { modified := false for _, sectionName := range sectionNames { var sectionChanged bool agentLines, sectionChanged = transformNestedSectionCheckoutPersistCredentials(agentLines, sectionName, agentIndent) modified = modified || sectionChanged } - if !modified { - return lines, false - } - - updatedJobsLines := make([]string, 0, len(jobsLines)) - updatedJobsLines = append(updatedJobsLines, jobsLines[:agentStart]...) - updatedJobsLines = append(updatedJobsLines, agentLines...) - updatedJobsLines = append(updatedJobsLines, jobsLines[agentEnd+1:]...) - - result := make([]string, 0, len(lines)) - result = append(result, lines[:jobsStart]...) - result = append(result, updatedJobsLines...) - result = append(result, lines[jobsEnd+1:]...) - return result, true + return agentLines, modified } func transformNestedSectionCheckoutPersistCredentials(lines []string, sectionName, parentIndent string) ([]string, bool) { @@ -306,13 +325,28 @@ func transformCheckoutWithinSection(sectionLines []string, sectionIndent string) } func ensureStepCheckoutPersistCredentials(stepLines []string, stepIndent string) ([]string, bool) { - usesIdx := -1 - usesIndent := "" - withStart := -1 - withEnd := -1 - withIndent := "" - persistIdx := -1 + scan, skip := ensureStepCheckoutPersistCredentialsScan(stepLines, stepIndent) + if skip || scan.usesIdx == -1 { + return stepLines, false + } + if scan.persistIdx != -1 { + persistLine := strings.TrimSpace(stepLines[scan.persistIdx]) + if persistExplicitTrue(persistLine) { + checkoutPersistCredentialsFalseCodemodLog.Print("Skipping checkout step update: explicit with.persist-credentials: true found") + } + return stepLines, false + } + + if scan.withStart != -1 { + return ensureStepCheckoutPersistCredentialsInsertIntoWith(stepLines, scan) + } + + return ensureStepCheckoutPersistCredentialsAddWith(stepLines, stepIndent, scan) +} + +func ensureStepCheckoutPersistCredentialsScan(stepLines []string, stepIndent string) (checkoutPersistCredentialsStepScan, bool) { + scan := checkoutPersistCredentialsStepScan{usesIdx: -1, withStart: -1, withEnd: -1, persistIdx: -1} for i := range stepLines { line := stepLines[i] trimmed := strings.TrimSpace(line) @@ -320,65 +354,59 @@ func ensureStepCheckoutPersistCredentials(stepLines []string, stepIndent string) usesMatch, usesValue, _ := parseStepKeyLine(trimmed, indent, stepIndent, "uses") if usesMatch && isCheckoutUsesValue(usesValue) { - usesIdx = i - isUsesInline := strings.HasPrefix(trimmed, "- uses:") && len(indent) == len(stepIndent) - if isUsesInline { - usesIndent = stepIndent + " " - } else { - usesIndent = indent + scan.usesIdx = i + scan.usesIndent = indent + if strings.HasPrefix(trimmed, "- uses:") && len(indent) == len(stepIndent) { + scan.usesIndent = stepIndent + " " } } withMatch, withValue, currentWithKeyIndentLen := parseStepKeyLine(trimmed, indent, stepIndent, "with") - if withMatch { - if withValue != "" && hasPersistKey(withValue) { - if persistExplicitTrue(withValue) { - checkoutPersistCredentialsFalseCodemodLog.Print("Skipping checkout step update: explicit with.persist-credentials: true found") - } - return stepLines, false - } - withStart = i - withEnd = i - withIndent = indent - withKeyIndentLen := currentWithKeyIndentLen - for j := i + 1; j < len(stepLines); j++ { - t := strings.TrimSpace(stepLines[j]) - if t == "" { - withEnd = j - continue - } - if effectiveStepLineIndentLen(t, getIndentation(stepLines[j]), stepIndent) <= withKeyIndentLen { - break - } - withEnd = j - if parseYAMLMapKey(t) == "persist-credentials" { - persistIdx = j - } + if withMatch && withValue != "" && hasPersistKey(withValue) { + if persistExplicitTrue(withValue) { + checkoutPersistCredentialsFalseCodemodLog.Print("Skipping checkout step update: explicit with.persist-credentials: true found") } + return scan, true + } + if withMatch { + scan.withStart = i + scan.withEnd = i + scan.withIndent = indent + scan = ensureStepCheckoutPersistCredentialsScanWithBlock(stepLines, stepIndent, i, currentWithKeyIndentLen, scan) } } + return scan, false +} - if usesIdx == -1 { - return stepLines, false - } - - if persistIdx != -1 { - persistLine := strings.TrimSpace(stepLines[persistIdx]) - if persistExplicitTrue(persistLine) { - checkoutPersistCredentialsFalseCodemodLog.Print("Skipping checkout step update: explicit with.persist-credentials: true found") +func ensureStepCheckoutPersistCredentialsScanWithBlock(stepLines []string, stepIndent string, start int, withKeyIndentLen int, scan checkoutPersistCredentialsStepScan) checkoutPersistCredentialsStepScan { + for j := start + 1; j < len(stepLines); j++ { + t := strings.TrimSpace(stepLines[j]) + if t == "" { + scan.withEnd = j + continue + } + if effectiveStepLineIndentLen(t, getIndentation(stepLines[j]), stepIndent) <= withKeyIndentLen { + break + } + scan.withEnd = j + if parseYAMLMapKey(t) == "persist-credentials" { + scan.persistIdx = j } - return stepLines, false } + return scan +} - if withStart != -1 { - insertAt := withEnd + 1 - insertLine := withIndent + " persist-credentials: false" - updated := append([]string{}, stepLines[:insertAt]...) - updated = append(updated, insertLine) - updated = append(updated, stepLines[insertAt:]...) - return updated, true - } +func ensureStepCheckoutPersistCredentialsInsertIntoWith(stepLines []string, scan checkoutPersistCredentialsStepScan) ([]string, bool) { + insertAt := scan.withEnd + 1 + insertLine := scan.withIndent + " persist-credentials: false" + updated := append([]string{}, stepLines[:insertAt]...) + updated = append(updated, insertLine) + updated = append(updated, stepLines[insertAt:]...) + return updated, true +} +func ensureStepCheckoutPersistCredentialsAddWith(stepLines []string, stepIndent string, scan checkoutPersistCredentialsStepScan) ([]string, bool) { + usesIndent := scan.usesIndent if usesIndent == "" { usesIndent = stepIndent + " " } @@ -386,7 +414,7 @@ func ensureStepCheckoutPersistCredentials(stepLines []string, stepIndent string) usesIndent + "with:", usesIndent + " persist-credentials: false", } - insertAt := usesIdx + 1 + insertAt := scan.usesIdx + 1 updated := append([]string{}, stepLines[:insertAt]...) updated = append(updated, insertLines...) updated = append(updated, stepLines[insertAt:]...) diff --git a/pkg/cli/codemod_cli_proxy_mode.go b/pkg/cli/codemod_cli_proxy_mode.go index bc4edf2dd43..ddfee7b7c9a 100644 --- a/pkg/cli/codemod_cli_proxy_mode.go +++ b/pkg/cli/codemod_cli_proxy_mode.go @@ -78,37 +78,12 @@ func hasToolsGitHubMode(frontmatter map[string]any) bool { } func addGitHubModeGhProxyToTools(lines []string) []string { - toolsLine := -1 - toolsIndent := "" - - for i, line := range lines { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "tools:") { - toolsLine = i - toolsIndent = getIndentation(line) - break - } - } - + toolsLine, toolsIndent := addGitHubModeGhProxyToToolsFindTools(lines) if toolsLine == -1 { return append(lines, "tools:", " github:", " mode: gh-proxy") } - githubLine := -1 - githubIndent := "" - toolsEnd := len(lines) - for i := toolsLine + 1; i < len(lines); i++ { - trimmed := strings.TrimSpace(lines[i]) - if trimmed != "" && !strings.HasPrefix(trimmed, "#") && hasExitedBlock(lines[i], toolsIndent) { - toolsEnd = i - break - } - if strings.HasPrefix(trimmed, "github:") && strings.HasPrefix(getIndentation(lines[i]), toolsIndent+" ") { - githubLine = i - githubIndent = getIndentation(lines[i]) - break - } - } + githubLine, githubIndent, toolsEnd := addGitHubModeGhProxyToToolsFindGitHub(lines, toolsLine, toolsIndent) if githubLine == -1 { result := make([]string, 0, len(lines)+2) @@ -145,3 +120,27 @@ func addGitHubModeGhProxyToTools(lines []string) []string { result = append(result, lines[insertAt:]...) return result } + +func addGitHubModeGhProxyToToolsFindTools(lines []string) (int, string) { + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "tools:") { + return i, getIndentation(line) + } + } + return -1, "" +} + +func addGitHubModeGhProxyToToolsFindGitHub(lines []string, toolsLine int, toolsIndent string) (int, string, int) { + toolsEnd := len(lines) + for i := toolsLine + 1; i < len(lines); i++ { + trimmed := strings.TrimSpace(lines[i]) + if trimmed != "" && !strings.HasPrefix(trimmed, "#") && hasExitedBlock(lines[i], toolsIndent) { + return -1, "", i + } + if strings.HasPrefix(trimmed, "github:") && strings.HasPrefix(getIndentation(lines[i]), toolsIndent+" ") { + return i, getIndentation(lines[i]), toolsEnd + } + } + return -1, "", toolsEnd +} diff --git a/pkg/cli/codemod_dependabot_permissions.go b/pkg/cli/codemod_dependabot_permissions.go index b41b1381688..87af83baeee 100644 --- a/pkg/cli/codemod_dependabot_permissions.go +++ b/pkg/cli/codemod_dependabot_permissions.go @@ -118,60 +118,25 @@ func ensureToolsetPermissions(lines []string, missing map[workflow.PermissionSco return lines, false } - permissionsIdx := -1 - permissionsIndent := "" - permissionsEnd := len(lines) - - for i, line := range lines { - if isTopLevelKey(line) && strings.HasPrefix(strings.TrimSpace(line), "permissions:") { - permissionsIdx = i - permissionsIndent = getIndentation(line) - for j := i + 1; j < len(lines); j++ { - if isTopLevelKey(lines[j]) { - permissionsEnd = j - break - } - } - break - } - } - + permissionsIdx, permissionsIndent, permissionsEnd := ensureToolsetPermissionsFindBlock(lines) if permissionsIdx == -1 { insertAt := findPermissionsInsertIndex(lines) - block := []string{"permissions:"} - for _, key := range sortedMissingPermissionKeys(missing) { - block = append(block, fmt.Sprintf(" %s: %s", key, missing[workflow.PermissionScope(key)])) - } - - result := make([]string, 0, len(lines)+len(block)) - result = append(result, lines[:insertAt]...) - result = append(result, block...) - result = append(result, lines[insertAt:]...) - return result, true + return ensureToolsetPermissionsInsertBlock(lines, insertAt, "", missing) } trimmedPermissionsLine := strings.TrimSpace(lines[permissionsIdx]) inlineValue := strings.TrimSpace(strings.TrimPrefix(trimmedPermissionsLine, "permissions:")) if inlineValue != "" && !strings.HasPrefix(inlineValue, "#") { - block := []string{"permissions:"} - for _, key := range sortedMissingPermissionKeys(missing) { - block = append(block, fmt.Sprintf(" %s: %s", key, missing[workflow.PermissionScope(key)])) - } - result := make([]string, 0, len(lines)+len(block)) - result = append(result, lines[:permissionsIdx]...) - result = append(result, block...) - result = append(result, lines[permissionsIdx+1:]...) - return result, true + return ensureToolsetPermissionsReplaceInline(lines, permissionsIdx, missing) } - updated := make([]string, len(lines)) - copy(updated, lines) - modified := false - remaining := make(map[string]workflow.PermissionLevel, len(missing)) - for scope, level := range missing { - remaining[string(scope)] = level - } + return ensureToolsetPermissionsMergeExisting(lines, permissionsIdx, permissionsIndent, permissionsEnd, missing) +} +func ensureToolsetPermissionsMergeExisting(lines []string, permissionsIdx int, permissionsIndent string, permissionsEnd int, missing map[workflow.PermissionScope]workflow.PermissionLevel) ([]string, bool) { + updated := append([]string(nil), lines...) + modified := false + remaining := ensureToolsetPermissionsRemaining(missing) for i := permissionsIdx + 1; i < permissionsEnd; i++ { trimmed := strings.TrimSpace(updated[i]) key := parseYAMLMapKey(trimmed) @@ -208,6 +173,58 @@ func ensureToolsetPermissions(lines []string, missing map[workflow.PermissionSco return result, true } +func ensureToolsetPermissionsRemaining(missing map[workflow.PermissionScope]workflow.PermissionLevel) map[string]workflow.PermissionLevel { + remaining := make(map[string]workflow.PermissionLevel, len(missing)) + for scope, level := range missing { + remaining[string(scope)] = level + } + return remaining +} + +func ensureToolsetPermissionsReplaceInline(lines []string, permissionsIdx int, missing map[workflow.PermissionScope]workflow.PermissionLevel) ([]string, bool) { + block := ensureToolsetPermissionsBuildBlock("", missing) + result := make([]string, 0, len(lines)+len(block)) + result = append(result, lines[:permissionsIdx]...) + result = append(result, block...) + result = append(result, lines[permissionsIdx+1:]...) + return result, true +} + +func ensureToolsetPermissionsFindBlock(lines []string) (int, string, int) { + for i, line := range lines { + if isTopLevelKey(line) && strings.HasPrefix(strings.TrimSpace(line), "permissions:") { + return i, getIndentation(line), ensureToolsetPermissionsBlockEnd(lines, i) + } + } + return -1, "", len(lines) +} + +func ensureToolsetPermissionsBlockEnd(lines []string, start int) int { + for j := start + 1; j < len(lines); j++ { + if isTopLevelKey(lines[j]) { + return j + } + } + return len(lines) +} + +func ensureToolsetPermissionsBuildBlock(indent string, missing map[workflow.PermissionScope]workflow.PermissionLevel) []string { + block := []string{indent + "permissions:"} + for _, key := range sortedMissingPermissionKeys(missing) { + block = append(block, fmt.Sprintf("%s %s: %s", indent, key, missing[workflow.PermissionScope(key)])) + } + return block +} + +func ensureToolsetPermissionsInsertBlock(lines []string, insertAt int, indent string, missing map[workflow.PermissionScope]workflow.PermissionLevel) ([]string, bool) { + block := ensureToolsetPermissionsBuildBlock(indent, missing) + result := make([]string, 0, len(lines)+len(block)) + result = append(result, lines[:insertAt]...) + result = append(result, block...) + result = append(result, lines[insertAt:]...) + return result, true +} + func findPermissionsInsertIndex(lines []string) int { onIdx := -1 onEnd := len(lines) diff --git a/pkg/cli/codemod_difc_proxy.go b/pkg/cli/codemod_difc_proxy.go index 64c6e815266..a3b4d333bb4 100644 --- a/pkg/cli/codemod_difc_proxy.go +++ b/pkg/cli/codemod_difc_proxy.go @@ -146,17 +146,11 @@ func addIntegrityProxyFalseToToolsGitHub(lines []string) []string { // Inside github block: inject integrity-proxy: false before the first sub-field if inGitHub && !fieldInserted && trimmed != "" && !strings.HasPrefix(trimmed, "#") { if hasExitedBlock(line, githubIndent) { - // Exited github block without seeing any sub-fields; use default indentation - fieldIndent := githubIndent + " " - result = append(result, fieldIndent+"integrity-proxy: false") - difcProxyCodemodLog.Printf("Added integrity-proxy: false to tools.github (before exit)") + result = addIntegrityProxyFalseToToolsGitHubAppend(result, githubIndent+" ", "before exit") fieldInserted = true inGitHub = false } else { - // Use the indentation of the first existing sub-field - fieldIndent := getIndentation(line) - result = append(result, fieldIndent+"integrity-proxy: false") - difcProxyCodemodLog.Printf("Added integrity-proxy: false to tools.github") + result = addIntegrityProxyFalseToToolsGitHubAppend(result, getIndentation(line), "") fieldInserted = true inGitHub = false } @@ -167,10 +161,18 @@ func addIntegrityProxyFalseToToolsGitHub(lines []string) []string { // Edge case: github block was the last entry in the file if inGitHub && !fieldInserted { - fieldIndent := githubIndent + " " - result = append(result, fieldIndent+"integrity-proxy: false") - difcProxyCodemodLog.Printf("Added integrity-proxy: false to tools.github (end of file)") + result = addIntegrityProxyFalseToToolsGitHubAppend(result, githubIndent+" ", "end of file") } return result } + +func addIntegrityProxyFalseToToolsGitHubAppend(result []string, fieldIndent string, detail string) []string { + result = append(result, fieldIndent+"integrity-proxy: false") + if detail == "" { + difcProxyCodemodLog.Printf("Added integrity-proxy: false to tools.github") + } else { + difcProxyCodemodLog.Printf("Added integrity-proxy: false to tools.github (%s)", detail) + } + return result +} diff --git a/pkg/cli/codemod_discussion_flag.go b/pkg/cli/codemod_discussion_flag.go index b62afc57faf..ac9b6dfc780 100644 --- a/pkg/cli/codemod_discussion_flag.go +++ b/pkg/cli/codemod_discussion_flag.go @@ -8,6 +8,14 @@ import ( var discussionFlagCodemodLog = logger.New("cli:codemod_discussion_flag") +type getDiscussionFlagRemovalCodemodTransformState struct { + inSafeOutputsBlock bool + safeOutputsIndent string + inAddCommentBlock bool + addCommentIndent string + inDiscussionField bool +} + // getDiscussionFlagRemovalCodemod creates a codemod for converting the deprecated discussion field in add-comment func getDiscussionFlagRemovalCodemod() Codemod { return Codemod{ @@ -16,112 +24,122 @@ func getDiscussionFlagRemovalCodemod() Codemod { Description: "Removes the deprecated 'safe-outputs.add-comment.discussion' field. Discussion targeting is now automatic based on context. Use 'discussions: false' to opt out of discussions:write permission.", IntroducedIn: "0.3.0", Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Check if safe-outputs exists - safeOutputsValue, hasSafeOutputs := frontmatter["safe-outputs"] - if !hasSafeOutputs { - return content, false, nil - } - - safeOutputsMap, ok := safeOutputsValue.(map[string]any) - if !ok { - return content, false, nil - } - - // Check if add-comment exists in safe-outputs - addCommentValue, hasAddComment := safeOutputsMap["add-comment"] - if !hasAddComment { - return content, false, nil - } - - addCommentMap, ok := addCommentValue.(map[string]any) - if !ok { - return content, false, nil - } - - // Check if discussion field exists in add-comment - _, hasDiscussion := addCommentMap["discussion"] - if !hasDiscussion { - return content, false, nil - } - - newContent, applied, err := applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - var result []string - var modified bool - var inSafeOutputsBlock bool - var safeOutputsIndent string - var inAddCommentBlock bool - var addCommentIndent string - var inDiscussionField bool - - for i, line := range lines { - trimmedLine := strings.TrimSpace(line) - - // Track if we're in the safe-outputs block - if strings.HasPrefix(trimmedLine, "safe-outputs:") { - inSafeOutputsBlock = true - safeOutputsIndent = getIndentation(line) - result = append(result, line) - continue - } - - // Check if we've left the safe-outputs block - if inSafeOutputsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputsBlock = false - inAddCommentBlock = false - } - } - - // Track if we're in the add-comment block within safe-outputs - if inSafeOutputsBlock && strings.HasPrefix(trimmedLine, "add-comment:") { - inAddCommentBlock = true - addCommentIndent = getIndentation(line) - result = append(result, line) - continue - } - - // Check if we've left the add-comment block - if inAddCommentBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, addCommentIndent) { - inAddCommentBlock = false - } - } - - // Remove discussion field line if in add-comment block - if inAddCommentBlock && strings.HasPrefix(trimmedLine, "discussion:") { - modified = true - inDiscussionField = true - discussionFlagCodemodLog.Printf("Removed safe-outputs.add-comment.discussion on line %d", i+1) - continue - } - - // Skip any nested content under the discussion field (shouldn't be any, but for completeness) - if inDiscussionField { - // Empty lines within the field block should be removed - if trimmedLine == "" { - continue - } - - currentIndent := getIndentation(line) - discussionIndent := addCommentIndent + " " // discussion would be 2 spaces more than add-comment - - // If this line has more indentation than discussion field, skip it - if len(currentIndent) > len(discussionIndent) { - discussionFlagCodemodLog.Printf("Removed nested discussion property on line %d: %s", i+1, trimmedLine) - continue - } - // We've exited the discussion field - inDiscussionField = false - } - - result = append(result, line) - } - return result, modified - }) - if applied { - discussionFlagCodemodLog.Print("Applied add-comment.discussion removal") - } - return newContent, applied, err + return getDiscussionFlagRemovalCodemodApply(content, frontmatter) }, } } + +func getDiscussionFlagRemovalCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + // Check if safe-outputs exists + safeOutputsValue, hasSafeOutputs := frontmatter["safe-outputs"] + if !hasSafeOutputs { + return content, false, nil + } + + safeOutputsMap, ok := safeOutputsValue.(map[string]any) + if !ok { + return content, false, nil + } + + // Check if add-comment exists in safe-outputs + addCommentValue, hasAddComment := safeOutputsMap["add-comment"] + if !hasAddComment { + return content, false, nil + } + + addCommentMap, ok := addCommentValue.(map[string]any) + if !ok { + return content, false, nil + } + + // Check if discussion field exists in add-comment + _, hasDiscussion := addCommentMap["discussion"] + if !hasDiscussion { + return content, false, nil + } + + newContent, applied, err := applyFrontmatterLineTransform(content, getDiscussionFlagRemovalCodemodTransformLines) + if applied { + discussionFlagCodemodLog.Print("Applied add-comment.discussion removal") + } + return newContent, applied, err +} + +func getDiscussionFlagRemovalCodemodTransformLines(lines []string) ([]string, bool) { + var result []string + var modified bool + var state getDiscussionFlagRemovalCodemodTransformState + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) + if getDiscussionFlagRemovalCodemodStartSafeOutputs(line, trimmedLine, &state, &result) { + continue + } + getDiscussionFlagRemovalCodemodUpdateBlockExits(line, trimmedLine, &state) + if getDiscussionFlagRemovalCodemodStartAddComment(line, trimmedLine, &state, &result) { + continue + } + if state.inAddCommentBlock && strings.HasPrefix(trimmedLine, "discussion:") { + modified = true + state.inDiscussionField = true + discussionFlagCodemodLog.Printf("Removed safe-outputs.add-comment.discussion on line %d", i+1) + continue + } + if getDiscussionFlagRemovalCodemodSkipDiscussionField(line, trimmedLine, i, &state) { + continue + } + result = append(result, line) + } + return result, modified +} + +func getDiscussionFlagRemovalCodemodStartSafeOutputs(line string, trimmedLine string, state *getDiscussionFlagRemovalCodemodTransformState, result *[]string) bool { + if strings.HasPrefix(trimmedLine, "safe-outputs:") { + state.inSafeOutputsBlock = true + state.safeOutputsIndent = getIndentation(line) + *result = append(*result, line) + return true + } + return false +} + +func getDiscussionFlagRemovalCodemodStartAddComment(line string, trimmedLine string, state *getDiscussionFlagRemovalCodemodTransformState, result *[]string) bool { + if state.inSafeOutputsBlock && strings.HasPrefix(trimmedLine, "add-comment:") { + state.inAddCommentBlock = true + state.addCommentIndent = getIndentation(line) + *result = append(*result, line) + return true + } + return false +} + +func getDiscussionFlagRemovalCodemodUpdateBlockExits(line string, trimmedLine string, state *getDiscussionFlagRemovalCodemodTransformState) { + if state.inSafeOutputsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + if hasExitedBlock(line, state.safeOutputsIndent) { + state.inSafeOutputsBlock = false + state.inAddCommentBlock = false + } + } + if state.inAddCommentBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + if hasExitedBlock(line, state.addCommentIndent) { + state.inAddCommentBlock = false + } + } +} + +func getDiscussionFlagRemovalCodemodSkipDiscussionField(line string, trimmedLine string, index int, state *getDiscussionFlagRemovalCodemodTransformState) bool { + if !state.inDiscussionField { + return false + } + if trimmedLine == "" { + return true + } + + currentIndent := getIndentation(line) + discussionIndent := state.addCommentIndent + " " + if len(currentIndent) > len(discussionIndent) { + discussionFlagCodemodLog.Printf("Removed nested discussion property on line %d: %s", index+1, trimmedLine) + return true + } + state.inDiscussionField = false + return false +} diff --git a/pkg/cli/codemod_discussion_trigger_categories.go b/pkg/cli/codemod_discussion_trigger_categories.go index 131933d903f..c8f1923c432 100644 --- a/pkg/cli/codemod_discussion_trigger_categories.go +++ b/pkg/cli/codemod_discussion_trigger_categories.go @@ -8,6 +8,15 @@ import ( var discussionTriggerCategoriesCodemodLog = logger.New("cli:codemod_discussion_trigger_categories") +type lowercaseDiscussionTriggerTypesInLinesState struct { + inOn bool + onIndent string + currentTrigger string + triggerIndent string + inTypes bool + typesIndent string +} + // getDiscussionTriggerCategoriesLowercaseCodemod lowercases discussion trigger category values // so source matches compile-time normalized values. func getDiscussionTriggerCategoriesLowercaseCodemod() Codemod { @@ -78,75 +87,38 @@ func lowercaseDiscussionTriggerTypesInLines(lines []string) ([]string, bool) { copy(result, lines) var modified bool - var inOn bool - var onIndent string - var currentTrigger string - var triggerIndent string - var inTypes bool - var typesIndent string + var state lowercaseDiscussionTriggerTypesInLinesState for i, line := range result { trimmed := strings.TrimSpace(line) indent := getIndentation(line) - if isTopLevelKey(line) && isOnBlockStartLine(trimmed) { - inOn = true - onIndent = indent - currentTrigger = "" - inTypes = false + if lowercaseDiscussionTriggerTypesInLinesStartOn(line, trimmed, indent, &state) { continue } - if inOn && isTopLevelKey(line) && len(indent) <= len(onIndent) && !isOnBlockStartLine(trimmed) { - inOn = false - currentTrigger = "" - inTypes = false - } - - if !inOn || trimmed == "" || strings.HasPrefix(trimmed, "#") { + lowercaseDiscussionTriggerTypesInLinesUpdateExits(line, trimmed, indent, &state) + if !state.inOn || trimmed == "" || strings.HasPrefix(trimmed, "#") { continue } - if currentTrigger != "" && len(indent) <= len(triggerIndent) { - currentTrigger = "" - inTypes = false - } - - if len(indent) > len(onIndent) { - trigger, isTrigger := getDiscussionTriggerFromLine(trimmed) - if isTrigger { - currentTrigger = trigger - triggerIndent = indent - inTypes = false - continue - } + if lowercaseDiscussionTriggerTypesInLinesStartTrigger(trimmed, indent, &state) { + continue } - - if currentTrigger == "" { + if state.currentTrigger == "" { continue } - if inTypes && len(indent) <= len(typesIndent) { - inTypes = false + updatedLine, changed, handled := lowercaseDiscussionTriggerTypesInLinesUpdateTypeLine(line, trimmed, indent, &state) + if changed { + result[i] = updatedLine + modified = true } - - if strings.HasPrefix(trimmed, "types:") { - typesIndent = indent - afterColon := strings.TrimSpace(strings.TrimPrefix(trimmed, "types:")) - if strings.HasPrefix(afterColon, "[") && strings.HasSuffix(afterColon, "]") { - updatedLine, changed := lowercaseInlineTypesArrayLine(line) - if changed { - result[i] = updatedLine - modified = true - } - inTypes = false - } else if afterColon == "" { - inTypes = true - } + if handled { continue } - if inTypes && strings.HasPrefix(strings.TrimSpace(line), "- ") { + if state.inTypes && strings.HasPrefix(strings.TrimSpace(line), "- ") { updatedLine, changed := lowercaseYAMLListItemLine(line) if changed { result[i] = updatedLine @@ -158,6 +130,63 @@ func lowercaseDiscussionTriggerTypesInLines(lines []string) ([]string, bool) { return result, modified } +func lowercaseDiscussionTriggerTypesInLinesStartOn(line string, trimmed string, indent string, state *lowercaseDiscussionTriggerTypesInLinesState) bool { + if isTopLevelKey(line) && isOnBlockStartLine(trimmed) { + state.inOn = true + state.onIndent = indent + state.currentTrigger = "" + state.inTypes = false + return true + } + return false +} + +func lowercaseDiscussionTriggerTypesInLinesUpdateExits(line string, trimmed string, indent string, state *lowercaseDiscussionTriggerTypesInLinesState) { + if state.inOn && isTopLevelKey(line) && len(indent) <= len(state.onIndent) && !isOnBlockStartLine(trimmed) { + state.inOn = false + state.currentTrigger = "" + state.inTypes = false + } + if state.currentTrigger != "" && len(indent) <= len(state.triggerIndent) { + state.currentTrigger = "" + state.inTypes = false + } + if state.inTypes && len(indent) <= len(state.typesIndent) { + state.inTypes = false + } +} + +func lowercaseDiscussionTriggerTypesInLinesStartTrigger(trimmed string, indent string, state *lowercaseDiscussionTriggerTypesInLinesState) bool { + if len(indent) <= len(state.onIndent) { + return false + } + trigger, isTrigger := getDiscussionTriggerFromLine(trimmed) + if !isTrigger { + return false + } + state.currentTrigger = trigger + state.triggerIndent = indent + state.inTypes = false + return true +} + +func lowercaseDiscussionTriggerTypesInLinesUpdateTypeLine(line string, trimmed string, indent string, state *lowercaseDiscussionTriggerTypesInLinesState) (string, bool, bool) { + if !strings.HasPrefix(trimmed, "types:") { + return line, false, false + } + state.typesIndent = indent + afterColon := strings.TrimSpace(strings.TrimPrefix(trimmed, "types:")) + if strings.HasPrefix(afterColon, "[") && strings.HasSuffix(afterColon, "]") { + updatedLine, changed := lowercaseInlineTypesArrayLine(line) + state.inTypes = false + return updatedLine, changed, true + } + if afterColon == "" { + state.inTypes = true + } + return line, false, true +} + func isOnBlockStartLine(trimmed string) bool { key, isBlockMappingKey := getBlockMappingKey(trimmed) return isBlockMappingKey && key == "on" diff --git a/pkg/cli/codemod_effective_tokens_to_ai_credits.go b/pkg/cli/codemod_effective_tokens_to_ai_credits.go index 186af90b4e6..fb0c32a451a 100644 --- a/pkg/cli/codemod_effective_tokens_to_ai_credits.go +++ b/pkg/cli/codemod_effective_tokens_to_ai_credits.go @@ -13,6 +13,13 @@ var effectiveTokensToAICreditsCodemodLog = logger.New("cli:codemod_effective_tok const effectiveTokensPerAICredit = 10000 +type getEffectiveTokensToAICreditsCodemodMigration struct { + maxAICredits bool + maxAICreditsNormalized string + maxDailyAICredits bool + maxDailyAICreditsNormalized string +} + // getEffectiveTokensToAICreditsCodemod migrates obsolete ET-based budget fields // to AI Credits equivalents: // - max-effective-tokens -> max-ai-credits @@ -27,72 +34,75 @@ func getEffectiveTokensToAICreditsCodemod() Codemod { Description: "Migrates obsolete 'max-effective-tokens' and 'max-daily-effective-tokens' to AI Credits equivalents, normalizing numeric values and skipping expressions.", IntroducedIn: "1.0.47", Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - if frontmatter == nil { - return content, false, nil - } + return getEffectiveTokensToAICreditsCodemodApply(content, frontmatter) + }, + } +} - _, hasMaxAICredits := frontmatter["max-ai-credits"] - _, hasMaxDailyAICredits := frontmatter["max-daily-ai-credits"] - - var maxAICreditsNormalized string - migrateMaxAICredits := false - if !hasMaxAICredits { - if raw, exists := frontmatter["max-effective-tokens"]; exists { - if normalized, ok := normalizeLegacyBudgetValue(raw, true); ok { - maxAICreditsNormalized = normalized - migrateMaxAICredits = true - } - } - } +func getEffectiveTokensToAICreditsCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + if frontmatter == nil { + return content, false, nil + } - var maxDailyAICreditsNormalized string - migrateMaxDailyAICredits := false - if !hasMaxDailyAICredits { - if raw, exists := frontmatter["max-daily-effective-tokens"]; exists { - if normalized, ok := normalizeLegacyBudgetValue(raw, true); ok { - maxDailyAICreditsNormalized = normalized - migrateMaxDailyAICredits = true - } - } - } + migration := getEffectiveTokensToAICreditsCodemodMigrationFromFrontmatter(frontmatter) + if !migration.maxAICredits && !migration.maxDailyAICredits { + return content, false, nil + } - if !migrateMaxAICredits && !migrateMaxDailyAICredits { - return content, false, nil - } + newContent, applied, err := applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { + return getEffectiveTokensToAICreditsCodemodTransformLines(lines, migration) + }) + if applied { + effectiveTokensToAICreditsCodemodLog.Printf( + "Migrated effective-token legacy fields (max-ai-credits=%t max-daily-ai-credits=%t)", + migration.maxAICredits, + migration.maxDailyAICredits, + ) + } + return newContent, applied, err +} - newContent, applied, err := applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - modified := false - result := make([]string, 0, len(lines)) - - for _, line := range lines { - trimmed := strings.TrimSpace(line) - if isTopLevelKey(line) { - if migrateMaxAICredits && strings.HasPrefix(trimmed, "max-effective-tokens:") { - result = append(result, rewriteTopLevelScalarLine(line, "max-ai-credits", maxAICreditsNormalized)) - modified = true - continue - } - if migrateMaxDailyAICredits && strings.HasPrefix(trimmed, "max-daily-effective-tokens:") { - result = append(result, rewriteTopLevelScalarLine(line, "max-daily-ai-credits", maxDailyAICreditsNormalized)) - modified = true - continue - } - } - result = append(result, line) - } - - return result, modified - }) - if applied { - effectiveTokensToAICreditsCodemodLog.Printf( - "Migrated effective-token legacy fields (max-ai-credits=%t max-daily-ai-credits=%t)", - migrateMaxAICredits, - migrateMaxDailyAICredits, - ) +func getEffectiveTokensToAICreditsCodemodMigrationFromFrontmatter(frontmatter map[string]any) getEffectiveTokensToAICreditsCodemodMigration { + var migration getEffectiveTokensToAICreditsCodemodMigration + if _, hasMaxAICredits := frontmatter["max-ai-credits"]; !hasMaxAICredits { + if raw, exists := frontmatter["max-effective-tokens"]; exists { + if normalized, ok := normalizeLegacyBudgetValue(raw, true); ok { + migration.maxAICreditsNormalized = normalized + migration.maxAICredits = true } - return newContent, applied, err - }, + } + } + if _, hasMaxDailyAICredits := frontmatter["max-daily-ai-credits"]; !hasMaxDailyAICredits { + if raw, exists := frontmatter["max-daily-effective-tokens"]; exists { + if normalized, ok := normalizeLegacyBudgetValue(raw, true); ok { + migration.maxDailyAICreditsNormalized = normalized + migration.maxDailyAICredits = true + } + } + } + return migration +} + +func getEffectiveTokensToAICreditsCodemodTransformLines(lines []string, migration getEffectiveTokensToAICreditsCodemodMigration) ([]string, bool) { + modified := false + result := make([]string, 0, len(lines)) + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if isTopLevelKey(line) && migration.maxAICredits && strings.HasPrefix(trimmed, "max-effective-tokens:") { + result = append(result, rewriteTopLevelScalarLine(line, "max-ai-credits", migration.maxAICreditsNormalized)) + modified = true + continue + } + if isTopLevelKey(line) && migration.maxDailyAICredits && strings.HasPrefix(trimmed, "max-daily-effective-tokens:") { + result = append(result, rewriteTopLevelScalarLine(line, "max-daily-ai-credits", migration.maxDailyAICreditsNormalized)) + modified = true + continue + } + result = append(result, line) } + + return result, modified } func normalizeLegacyBudgetValue(raw any, allowNegativeOne bool) (string, bool) { diff --git a/pkg/cli/codemod_engine_env_secrets.go b/pkg/cli/codemod_engine_env_secrets.go index 9247a2b9bdb..3947bd19010 100644 --- a/pkg/cli/codemod_engine_env_secrets.go +++ b/pkg/cli/codemod_engine_env_secrets.go @@ -150,77 +150,88 @@ func findUnsafeEngineEnvSecretKeys(envMap map[string]any, allowed map[string]str return unsafe } +type removeUnsafeEngineEnvKeysState struct { + inEngine bool + engineIndent string + inEnv bool + envIndent string + removingKey bool + removingKeyIndent string +} + func removeUnsafeEngineEnvKeys(lines []string, unsafeKeys map[string]struct { }) ([]string, bool) { result := make([]string, 0, len(lines)) modified := false - - inEngine := false - engineIndent := "" - inEnv := false - envIndent := "" - removingKey := false - removingKeyIndent := "" + state := removeUnsafeEngineEnvKeysState{} for _, line := range lines { - trimmed := strings.TrimSpace(line) - indent := getIndentation(line) - - if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { - inEngine = true - engineIndent = indent - inEnv = false - removingKey = false + keepLine, lineModified := removeUnsafeEngineEnvKeysKeepLine(line, &state, unsafeKeys) + if lineModified { + modified = true + } + if keepLine { result = append(result, line) - continue } + } - if inEngine && trimmed != "" && !strings.HasPrefix(trimmed, "#") && len(indent) <= len(engineIndent) { - inEngine = false - inEnv = false - removingKey = false - } + return result, modified +} - if inEngine && !inEnv && strings.HasPrefix(trimmed, "env:") && strings.TrimSpace(strings.TrimPrefix(trimmed, "env:")) == "" { - inEnv = true - envIndent = indent - removingKey = false - result = append(result, line) - continue - } +func removeUnsafeEngineEnvKeysKeepLine(line string, state *removeUnsafeEngineEnvKeysState, unsafeKeys map[string]struct { +}) (bool, bool) { + trimmed := strings.TrimSpace(line) + indent := getIndentation(line) - if inEnv && trimmed != "" && !strings.HasPrefix(trimmed, "#") && len(indent) <= len(envIndent) { - inEnv = false - removingKey = false - } + if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { + state.inEngine = true + state.engineIndent = indent + state.inEnv = false + state.removingKey = false + return true, false + } - if inEnv && removingKey { - if trimmed == "" { - continue - } - if strings.HasPrefix(trimmed, "#") && len(indent) > len(removingKeyIndent) { - continue - } - if len(indent) > len(removingKeyIndent) { - continue - } - removingKey = false - } + if state.inEngine && trimmed != "" && !strings.HasPrefix(trimmed, "#") && len(indent) <= len(state.engineIndent) { + state.inEngine = false + state.inEnv = false + state.removingKey = false + } - if inEnv && !removingKey && trimmed != "" && !strings.HasPrefix(trimmed, "#") && len(indent) > len(envIndent) { - key := parseYAMLMapKey(trimmed) - if key != "" && setutil.Contains(unsafeKeys, key) { - modified = true - removingKey = true - removingKeyIndent = indent - continue - } + if state.inEngine && !state.inEnv && strings.HasPrefix(trimmed, "env:") && strings.TrimSpace(strings.TrimPrefix(trimmed, "env:")) == "" { + state.inEnv = true + state.envIndent = indent + state.removingKey = false + return true, false + } + + if state.inEnv && trimmed != "" && !strings.HasPrefix(trimmed, "#") && len(indent) <= len(state.envIndent) { + state.inEnv = false + state.removingKey = false + } + + if state.inEnv && state.removingKey { + if trimmed == "" { + return false, false + } + if strings.HasPrefix(trimmed, "#") && len(indent) > len(state.removingKeyIndent) { + return false, false + } + if len(indent) > len(state.removingKeyIndent) { + return false, false } + state.removingKey = false + } - result = append(result, line) + if state.inEnv && !state.removingKey && trimmed != "" && !strings.HasPrefix(trimmed, "#") && len(indent) > len(state.envIndent) { + key := parseYAMLMapKey(trimmed) + if key != "" && setutil.Contains(unsafeKeys, key) { + state.removingKey = true + state.removingKeyIndent = indent + return false, true + } } - return result, modified + return true, false } func removeEmptyEngineEnvBlock(lines []string) []string { diff --git a/pkg/cli/codemod_engine_steps.go b/pkg/cli/codemod_engine_steps.go index bc5ac73c9b0..4f2a4098231 100644 --- a/pkg/cli/codemod_engine_steps.go +++ b/pkg/cli/codemod_engine_steps.go @@ -15,250 +15,272 @@ func getEngineStepsToTopLevelCodemod() Codemod { Name: "Move engine.steps to top-level steps", Description: "Moves the 'steps' field from under 'engine' to the top-level 'steps' field, as 'engine.steps' is no longer supported", IntroducedIn: "0.11.0", - Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Check if engine.steps exists in frontmatter - engineValue, hasEngine := frontmatter["engine"] - if !hasEngine { - return content, false, nil - } + Apply: getEngineStepsToTopLevelCodemodApply, + } +} - engineMap, isMap := engineValue.(map[string]any) - if !isMap { - // engine is a string, no steps to move - return content, false, nil - } +func getEngineStepsToTopLevelCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + // Check if engine.steps exists in frontmatter + engineValue, hasEngine := frontmatter["engine"] + if !hasEngine { + return content, false, nil + } - if _, hasSteps := engineMap["steps"]; !hasSteps { - return content, false, nil - } + engineMap, isMap := engineValue.(map[string]any) + if !isMap { + // engine is a string, no steps to move + return content, false, nil + } - // Determine if existing top-level steps is a sequence - hasTopLevelSteps := false - if stepsVal, exists := frontmatter["steps"]; exists { - if _, isSlice := stepsVal.([]any); isSlice { - hasTopLevelSteps = true - engineStepsCodemodLog.Print("Found existing top-level 'steps'") - } else { - engineStepsCodemodLog.Print("Top-level 'steps' exists but is not a sequence; treating as absent") - } - } + if _, hasSteps := engineMap["steps"]; !hasSteps { + return content, false, nil + } - return applyFrontmatterLineTransform(content, func(frontmatterLines []string) ([]string, bool) { - // Find engine block and the steps field within it - engineIndent := "" - stepsStartIdx := -1 - inEngineBlock := false + hasTopLevelSteps := getEngineStepsToTopLevelCodemodHasTopLevelSteps(frontmatter) + return applyFrontmatterLineTransform(content, func(frontmatterLines []string) ([]string, bool) { + return getEngineStepsToTopLevelCodemodTransform(frontmatterLines, hasTopLevelSteps) + }) +} - for i, line := range frontmatterLines { - trimmed := strings.TrimSpace(line) +func getEngineStepsToTopLevelCodemodHasTopLevelSteps(frontmatter map[string]any) bool { + if stepsVal, exists := frontmatter["steps"]; exists { + if _, isSlice := stepsVal.([]any); isSlice { + engineStepsCodemodLog.Print("Found existing top-level 'steps'") + return true + } + engineStepsCodemodLog.Print("Top-level 'steps' exists but is not a sequence; treating as absent") + } + return false +} - if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { - engineIndent = getIndentation(line) - inEngineBlock = true - engineStepsCodemodLog.Printf("Found 'engine:' block at line %d", i+1) - continue - } +func getEngineStepsToTopLevelCodemodTransform(frontmatterLines []string, hasTopLevelSteps bool) ([]string, bool) { + stepsStartIdx := getEngineStepsToTopLevelCodemodFindStepsStart(frontmatterLines) + if stepsStartIdx == -1 { + return frontmatterLines, false + } - // Check if we've exited the engine block - if inEngineBlock && trimmed != "" && !strings.HasPrefix(trimmed, "#") { - lineIndent := getIndentation(line) - if len(lineIndent) <= len(engineIndent) { - inEngineBlock = false - } - } + stepsEndIdx := getEngineStepsToTopLevelCodemodFindBlockEnd(frontmatterLines, stepsStartIdx) + engineStepsCodemodLog.Printf("'engine.steps' spans lines %d to %d", stepsStartIdx+1, stepsEndIdx+1) - // Look for steps: within engine block - if inEngineBlock && stepsStartIdx == -1 && strings.HasPrefix(trimmed, "steps:") { - stepsStartIdx = i - engineStepsCodemodLog.Printf("Found 'engine.steps' at line %d", i+1) - } - } + topLevelStepsLines := getEngineStepsToTopLevelCodemodUnindentSteps(frontmatterLines, stepsStartIdx, stepsEndIdx) + topLevelStepsEndIdx := -1 + if hasTopLevelSteps { + topLevelStepsEndIdx = getEngineStepsToTopLevelCodemodFindTopLevelStepsEnd(frontmatterLines) + } - if stepsStartIdx == -1 { - return frontmatterLines, false - } + withoutEngineSteps := getEngineStepsToTopLevelCodemodRemoveRange(frontmatterLines, stepsStartIdx, stepsEndIdx) + if getEngineStepsToTopLevelCodemodEngineBlockIsEmpty(withoutEngineSteps) { + engineStepsCodemodLog.Print("Engine block is empty after removing 'steps', removing it") + withoutEngineSteps = getEngineStepsToTopLevelCodemodRemoveEmptyEngineBlock(withoutEngineSteps) + } - // Find end of the steps block within engine - stepsIndent := getIndentation(frontmatterLines[stepsStartIdx]) - stepsEndIdx := stepsStartIdx - for j := stepsStartIdx + 1; j < len(frontmatterLines); j++ { - line := frontmatterLines[j] - trimmed := strings.TrimSpace(line) + result := getEngineStepsToTopLevelCodemodInsertSteps( + withoutEngineSteps, + topLevelStepsLines, + hasTopLevelSteps, + topLevelStepsEndIdx, + stepsStartIdx, + stepsEndIdx, + ) + engineStepsCodemodLog.Print("Successfully migrated 'engine.steps' to top-level 'steps'") + return result, true +} - if trimmed == "" { - continue - } +func getEngineStepsToTopLevelCodemodFindStepsStart(frontmatterLines []string) int { + engineIndent := "" + stepsStartIdx := -1 + inEngineBlock := false + for i, line := range frontmatterLines { + trimmed := strings.TrimSpace(line) - lineIndent := getIndentation(line) - if len(lineIndent) > len(stepsIndent) { - stepsEndIdx = j - } else { - break - } - } + if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { + engineIndent = getIndentation(line) + inEngineBlock = true + engineStepsCodemodLog.Printf("Found 'engine:' block at line %d", i+1) + continue + } - engineStepsCodemodLog.Printf("'engine.steps' spans lines %d to %d", stepsStartIdx+1, stepsEndIdx+1) + // Check if we've exited the engine block + if inEngineBlock && trimmed != "" && !strings.HasPrefix(trimmed, "#") { + lineIndent := getIndentation(line) + if len(lineIndent) <= len(engineIndent) { + inEngineBlock = false + } + } - // Extract the steps lines and un-indent them (remove the engine-level indentation) - topLevelStepsLines := make([]string, 0, stepsEndIdx-stepsStartIdx+1) - for i := stepsStartIdx; i <= stepsEndIdx; i++ { - line := frontmatterLines[i] - trimmed := strings.TrimSpace(line) - if trimmed == "" { - topLevelStepsLines = append(topLevelStepsLines, "") - continue - } - // Strip the stepsIndent prefix to un-indent to top level - if strings.HasPrefix(line, stepsIndent) { - topLevelStepsLines = append(topLevelStepsLines, line[len(stepsIndent):]) - } else { - topLevelStepsLines = append(topLevelStepsLines, trimmed) - } - } + // Look for steps: within engine block + if inEngineBlock && stepsStartIdx == -1 && strings.HasPrefix(trimmed, "steps:") { + stepsStartIdx = i + engineStepsCodemodLog.Printf("Found 'engine.steps' at line %d", i+1) + } + } + return stepsStartIdx +} - // Find existing top-level steps block end (if any) - topLevelStepsEndIdx := -1 - if hasTopLevelSteps { - // Find the end of the top-level steps block in the lines - for i, line := range frontmatterLines { - trimmed := strings.TrimSpace(line) - if isTopLevelKey(line) && strings.HasPrefix(trimmed, "steps:") { - topStepsIndent := getIndentation(line) - topLevelStepsEndIdx = i - for j := i + 1; j < len(frontmatterLines); j++ { - l := frontmatterLines[j] - t := strings.TrimSpace(l) - if t == "" { - continue - } - if len(getIndentation(l)) > len(topStepsIndent) { - topLevelStepsEndIdx = j - } else { - break - } - } - engineStepsCodemodLog.Printf("Top-level 'steps:' ends at line %d", topLevelStepsEndIdx+1) - break - } - } - } +func getEngineStepsToTopLevelCodemodFindBlockEnd(frontmatterLines []string, startIdx int) int { + stepsIndent := getIndentation(frontmatterLines[startIdx]) + stepsEndIdx := startIdx + for j := startIdx + 1; j < len(frontmatterLines); j++ { + line := frontmatterLines[j] + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + lineIndent := getIndentation(line) + if len(lineIndent) > len(stepsIndent) { + stepsEndIdx = j + } else { + break + } + } + return stepsEndIdx +} - // Build new frontmatter: remove engine.steps lines and insert at top level - // Pass 1: build lines without engine.steps - withoutEngineSteps := make([]string, 0, len(frontmatterLines)) - for i, line := range frontmatterLines { - if i >= stepsStartIdx && i <= stepsEndIdx { - continue - } - withoutEngineSteps = append(withoutEngineSteps, line) - } +func getEngineStepsToTopLevelCodemodUnindentSteps(frontmatterLines []string, stepsStartIdx int, stepsEndIdx int) []string { + stepsIndent := getIndentation(frontmatterLines[stepsStartIdx]) + topLevelStepsLines := make([]string, 0, stepsEndIdx-stepsStartIdx+1) + for i := stepsStartIdx; i <= stepsEndIdx; i++ { + line := frontmatterLines[i] + trimmed := strings.TrimSpace(line) + if trimmed == "" { + topLevelStepsLines = append(topLevelStepsLines, "") + continue + } + // Strip the stepsIndent prefix to un-indent to top level + if strings.HasPrefix(line, stepsIndent) { + topLevelStepsLines = append(topLevelStepsLines, line[len(stepsIndent):]) + } else { + topLevelStepsLines = append(topLevelStepsLines, trimmed) + } + } + return topLevelStepsLines +} - // Pass 1b: if the engine block is now empty (only blank lines or id: key), - // check whether any non-steps content remains under engine: - engineBlockIsEmpty := func() bool { - inEngine := false - engineIndentLen := 0 - for _, line := range withoutEngineSteps { - trimmed := strings.TrimSpace(line) - if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { - inEngine = true - engineIndentLen = len(getIndentation(line)) - // Check for inline value (e.g., "engine: claude") - val := strings.TrimPrefix(trimmed, "engine:") - if strings.TrimSpace(val) != "" { - return false - } - continue - } - if inEngine { - if trimmed == "" { - continue - } - lineIndentLen := len(getIndentation(line)) - if lineIndentLen <= engineIndentLen { - // Exited engine block with no content found - return true - } - // There is content under engine (e.g., id:, model:, env:) - return false - } - } - return inEngine // if we're still in engine at EOF, it's empty - }() +func getEngineStepsToTopLevelCodemodFindTopLevelStepsEnd(frontmatterLines []string) int { + for i, line := range frontmatterLines { + trimmed := strings.TrimSpace(line) + if isTopLevelKey(line) && strings.HasPrefix(trimmed, "steps:") { + topStepsIndent := getIndentation(line) + topLevelStepsEndIdx := getEngineStepsToTopLevelCodemodFindBlockEnd(frontmatterLines, i) + if topStepsIndent == getIndentation(line) { + engineStepsCodemodLog.Printf("Top-level 'steps:' ends at line %d", topLevelStepsEndIdx+1) + } + return topLevelStepsEndIdx + } + } + return -1 +} + +func getEngineStepsToTopLevelCodemodRemoveRange(frontmatterLines []string, stepsStartIdx int, stepsEndIdx int) []string { + withoutEngineSteps := make([]string, 0, len(frontmatterLines)) + for i, line := range frontmatterLines { + if i >= stepsStartIdx && i <= stepsEndIdx { + continue + } + withoutEngineSteps = append(withoutEngineSteps, line) + } + return withoutEngineSteps +} + +func getEngineStepsToTopLevelCodemodEngineBlockIsEmpty(withoutEngineSteps []string) bool { + inEngine := false + engineIndentLen := 0 + for _, line := range withoutEngineSteps { + trimmed := strings.TrimSpace(line) + if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { + inEngine = true + engineIndentLen = len(getIndentation(line)) + // Check for inline value (e.g., "engine: claude") + val := strings.TrimPrefix(trimmed, "engine:") + if strings.TrimSpace(val) != "" { + return false + } + continue + } + if inEngine { + if trimmed == "" { + continue + } + lineIndentLen := len(getIndentation(line)) + if lineIndentLen <= engineIndentLen { + return true + } + return false + } + } + return inEngine // if we're still in engine at EOF, it's empty +} - if engineBlockIsEmpty { - engineStepsCodemodLog.Print("Engine block is empty after removing 'steps', removing it") - // Remove the engine block (the engine: line and any blank lines around it) - cleaned := make([]string, 0, len(withoutEngineSteps)) - engineIndentLen := 0 - inEngine := false - for i, line := range withoutEngineSteps { - trimmed := strings.TrimSpace(line) - if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { - inEngine = true - engineIndentLen = len(getIndentation(line)) - // Remove trailing blank lines already added - for len(cleaned) > 0 && strings.TrimSpace(cleaned[len(cleaned)-1]) == "" { - cleaned = cleaned[:len(cleaned)-1] - } - _ = i - continue - } - if inEngine { - if trimmed == "" { - continue - } - if len(getIndentation(line)) <= engineIndentLen { - inEngine = false - } else { - continue - } - } - cleaned = append(cleaned, line) - } - withoutEngineSteps = cleaned - } +func getEngineStepsToTopLevelCodemodRemoveEmptyEngineBlock(withoutEngineSteps []string) []string { + cleaned := make([]string, 0, len(withoutEngineSteps)) + engineIndentLen := 0 + inEngine := false + for _, line := range withoutEngineSteps { + trimmed := strings.TrimSpace(line) + if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { + inEngine = true + engineIndentLen = len(getIndentation(line)) + // Remove trailing blank lines already added + for len(cleaned) > 0 && strings.TrimSpace(cleaned[len(cleaned)-1]) == "" { + cleaned = cleaned[:len(cleaned)-1] + } + continue + } + if inEngine { + if trimmed == "" { + continue + } + if len(getIndentation(line)) <= engineIndentLen { + inEngine = false + } else { + continue + } + } + cleaned = append(cleaned, line) + } + return cleaned +} - // Pass 2: insert engine steps at top level - var result []string - if !hasTopLevelSteps { - // Append engine steps at the end (as new top-level steps field) - result = append(withoutEngineSteps, topLevelStepsLines...) - engineStepsCodemodLog.Print("Added engine steps as new top-level 'steps'") - } else { - // Append engine step items after the top-level steps block - // Since we removed engine.steps lines, re-find the end of top-level steps - adjustedTopLevelEnd := topLevelStepsEndIdx - removedCount := stepsEndIdx - stepsStartIdx + 1 - // Only adjust if the engine.steps came before the top-level steps end - if stepsEndIdx < topLevelStepsEndIdx { - adjustedTopLevelEnd -= removedCount - } else if stepsStartIdx <= topLevelStepsEndIdx && stepsEndIdx >= topLevelStepsEndIdx { - // engine.steps overlaps with top-level steps end (shouldn't happen but handle gracefully) - adjustedTopLevelEnd -= removedCount - } +func getEngineStepsToTopLevelCodemodInsertSteps( + withoutEngineSteps []string, + topLevelStepsLines []string, + hasTopLevelSteps bool, + topLevelStepsEndIdx int, + stepsStartIdx int, + stepsEndIdx int, +) []string { + if !hasTopLevelSteps { + engineStepsCodemodLog.Print("Added engine steps as new top-level 'steps'") + return append(withoutEngineSteps, topLevelStepsLines...) + } + + adjustedTopLevelEnd := topLevelStepsEndIdx + removedCount := stepsEndIdx - stepsStartIdx + 1 + if stepsEndIdx < topLevelStepsEndIdx { + adjustedTopLevelEnd -= removedCount + } else if stepsStartIdx <= topLevelStepsEndIdx && stepsEndIdx >= topLevelStepsEndIdx { + adjustedTopLevelEnd -= removedCount + } - result = make([]string, 0, len(withoutEngineSteps)+len(topLevelStepsLines)) - insertedSteps := false - for i, line := range withoutEngineSteps { - result = append(result, line) - if !insertedSteps && i == adjustedTopLevelEnd { - // Append the step items (skip the "steps:" header since one already exists) - for _, stepLine := range topLevelStepsLines { - if strings.TrimSpace(stepLine) == "steps:" { - continue - } - result = append(result, stepLine) - } - insertedSteps = true - engineStepsCodemodLog.Print("Appended engine steps to existing top-level 'steps'") - } - } - } + result := make([]string, 0, len(withoutEngineSteps)+len(topLevelStepsLines)) + insertedSteps := false + for i, line := range withoutEngineSteps { + result = append(result, line) + if !insertedSteps && i == adjustedTopLevelEnd { + result = getEngineStepsToTopLevelCodemodAppendStepItems(result, topLevelStepsLines) + insertedSteps = true + engineStepsCodemodLog.Print("Appended engine steps to existing top-level 'steps'") + } + } + return result +} - engineStepsCodemodLog.Print("Successfully migrated 'engine.steps' to top-level 'steps'") - return result, true - }) - }, +func getEngineStepsToTopLevelCodemodAppendStepItems(result []string, topLevelStepsLines []string) []string { + for _, stepLine := range topLevelStepsLines { + if strings.TrimSpace(stepLine) == "steps:" { + continue + } + result = append(result, stepLine) } + return result } diff --git a/pkg/cli/codemod_engine_to_top_level_helpers.go b/pkg/cli/codemod_engine_to_top_level_helpers.go index 2ba6c2d7472..6356c82ba5e 100644 --- a/pkg/cli/codemod_engine_to_top_level_helpers.go +++ b/pkg/cli/codemod_engine_to_top_level_helpers.go @@ -42,72 +42,90 @@ func migrateEngineFieldToTopLevel( } return applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - for _, line := range lines { - trimmed := strings.TrimSpace(line) - if !isTopLevelKey(line) || !strings.HasPrefix(trimmed, "engine:") { - continue - } - inlineValue := strings.TrimSpace(strings.TrimPrefix(trimmed, "engine:")) - if strings.HasPrefix(inlineValue, "{") && strings.Contains(inlineValue, opts.engineField+":") { - if opts.log != nil { - opts.log.Print(opts.skipInlineMessage) - } - return lines, false - } - } + return migrateEngineFieldToTopLevelTransform(lines, opts, hasPreservedTopLevelField) + }) +} - fieldSuffix := "" - inEngineBlock := false - engineIndent := "" - engineFieldPrefix := opts.engineField + ":" - for _, line := range lines { - trimmed := strings.TrimSpace(line) - if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { - inEngineBlock = true - engineIndent = getIndentation(line) - continue - } - if inEngineBlock && trimmed != "" && !strings.HasPrefix(trimmed, "#") && len(getIndentation(line)) <= len(engineIndent) { - inEngineBlock = false - } - if inEngineBlock && strings.HasPrefix(trimmed, engineFieldPrefix) { - parts := strings.SplitN(line, ":", 2) - if len(parts) == 2 { - fieldSuffix = parts[1] - } - break - } - } +func migrateEngineFieldToTopLevelTransform(lines []string, opts migrateEngineFieldToTopLevelOptions, hasPreservedTopLevelField bool) ([]string, bool) { + if migrateEngineFieldToTopLevelHasInlineField(lines, opts) { + return lines, false + } + + fieldSuffix := migrateEngineFieldToTopLevelFieldSuffix(lines, opts.engineField) + result, removed := removeFieldFromBlock(lines, opts.engineField, "engine") + if !removed { + return lines, false + } - result, removed := removeFieldFromBlock(lines, opts.engineField, "engine") - if !removed { - return lines, false + if hasPreservedTopLevelField { + if opts.log != nil { + opts.log.Print(opts.removedMessage) } + return result, true + } - if hasPreservedTopLevelField { + withTopLevel := migrateEngineFieldToTopLevelInsertLine(result, opts.targetTopLevelField+":"+fieldSuffix) + if opts.log != nil { + opts.log.Print(opts.migratedMessage) + } + return withTopLevel, true +} + +func migrateEngineFieldToTopLevelHasInlineField(lines []string, opts migrateEngineFieldToTopLevelOptions) bool { + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if !isTopLevelKey(line) || !strings.HasPrefix(trimmed, "engine:") { + continue + } + inlineValue := strings.TrimSpace(strings.TrimPrefix(trimmed, "engine:")) + if strings.HasPrefix(inlineValue, "{") && strings.Contains(inlineValue, opts.engineField+":") { if opts.log != nil { - opts.log.Print(opts.removedMessage) + opts.log.Print(opts.skipInlineMessage) } - return result, true + return true } + } + return false +} - insertAt := 0 - for i, line := range result { - if isTopLevelKey(line) && strings.HasPrefix(strings.TrimSpace(line), "engine:") { - insertAt = i - break +func migrateEngineFieldToTopLevelFieldSuffix(lines []string, engineField string) string { + fieldSuffix := "" + inEngineBlock := false + engineIndent := "" + engineFieldPrefix := engineField + ":" + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if isTopLevelKey(line) && strings.HasPrefix(trimmed, "engine:") { + inEngineBlock = true + engineIndent = getIndentation(line) + continue + } + if inEngineBlock && trimmed != "" && !strings.HasPrefix(trimmed, "#") && len(getIndentation(line)) <= len(engineIndent) { + inEngineBlock = false + } + if inEngineBlock && strings.HasPrefix(trimmed, engineFieldPrefix) { + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + fieldSuffix = parts[1] } + break } + } + return fieldSuffix +} - topLevelLine := opts.targetTopLevelField + ":" + fieldSuffix - withTopLevel := make([]string, 0, len(result)+1) - withTopLevel = append(withTopLevel, result[:insertAt]...) - withTopLevel = append(withTopLevel, topLevelLine) - withTopLevel = append(withTopLevel, result[insertAt:]...) - - if opts.log != nil { - opts.log.Print(opts.migratedMessage) +func migrateEngineFieldToTopLevelInsertLine(lines []string, topLevelLine string) []string { + insertAt := 0 + for i, line := range lines { + if isTopLevelKey(line) && strings.HasPrefix(strings.TrimSpace(line), "engine:") { + insertAt = i + break } - return withTopLevel, true - }) + } + + withTopLevel := make([]string, 0, len(lines)+1) + withTopLevel = append(withTopLevel, lines[:insertAt]...) + withTopLevel = append(withTopLevel, topLevelLine) + withTopLevel = append(withTopLevel, lines[insertAt:]...) + return withTopLevel } diff --git a/pkg/cli/codemod_factory.go b/pkg/cli/codemod_factory.go index bf898f79356..01eb985dc34 100644 --- a/pkg/cli/codemod_factory.go +++ b/pkg/cli/codemod_factory.go @@ -105,148 +105,207 @@ func newMoveTopLevelKeyToOnBlockCodemod(cfg moveToOnBlockConfig) Codemod { Description: cfg.Description, IntroducedIn: cfg.IntroducedIn, Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Bail out if the top-level field does not exist. - if _, hasField := frontmatter[fieldKey]; !hasField { + return newMoveTopLevelKeyToOnBlockCodemodApply(content, frontmatter, cfg, fieldKey, fieldKeyPrefix) + }, + } +} + +func newMoveTopLevelKeyToOnBlockCodemodApply( + content string, + frontmatter map[string]any, + cfg moveToOnBlockConfig, + fieldKey string, + fieldKeyPrefix string, +) (string, bool, error) { + // Bail out if the top-level field does not exist. + if _, hasField := frontmatter[fieldKey]; !hasField { + return content, false, nil + } + + // Bail out if on. already exists. + if onValue, hasOn := frontmatter["on"]; hasOn { + if onMap, ok := onValue.(map[string]any); ok { + if _, hasOnField := onMap[fieldKey]; hasOnField { + cfg.Log.Printf("Both top-level '%s' and 'on.%s' exist - skipping migration", fieldKey, fieldKey) return content, false, nil } + } + } - // Bail out if on. already exists. - if onValue, hasOn := frontmatter["on"]; hasOn { - if onMap, ok := onValue.(map[string]any); ok { - if _, hasOnField := onMap[fieldKey]; hasOnField { - cfg.Log.Printf("Both top-level '%s' and 'on.%s' exist - skipping migration", fieldKey, fieldKey) - return content, false, nil - } - } + return applyFrontmatterLineTransform(content, func(frontmatterLines []string) ([]string, bool) { + return newMoveTopLevelKeyToOnBlockCodemodTransform(frontmatterLines, cfg, fieldKey, fieldKeyPrefix) + }) +} + +type newMoveTopLevelKeyToOnBlockCodemodLocations struct { + fieldLineIdx int + fieldLineValue string + onBlockIdx int + onIndent string +} + +func newMoveTopLevelKeyToOnBlockCodemodTransform( + frontmatterLines []string, + cfg moveToOnBlockConfig, + fieldKey string, + fieldKeyPrefix string, +) ([]string, bool) { + loc := newMoveTopLevelKeyToOnBlockCodemodFindLocations(frontmatterLines, cfg, fieldKey, fieldKeyPrefix) + if loc.fieldLineIdx == -1 { + return frontmatterLines, false + } + + fieldLines, fieldEndIdx := newMoveTopLevelKeyToOnBlockCodemodCollectFieldLines(frontmatterLines, loc, cfg) + cfg.Log.Printf("%s spans lines %d to %d (%d lines)", fieldKey, loc.fieldLineIdx+1, fieldEndIdx+1, len(fieldLines)) + + var result []string + if loc.onBlockIdx == -1 { + cfg.Log.Printf("No 'on:' block found - creating new one with %s", fieldKey) + result = newMoveTopLevelKeyToOnBlockCodemodCreateOnBlock(frontmatterLines, loc.fieldLineIdx, fieldEndIdx, fieldLines, fieldKeyPrefix) + } else { + cfg.Log.Printf("Found 'on:' block - adding %s to it", fieldKey) + result = newMoveTopLevelKeyToOnBlockCodemodInsertIntoOnBlock(frontmatterLines, loc, fieldEndIdx, fieldLines, fieldKey, fieldKeyPrefix) + } + + cfg.Log.Printf("Successfully migrated top-level '%s' to 'on.%s'", fieldKey, fieldKey) + return result, true +} + +func newMoveTopLevelKeyToOnBlockCodemodFindLocations( + frontmatterLines []string, + cfg moveToOnBlockConfig, + fieldKey string, + fieldKeyPrefix string, +) newMoveTopLevelKeyToOnBlockCodemodLocations { + loc := newMoveTopLevelKeyToOnBlockCodemodLocations{fieldLineIdx: -1, onBlockIdx: -1} + for i, line := range frontmatterLines { + trimmedLine := strings.TrimSpace(line) + + if isTopLevelKey(line) && strings.HasPrefix(trimmedLine, fieldKeyPrefix) { + loc.fieldLineIdx = i + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + loc.fieldLineValue = strings.TrimSpace(parts[1]) } + cfg.Log.Printf("Found top-level %s at line %d", fieldKey, i+1) + } + + if isTopLevelKey(line) && strings.HasPrefix(trimmedLine, "on:") { + loc.onBlockIdx = i + loc.onIndent = getIndentation(line) + cfg.Log.Printf("Found 'on:' block at line %d", i+1) + } + } + return loc +} - return applyFrontmatterLineTransform(content, func(frontmatterLines []string) ([]string, bool) { - var fieldLineIdx = -1 - var fieldLineValue string - var onBlockIdx = -1 - var onIndent string - - for i, line := range frontmatterLines { - trimmedLine := strings.TrimSpace(line) - - if isTopLevelKey(line) && strings.HasPrefix(trimmedLine, fieldKeyPrefix) { - fieldLineIdx = i - parts := strings.SplitN(line, ":", 2) - if len(parts) == 2 { - fieldLineValue = strings.TrimSpace(parts[1]) - } - cfg.Log.Printf("Found top-level %s at line %d", fieldKey, i+1) - } - - if isTopLevelKey(line) && strings.HasPrefix(trimmedLine, "on:") { - onBlockIdx = i - onIndent = getIndentation(line) - cfg.Log.Printf("Found 'on:' block at line %d", i+1) - } - } +func newMoveTopLevelKeyToOnBlockCodemodCollectFieldLines( + frontmatterLines []string, + loc newMoveTopLevelKeyToOnBlockCodemodLocations, + cfg moveToOnBlockConfig, +) ([]string, int) { + if cfg.IsInlineSingle != nil && cfg.IsInlineSingle(loc.fieldLineValue) { + return []string{frontmatterLines[loc.fieldLineIdx]}, loc.fieldLineIdx + } - if fieldLineIdx == -1 { - return frontmatterLines, false - } + fieldStartIndent := getIndentation(frontmatterLines[loc.fieldLineIdx]) + fieldLines := []string{frontmatterLines[loc.fieldLineIdx]} + fieldEndIdx := loc.fieldLineIdx + for j := loc.fieldLineIdx + 1; j < len(frontmatterLines); j++ { + line := frontmatterLines[j] + trimmed := strings.TrimSpace(line) + + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + fieldLines = append(fieldLines, line) + fieldEndIdx = j + continue + } + + if isNestedUnder(line, fieldStartIndent) { + fieldLines = append(fieldLines, line) + fieldEndIdx = j + } else { + break + } + } + return fieldLines, fieldEndIdx +} - // Collect all lines that belong to the field block. - var fieldLines []string - var fieldEndIdx int - - if cfg.IsInlineSingle != nil && cfg.IsInlineSingle(fieldLineValue) { - // Single-line format (e.g. "bots: [a, b]" or "roles: all") - fieldLines = []string{frontmatterLines[fieldLineIdx]} - fieldEndIdx = fieldLineIdx - } else { - // Multi-line or empty-value format — collect nested lines. - fieldStartIndent := getIndentation(frontmatterLines[fieldLineIdx]) - fieldLines = append(fieldLines, frontmatterLines[fieldLineIdx]) - fieldEndIdx = fieldLineIdx - - for j := fieldLineIdx + 1; j < len(frontmatterLines); j++ { - line := frontmatterLines[j] - trimmed := strings.TrimSpace(line) - - if trimmed == "" || strings.HasPrefix(trimmed, "#") { - fieldLines = append(fieldLines, line) - fieldEndIdx = j - continue - } - - if isNestedUnder(line, fieldStartIndent) { - fieldLines = append(fieldLines, line) - fieldEndIdx = j - } else { - break - } - } - } +func newMoveTopLevelKeyToOnBlockCodemodCreateOnBlock( + frontmatterLines []string, + fieldLineIdx int, + fieldEndIdx int, + fieldLines []string, + fieldKeyPrefix string, +) []string { + result := make([]string, 0, len(frontmatterLines)) + for i, line := range frontmatterLines { + if i >= fieldLineIdx && i <= fieldEndIdx { + if i == fieldLineIdx { + result = append(result, "on:") + result = newMoveTopLevelKeyToOnBlockCodemodAppendCreatedField(result, fieldLines, fieldKeyPrefix) + } + continue + } + result = append(result, line) + } + return result +} - cfg.Log.Printf("%s spans lines %d to %d (%d lines)", fieldKey, fieldLineIdx+1, fieldEndIdx+1, len(fieldLines)) - - result := make([]string, 0, len(frontmatterLines)) - - if onBlockIdx == -1 { - // No 'on:' block — create one and place the field inside it. - cfg.Log.Printf("No 'on:' block found - creating new one with %s", fieldKey) - - for i, line := range frontmatterLines { - if i >= fieldLineIdx && i <= fieldEndIdx { - if i == fieldLineIdx { - result = append(result, "on:") - for _, fl := range fieldLines { - trimmed := strings.TrimSpace(fl) - if trimmed == "" { - result = append(result, fl) - } else if strings.HasPrefix(trimmed, fieldKeyPrefix) { - result = append(result, " "+fl) - } else { - result = append(result, " "+trimmed) - } - } - } - continue - } - result = append(result, line) - } - } else { - // 'on:' block exists — insert the field right after the 'on:' line. - cfg.Log.Printf("Found 'on:' block - adding %s to it", fieldKey) - - onItemIndent := onIndent + " " - insertedField := false - - for i, line := range frontmatterLines { - if i >= fieldLineIdx && i <= fieldEndIdx { - continue - } - - result = append(result, line) - - if i == onBlockIdx && !insertedField { - for _, fl := range fieldLines { - trimmed := strings.TrimSpace(fl) - if trimmed == "" { - result = append(result, fl) - } else if strings.HasPrefix(trimmed, fieldKeyPrefix) { - parts := strings.SplitN(trimmed, ":", 2) - if len(parts) == 2 { - result = append(result, fmt.Sprintf("%s%s:%s", onItemIndent, fieldKey, parts[1])) - } else { - result = append(result, onItemIndent+fieldKey+":") - } - } else { - result = append(result, onItemIndent+" "+trimmed) - } - } - insertedField = true - } - } - } +func newMoveTopLevelKeyToOnBlockCodemodAppendCreatedField(result []string, fieldLines []string, fieldKeyPrefix string) []string { + for _, fl := range fieldLines { + trimmed := strings.TrimSpace(fl) + if trimmed == "" { + result = append(result, fl) + } else if strings.HasPrefix(trimmed, fieldKeyPrefix) { + result = append(result, " "+fl) + } else { + result = append(result, " "+trimmed) + } + } + return result +} - cfg.Log.Printf("Successfully migrated top-level '%s' to 'on.%s'", fieldKey, fieldKey) - return result, true - }) - }, +func newMoveTopLevelKeyToOnBlockCodemodInsertIntoOnBlock( + frontmatterLines []string, + loc newMoveTopLevelKeyToOnBlockCodemodLocations, + fieldEndIdx int, + fieldLines []string, + fieldKey string, + fieldKeyPrefix string, +) []string { + result := make([]string, 0, len(frontmatterLines)) + onItemIndent := loc.onIndent + " " + insertedField := false + for i, line := range frontmatterLines { + if i >= loc.fieldLineIdx && i <= fieldEndIdx { + continue + } + + result = append(result, line) + if i == loc.onBlockIdx && !insertedField { + result = newMoveTopLevelKeyToOnBlockCodemodAppendExistingField(result, fieldLines, fieldKey, fieldKeyPrefix, onItemIndent) + insertedField = true + } + } + return result +} + +func newMoveTopLevelKeyToOnBlockCodemodAppendExistingField(result []string, fieldLines []string, fieldKey string, fieldKeyPrefix string, onItemIndent string) []string { + for _, fl := range fieldLines { + trimmed := strings.TrimSpace(fl) + if trimmed == "" { + result = append(result, fl) + } else if strings.HasPrefix(trimmed, fieldKeyPrefix) { + parts := strings.SplitN(trimmed, ":", 2) + if len(parts) == 2 { + result = append(result, fmt.Sprintf("%s%s:%s", onItemIndent, fieldKey, parts[1])) + } else { + result = append(result, onItemIndent+fieldKey+":") + } + } else { + result = append(result, onItemIndent+" "+trimmed) + } } + return result } diff --git a/pkg/cli/codemod_github_app.go b/pkg/cli/codemod_github_app.go index 6057e70511f..b6f7fc472bd 100644 --- a/pkg/cli/codemod_github_app.go +++ b/pkg/cli/codemod_github_app.go @@ -92,9 +92,7 @@ func renameAppToGitHubApp(lines []string) ([]string, bool) { var result []string modified := false - // Block tracking - var inTools, inToolsGithub, inSafeOutputs, inCheckout bool - var toolsIndent, toolsGithubIndent, safeOutputsIndent, checkoutIndent string + state := renameAppToGitHubAppState{} for i, line := range lines { trimmed := strings.TrimSpace(line) @@ -106,96 +104,109 @@ func renameAppToGitHubApp(lines []string) ([]string, bool) { } // Exit blocks when indentation signals we've left them - if !strings.HasPrefix(trimmed, "#") { - if inToolsGithub && hasExitedBlock(line, toolsGithubIndent) { - inToolsGithub = false - } - if inTools && hasExitedBlock(line, toolsIndent) { - inTools = false - inToolsGithub = false - } - if inSafeOutputs && hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputs = false - } - if inCheckout && hasExitedBlock(line, checkoutIndent) { - inCheckout = false - } - } + renameAppToGitHubAppExitBlocks(line, trimmed, &state) // Detect block entries at any indentation level - if strings.HasPrefix(trimmed, "tools:") { - inTools = true - inToolsGithub = false - toolsIndent = getIndentation(line) - result = append(result, line) - continue - } - - if inTools && strings.HasPrefix(trimmed, "github:") { - inToolsGithub = true - toolsGithubIndent = getIndentation(line) + if renameAppToGitHubAppEnterBlock(line, trimmed, &state) { result = append(result, line) continue } - if strings.HasPrefix(trimmed, "safe-outputs:") { - inSafeOutputs = true - safeOutputsIndent = getIndentation(line) - result = append(result, line) - continue - } - - if strings.HasPrefix(trimmed, "checkout:") { - inCheckout = true - checkoutIndent = getIndentation(line) - result = append(result, line) + if newLine, replaced := renameAppToGitHubAppLine(line, trimmed, &state, i); replaced { + result = append(result, newLine) + modified = true continue } - // Rename a top-level 'app:' key. - if strings.HasPrefix(trimmed, "app:") && isTopLevelKey(line) { - newLine, replaced := findAndReplaceInLine(line, "app", "github-app") - if replaced { - result = append(result, newLine) - modified = true - githubAppCodemodLog.Printf("Renamed top-level 'app' to 'github-app' on line %d", i+1) - continue - } - } + result = append(result, line) + } - // Rename nested 'app:' keys when inside a target block - if strings.HasPrefix(trimmed, "app:") { - lineIndent := getIndentation(line) - shouldRename := false + return result, modified +} - // Child of tools.github (inside github: block) - if inToolsGithub && isDescendant(lineIndent, toolsGithubIndent) { - shouldRename = true - } +type renameAppToGitHubAppState struct { + inTools bool + inToolsGithub bool + inSafeOutputs bool + inCheckout bool + toolsIndent string + toolsGithubIndent string + safeOutputsIndent string + checkoutIndent string +} - // Child of safe-outputs - if inSafeOutputs && isDescendant(lineIndent, safeOutputsIndent) { - shouldRename = true - } +func renameAppToGitHubAppExitBlocks(line string, trimmed string, state *renameAppToGitHubAppState) { + if strings.HasPrefix(trimmed, "#") { + return + } + if state.inToolsGithub && hasExitedBlock(line, state.toolsGithubIndent) { + state.inToolsGithub = false + } + if state.inTools && hasExitedBlock(line, state.toolsIndent) { + state.inTools = false + state.inToolsGithub = false + } + if state.inSafeOutputs && hasExitedBlock(line, state.safeOutputsIndent) { + state.inSafeOutputs = false + } + if state.inCheckout && hasExitedBlock(line, state.checkoutIndent) { + state.inCheckout = false + } +} - // Child of checkout (or a list item inside checkout) - if inCheckout && isDescendant(lineIndent, checkoutIndent) { - shouldRename = true - } +func renameAppToGitHubAppEnterBlock(line string, trimmed string, state *renameAppToGitHubAppState) bool { + if strings.HasPrefix(trimmed, "tools:") { + state.inTools = true + state.inToolsGithub = false + state.toolsIndent = getIndentation(line) + return true + } + if state.inTools && strings.HasPrefix(trimmed, "github:") { + state.inToolsGithub = true + state.toolsGithubIndent = getIndentation(line) + return true + } + if strings.HasPrefix(trimmed, "safe-outputs:") { + state.inSafeOutputs = true + state.safeOutputsIndent = getIndentation(line) + return true + } + if strings.HasPrefix(trimmed, "checkout:") { + state.inCheckout = true + state.checkoutIndent = getIndentation(line) + return true + } + return false +} - if shouldRename { - newLine, replaced := findAndReplaceInLine(line, "app", "github-app") - if replaced { - result = append(result, newLine) - modified = true - githubAppCodemodLog.Printf("Renamed 'app' to 'github-app' on line %d", i+1) - continue - } - } - } +func renameAppToGitHubAppLine(line string, trimmed string, state *renameAppToGitHubAppState, index int) (string, bool) { + if !strings.HasPrefix(trimmed, "app:") { + return line, false + } + if isTopLevelKey(line) { + return renameAppToGitHubAppReplace(line, "top-level 'app' to 'github-app'", index) + } + if renameAppToGitHubAppShouldRenameNested(line, state) { + return renameAppToGitHubAppReplace(line, "'app' to 'github-app'", index) + } + return line, false +} - result = append(result, line) +func renameAppToGitHubAppShouldRenameNested(line string, state *renameAppToGitHubAppState) bool { + lineIndent := getIndentation(line) + if state.inToolsGithub && isDescendant(lineIndent, state.toolsGithubIndent) { + return true + } + if state.inSafeOutputs && isDescendant(lineIndent, state.safeOutputsIndent) { + return true } + return state.inCheckout && isDescendant(lineIndent, state.checkoutIndent) +} - return result, modified +func renameAppToGitHubAppReplace(line string, label string, index int) (string, bool) { + newLine, replaced := findAndReplaceInLine(line, "app", "github-app") + if replaced { + githubAppCodemodLog.Printf("Renamed %s on line %d", label, index+1) + } + return newLine, replaced } diff --git a/pkg/cli/codemod_github_repos.go b/pkg/cli/codemod_github_repos.go index 6018ad20bae..148afe4b074 100644 --- a/pkg/cli/codemod_github_repos.go +++ b/pkg/cli/codemod_github_repos.go @@ -62,8 +62,7 @@ func renameGitHubReposToAllowedRepos(lines []string) ([]string, bool) { var result []string modified := false - var inTools, inToolsGithub bool - var toolsIndent, toolsGithubIndent string + state := renameGitHubReposToAllowedReposState{} for i, line := range lines { trimmed := strings.TrimSpace(line) @@ -75,45 +74,30 @@ func renameGitHubReposToAllowedRepos(lines []string) ([]string, bool) { } // Exit blocks when indentation signals we've left them - if !strings.HasPrefix(trimmed, "#") { - if inToolsGithub && hasExitedBlock(line, toolsGithubIndent) { - inToolsGithub = false - } - if inTools && hasExitedBlock(line, toolsIndent) { - inTools = false - inToolsGithub = false - } - } + renameGitHubReposToAllowedReposExitBlocks(line, trimmed, &state) // Detect 'tools:' block if strings.HasPrefix(trimmed, "tools:") { - inTools = true - inToolsGithub = false - toolsIndent = getIndentation(line) + state.inTools = true + state.inToolsGithub = false + state.toolsIndent = getIndentation(line) result = append(result, line) continue } // Detect 'github:' block inside 'tools:' - if inTools && strings.HasPrefix(trimmed, "github:") { - inToolsGithub = true - toolsGithubIndent = getIndentation(line) + if state.inTools && strings.HasPrefix(trimmed, "github:") { + state.inToolsGithub = true + state.toolsGithubIndent = getIndentation(line) result = append(result, line) continue } // Rename 'repos:' to 'allowed-repos:' when inside tools.github - if inToolsGithub && strings.HasPrefix(trimmed, "repos:") { - lineIndent := getIndentation(line) - if isDescendant(lineIndent, toolsGithubIndent) { - newLine, replaced := findAndReplaceInLine(line, "repos", "allowed-repos") - if replaced { - result = append(result, newLine) - modified = true - githubReposCodemodLog.Printf("Renamed 'repos' to 'allowed-repos' on line %d", i+1) - continue - } - } + if newLine, replaced := renameGitHubReposToAllowedReposLine(line, trimmed, &state, i); replaced { + result = append(result, newLine) + modified = true + continue } result = append(result, line) @@ -121,3 +105,38 @@ func renameGitHubReposToAllowedRepos(lines []string) ([]string, bool) { return result, modified } + +type renameGitHubReposToAllowedReposState struct { + inTools bool + inToolsGithub bool + toolsIndent string + toolsGithubIndent string +} + +func renameGitHubReposToAllowedReposExitBlocks(line string, trimmed string, state *renameGitHubReposToAllowedReposState) { + if strings.HasPrefix(trimmed, "#") { + return + } + if state.inToolsGithub && hasExitedBlock(line, state.toolsGithubIndent) { + state.inToolsGithub = false + } + if state.inTools && hasExitedBlock(line, state.toolsIndent) { + state.inTools = false + state.inToolsGithub = false + } +} + +func renameGitHubReposToAllowedReposLine(line string, trimmed string, state *renameGitHubReposToAllowedReposState, index int) (string, bool) { + if !state.inToolsGithub || !strings.HasPrefix(trimmed, "repos:") { + return line, false + } + lineIndent := getIndentation(line) + if !isDescendant(lineIndent, state.toolsGithubIndent) { + return line, false + } + newLine, replaced := findAndReplaceInLine(line, "repos", "allowed-repos") + if replaced { + githubReposCodemodLog.Printf("Renamed 'repos' to 'allowed-repos' on line %d", index+1) + } + return newLine, replaced +} diff --git a/pkg/cli/codemod_mcp_network.go b/pkg/cli/codemod_mcp_network.go index ba1423f7a83..9ceb484db8e 100644 --- a/pkg/cli/codemod_mcp_network.go +++ b/pkg/cli/codemod_mcp_network.go @@ -18,225 +18,232 @@ func getMCPNetworkMigrationCodemod() Codemod { Name: "Migrate MCP network config to top-level", Description: "Moves per-server MCP 'network.allowed' configuration to top-level workflow 'network.allowed'. Per-server network configuration is deprecated.", IntroducedIn: "0.6.0", - Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Check if mcp-servers section exists - mcpServersValue, hasMCPServers := frontmatter["mcp-servers"] - if !hasMCPServers { - return content, false, nil - } - - mcpServersMap, ok := mcpServersValue.(map[string]any) - if !ok { - return content, false, nil - } + Apply: getMCPNetworkMigrationCodemodApply, + } +} - // Collect all network.allowed domains from MCP servers - var allAllowedDomains []string - serversWithNetwork := make(map[string]struct { - }) +func getMCPNetworkMigrationCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + mcpServersMap, ok := getMCPNetworkMigrationCodemodServers(frontmatter) + if !ok { + return content, false, nil + } - for serverName, serverValue := range mcpServersMap { - serverConfig, ok := serverValue.(map[string]any) - if !ok { - continue - } + allAllowedDomains, serversWithNetwork := getMCPNetworkMigrationCodemodServerDomains(mcpServersMap) + if len(serversWithNetwork) == 0 { + return content, false, nil + } + allAllowedDomains = sliceutil.Deduplicate(allAllowedDomains) - // Check if this server has a network configuration - networkValue, hasNetwork := serverConfig["network"] - if !hasNetwork { - continue - } + existingAllowed, hasTopLevelNetwork := getMCPNetworkMigrationCodemodExistingAllowed(frontmatter) + mergedDomains := sliceutil.Deduplicate(append(existingAllowed, allAllowedDomains...)) - networkMap, ok := networkValue.(map[string]any) - if !ok { - continue - } + return applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { + return getMCPNetworkMigrationCodemodTransform(lines, serversWithNetwork, hasTopLevelNetwork, mergedDomains) + }) +} - // Extract allowed domains - allowedValue, hasAllowed := networkMap["allowed"] - if !hasAllowed { - continue - } +func getMCPNetworkMigrationCodemodServers(frontmatter map[string]any) (map[string]any, bool) { + mcpServersValue, hasMCPServers := frontmatter["mcp-servers"] + if !hasMCPServers { + return nil, false + } + mcpServersMap, ok := mcpServersValue.(map[string]any) + return mcpServersMap, ok +} - // Convert allowed to []string - switch allowed := allowedValue.(type) { - case []any: - for _, domain := range allowed { - if domainStr, ok := domain.(string); ok { - allAllowedDomains = append(allAllowedDomains, domainStr) - } - } - // Only mark server as having network if it has domains - if len(allowed) > 0 { - serversWithNetwork[serverName] = struct { - }{} - } - case []string: - allAllowedDomains = append(allAllowedDomains, allowed...) - // Only mark server as having network if it has domains - if len(allowed) > 0 { - serversWithNetwork[serverName] = struct { - }{} - } - } - } +func getMCPNetworkMigrationCodemodServerDomains(mcpServersMap map[string]any) ([]string, map[string]struct{}) { + var allAllowedDomains []string + serversWithNetwork := make(map[string]struct{}) + for serverName, serverValue := range mcpServersMap { + serverConfig, ok := serverValue.(map[string]any) + if !ok { + continue + } + networkMap, ok := getMCPNetworkMigrationCodemodNetworkMap(serverConfig) + if !ok { + continue + } + allowedValue, hasAllowed := networkMap["allowed"] + if !hasAllowed { + continue + } + domains, hasDomains := getMCPNetworkMigrationCodemodAllowedStrings(allowedValue) + allAllowedDomains = append(allAllowedDomains, domains...) + if hasDomains { + serversWithNetwork[serverName] = struct{}{} + } + } + return allAllowedDomains, serversWithNetwork +} - // If no servers have network configuration, nothing to do - if len(serversWithNetwork) == 0 { - return content, false, nil - } +func getMCPNetworkMigrationCodemodNetworkMap(serverConfig map[string]any) (map[string]any, bool) { + networkValue, hasNetwork := serverConfig["network"] + if !hasNetwork { + return nil, false + } + networkMap, ok := networkValue.(map[string]any) + return networkMap, ok +} - // Remove duplicates from collected domains - allAllowedDomains = sliceutil.Deduplicate(allAllowedDomains) - - // Check if top-level network configuration already exists - existingNetworkValue, hasTopLevelNetwork := frontmatter["network"] - var existingAllowed []string - - if hasTopLevelNetwork { - if existingNetworkMap, ok := existingNetworkValue.(map[string]any); ok { - if existingAllowedValue, hasExistingAllowed := existingNetworkMap["allowed"]; hasExistingAllowed { - switch allowed := existingAllowedValue.(type) { - case []any: - for _, domain := range allowed { - if domainStr, ok := domain.(string); ok { - existingAllowed = append(existingAllowed, domainStr) - } - } - case []string: - existingAllowed = append(existingAllowed, allowed...) - } - } - } +func getMCPNetworkMigrationCodemodAllowedStrings(allowedValue any) ([]string, bool) { + var domains []string + switch allowed := allowedValue.(type) { + case []any: + for _, domain := range allowed { + if domainStr, ok := domain.(string); ok { + domains = append(domains, domainStr) } + } + return domains, len(allowed) > 0 + case []string: + domains = append(domains, allowed...) + return domains, len(allowed) > 0 + } + return nil, false +} - // Merge existing and new domains, remove duplicates - mergedDomains := sliceutil.Deduplicate(append(existingAllowed, allAllowedDomains...)) - - return applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - // Remove network fields from all MCP servers - result := lines - var modified bool - for serverName := range serversWithNetwork { - var serverModified bool - result, serverModified = removeFieldFromMCPServer(result, serverName, "network") - if serverModified { - modified = true - mcpNetworkCodemodLog.Printf("Removed network configuration from MCP server '%s'", serverName) - } - } - - if !modified { - return lines, false - } - - // Add or update top-level network configuration - if hasTopLevelNetwork { - // Update existing network.allowed - result = updateNetworkAllowed(result, mergedDomains) - mcpNetworkCodemodLog.Printf("Updated top-level network.allowed with %d domains", len(mergedDomains)) - } else { - // Add new top-level network configuration - result = addTopLevelNetwork(result, mergedDomains) - mcpNetworkCodemodLog.Printf("Added top-level network.allowed with %d domains", len(mergedDomains)) - } +func getMCPNetworkMigrationCodemodExistingAllowed(frontmatter map[string]any) ([]string, bool) { + existingNetworkValue, hasTopLevelNetwork := frontmatter["network"] + if !hasTopLevelNetwork { + return nil, false + } + existingNetworkMap, ok := existingNetworkValue.(map[string]any) + if !ok { + return nil, true + } + existingAllowedValue, hasExistingAllowed := existingNetworkMap["allowed"] + if !hasExistingAllowed { + return nil, true + } + existingAllowed, _ := getMCPNetworkMigrationCodemodAllowedStrings(existingAllowedValue) + return existingAllowed, true +} - mcpNetworkCodemodLog.Print("Applied MCP network migration to top-level") - return result, true - }) - }, +func getMCPNetworkMigrationCodemodTransform( + lines []string, + serversWithNetwork map[string]struct{}, + hasTopLevelNetwork bool, + mergedDomains []string, +) ([]string, bool) { + result := lines + var modified bool + for serverName := range serversWithNetwork { + var serverModified bool + result, serverModified = removeFieldFromMCPServer(result, serverName, "network") + if serverModified { + modified = true + mcpNetworkCodemodLog.Printf("Removed network configuration from MCP server '%s'", serverName) + } + } + if !modified { + return lines, false } + if hasTopLevelNetwork { + result = updateNetworkAllowed(result, mergedDomains) + mcpNetworkCodemodLog.Printf("Updated top-level network.allowed with %d domains", len(mergedDomains)) + } else { + result = addTopLevelNetwork(result, mergedDomains) + mcpNetworkCodemodLog.Printf("Added top-level network.allowed with %d domains", len(mergedDomains)) + } + mcpNetworkCodemodLog.Print("Applied MCP network migration to top-level") + return result, true } // removeFieldFromMCPServer removes a field from a specific MCP server configuration func removeFieldFromMCPServer(lines []string, serverName string, fieldName string) ([]string, bool) { var result []string var modified bool - var inMCPServers bool - var mcpServersIndent string - var inServerBlock bool - var serverIndent string - var inFieldBlock bool - var fieldIndent string + state := removeFieldFromMCPServerState{} for i, line := range lines { - trimmedLine := strings.TrimSpace(line) - - // Track if we're in mcp-servers block - if strings.HasPrefix(trimmedLine, "mcp-servers:") { - inMCPServers = true - mcpServersIndent = getIndentation(line) + keepLine, lineModified := removeFieldFromMCPServerKeepLine(line, i, serverName, fieldName, &state) + if lineModified { + modified = true + } + if keepLine { result = append(result, line) - continue } + } - // Check if we've left mcp-servers block - if inMCPServers && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, mcpServersIndent) { - inMCPServers = false - inServerBlock = false - } - } + return result, modified +} - // Track if we're in the specific server block - if inMCPServers && strings.HasPrefix(trimmedLine, serverName+":") { - inServerBlock = true - serverIndent = getIndentation(line) - result = append(result, line) - continue - } +type removeFieldFromMCPServerState struct { + inMCPServers bool + mcpServersIndent string + inServerBlock bool + serverIndent string + inFieldBlock bool + fieldIndent string +} - // Check if we've left the server block - if inServerBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - currentIndent := getIndentation(line) - // Exit if we're back at mcp-servers level or less - if len(currentIndent) <= len(serverIndent) && strings.Contains(line, ":") { - inServerBlock = false - } - } +func removeFieldFromMCPServerKeepLine(line string, index int, serverName string, fieldName string, state *removeFieldFromMCPServerState) (bool, bool) { + trimmedLine := strings.TrimSpace(line) - // Remove field line if in server block - if inServerBlock && strings.HasPrefix(trimmedLine, fieldName+":") { - modified = true - inFieldBlock = true - fieldIndent = getIndentation(line) - mcpNetworkCodemodLog.Printf("Removed %s from mcp-server '%s' on line %d", fieldName, serverName, i+1) - continue - } + // Track if we're in mcp-servers block + if strings.HasPrefix(trimmedLine, "mcp-servers:") { + state.inMCPServers = true + state.mcpServersIndent = getIndentation(line) + return true, false + } - // Skip nested properties under the field - if inFieldBlock { - // Empty lines within the field block should be removed - if trimmedLine == "" { - continue - } + removeFieldFromMCPServerExitBlocks(line, trimmedLine, state) - currentIndent := getIndentation(line) + // Track if we're in the specific server block + if state.inMCPServers && strings.HasPrefix(trimmedLine, serverName+":") { + state.inServerBlock = true + state.serverIndent = getIndentation(line) + return true, false + } - // Comments need to check indentation - if strings.HasPrefix(trimmedLine, "#") { - if len(currentIndent) > len(fieldIndent) { - // Comment is nested under field, remove it - continue - } - // Comment is at same or less indentation, exit field block and keep it - inFieldBlock = false - result = append(result, line) - continue - } + // Remove field line if in server block + if state.inServerBlock && strings.HasPrefix(trimmedLine, fieldName+":") { + state.inFieldBlock = true + state.fieldIndent = getIndentation(line) + mcpNetworkCodemodLog.Printf("Removed %s from mcp-server '%s' on line %d", fieldName, serverName, index+1) + return false, true + } - // If this line has more indentation than field, it's a nested property - if len(currentIndent) > len(fieldIndent) { - continue - } - // We've exited the field block - inFieldBlock = false + if state.inFieldBlock { + return removeFieldFromMCPServerKeepAfterRemovedField(line, trimmedLine, state), false + } + return true, false +} + +func removeFieldFromMCPServerExitBlocks(line string, trimmedLine string, state *removeFieldFromMCPServerState) { + if state.inMCPServers && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + if hasExitedBlock(line, state.mcpServersIndent) { + state.inMCPServers = false + state.inServerBlock = false + } + } + if state.inServerBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + currentIndent := getIndentation(line) + if len(currentIndent) <= len(state.serverIndent) && strings.Contains(line, ":") { + state.inServerBlock = false } + } +} - result = append(result, line) +func removeFieldFromMCPServerKeepAfterRemovedField(line string, trimmedLine string, state *removeFieldFromMCPServerState) bool { + if trimmedLine == "" { + return false } - return result, modified + currentIndent := getIndentation(line) + if strings.HasPrefix(trimmedLine, "#") { + if len(currentIndent) > len(state.fieldIndent) { + return false + } + state.inFieldBlock = false + return true + } + + if len(currentIndent) > len(state.fieldIndent) { + return false + } + state.inFieldBlock = false + return true } // addTopLevelNetwork adds a new top-level network configuration @@ -328,20 +335,7 @@ func updateNetworkAllowed(lines []string, domains []string) []string { // Skip existing allowed array items if inAllowedBlock { - currentIndent := getIndentation(line) - - // Empty lines - skip - if trimmedLine == "" { - continue - } - - // Comments at deeper indentation - skip - if strings.HasPrefix(trimmedLine, "#") && len(currentIndent) > len(allowedIndent) { - continue - } - - // Array items (lines starting with -) - if strings.HasPrefix(trimmedLine, "-") && len(currentIndent) > len(allowedIndent) { + if updateNetworkAllowedShouldSkipAllowedLine(line, trimmedLine, allowedIndent) { continue } @@ -354,13 +348,23 @@ func updateNetworkAllowed(lines []string, domains []string) []string { // If we didn't find an allowed block, add it to the network block if !replacedAllowed { - // Find the end of the network block and insert allowed result = addAllowedToNetwork(result, domains) } return result } +func updateNetworkAllowedShouldSkipAllowedLine(line string, trimmedLine string, allowedIndent string) bool { + currentIndent := getIndentation(line) + if trimmedLine == "" { + return true + } + if strings.HasPrefix(trimmedLine, "#") && len(currentIndent) > len(allowedIndent) { + return true + } + return strings.HasPrefix(trimmedLine, "-") && len(currentIndent) > len(allowedIndent) +} + // addAllowedToNetwork adds an allowed field to an existing network block func addAllowedToNetwork(lines []string, domains []string) []string { var result []string diff --git a/pkg/cli/codemod_mentions_allow_team_members.go b/pkg/cli/codemod_mentions_allow_team_members.go index 3de95a3ac0c..a2b344986cb 100644 --- a/pkg/cli/codemod_mentions_allow_team_members.go +++ b/pkg/cli/codemod_mentions_allow_team_members.go @@ -57,91 +57,39 @@ func renameMentionsAllowTeamMembers(lines []string) ([]string, bool) { result := make([]string, 0, len(lines)) modified := false - inSafeOutputs := false - safeOutputsIndent := "" - safeOutputsChildIndent := "" - inMentions := false - mentionsIndent := "" - mentionsChildIndent := "" + state := renameMentionsAllowTeamMembersState{} for i, line := range lines { trimmed := strings.TrimSpace(line) indent := getIndentation(line) - if !strings.HasPrefix(trimmed, "#") { - if inSafeOutputs && hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputs = false - safeOutputsChildIndent = "" - inMentions = false - mentionsIndent = "" - mentionsChildIndent = "" - } - if inMentions && hasExitedBlock(line, mentionsIndent) { - inMentions = false - mentionsIndent = "" - mentionsChildIndent = "" - } - } + renameMentionsAllowTeamMembersExitBlocks(line, trimmed, &state) if strings.HasPrefix(trimmed, "safe-outputs:") { - inSafeOutputs = true - safeOutputsIndent = indent - safeOutputsChildIndent = "" - inMentions = false - mentionsIndent = "" - mentionsChildIndent = "" + state.inSafeOutputs = true + state.safeOutputsIndent = indent + state.safeOutputsChildIndent = "" + renameMentionsAllowTeamMembersResetMentions(&state) result = append(result, line) continue } - if inSafeOutputs && isDescendant(indent, safeOutputsIndent) && !strings.HasPrefix(trimmed, "#") { - if (safeOutputsChildIndent == "" || indent == safeOutputsChildIndent) && strings.HasPrefix(trimmed, "mentions:") { - if safeOutputsChildIndent == "" { - safeOutputsChildIndent = indent - } - if strings.HasSuffix(trimmed, ":") { - inMentions = true - mentionsIndent = indent - mentionsChildIndent = "" - } else { - inMentions = false - mentionsIndent = "" - mentionsChildIndent = "" - if strings.Contains(trimmed, "allow-team-members:") { - newLine := strings.Replace(line, "allow-team-members:", "allowed-collaborators:", 1) - result = append(result, newLine) - modified = true - mentionsAllowTeamMembersCodemodLog.Printf("Renamed allow-team-members to allowed-collaborators in safe-outputs.mentions on line %d", i+1) - continue - } - } - result = append(result, line) - continue - } - if strings.HasSuffix(trimmed, ":") && (safeOutputsChildIndent == "" || indent == safeOutputsChildIndent) { - if safeOutputsChildIndent == "" { - safeOutputsChildIndent = indent - } - inMentions = false - mentionsIndent = "" - mentionsChildIndent = "" - result = append(result, line) - continue + if newLine, changed, handled := renameMentionsAllowTeamMembersSafeOutputsLine(line, trimmed, indent, i, &state); handled { + if changed { + modified = true } + result = append(result, newLine) + continue } - if inMentions && mentionsChildIndent == "" && isDescendant(indent, mentionsIndent) && trimmed != "" && !strings.HasPrefix(trimmed, "#") { - mentionsChildIndent = indent + if state.inMentions && state.mentionsChildIndent == "" && isDescendant(indent, state.mentionsIndent) && trimmed != "" && !strings.HasPrefix(trimmed, "#") { + state.mentionsChildIndent = indent } - if inMentions && indent == mentionsChildIndent && strings.HasPrefix(trimmed, "allow-team-members:") { - newLine, replaced := findAndReplaceInLine(line, "allow-team-members", "allowed-collaborators") - if replaced { - result = append(result, newLine) - modified = true - mentionsAllowTeamMembersCodemodLog.Printf("Renamed allow-team-members to allowed-collaborators in safe-outputs.mentions on line %d", i+1) - continue - } + if newLine, replaced := renameMentionsAllowTeamMembersLine(line, trimmed, indent, i, &state); replaced { + result = append(result, newLine) + modified = true + continue } result = append(result, line) @@ -149,3 +97,97 @@ func renameMentionsAllowTeamMembers(lines []string) ([]string, bool) { return result, modified } + +type renameMentionsAllowTeamMembersState struct { + inSafeOutputs bool + safeOutputsIndent string + safeOutputsChildIndent string + inMentions bool + mentionsIndent string + mentionsChildIndent string +} + +func renameMentionsAllowTeamMembersResetMentions(state *renameMentionsAllowTeamMembersState) { + state.inMentions = false + state.mentionsIndent = "" + state.mentionsChildIndent = "" +} + +func renameMentionsAllowTeamMembersExitBlocks(line string, trimmed string, state *renameMentionsAllowTeamMembersState) { + if strings.HasPrefix(trimmed, "#") { + return + } + if state.inSafeOutputs && hasExitedBlock(line, state.safeOutputsIndent) { + state.inSafeOutputs = false + state.safeOutputsChildIndent = "" + renameMentionsAllowTeamMembersResetMentions(state) + } + if state.inMentions && hasExitedBlock(line, state.mentionsIndent) { + renameMentionsAllowTeamMembersResetMentions(state) + } +} + +func renameMentionsAllowTeamMembersSafeOutputsLine( + line string, + trimmed string, + indent string, + index int, + state *renameMentionsAllowTeamMembersState, +) (string, bool, bool) { + if !state.inSafeOutputs || !isDescendant(indent, state.safeOutputsIndent) || strings.HasPrefix(trimmed, "#") { + return line, false, false + } + if (state.safeOutputsChildIndent == "" || indent == state.safeOutputsChildIndent) && strings.HasPrefix(trimmed, "mentions:") { + return renameMentionsAllowTeamMembersMentionsLine(line, trimmed, indent, index, state) + } + if strings.HasSuffix(trimmed, ":") && (state.safeOutputsChildIndent == "" || indent == state.safeOutputsChildIndent) { + if state.safeOutputsChildIndent == "" { + state.safeOutputsChildIndent = indent + } + renameMentionsAllowTeamMembersResetMentions(state) + return line, false, true + } + return line, false, false +} + +func renameMentionsAllowTeamMembersMentionsLine( + line string, + trimmed string, + indent string, + index int, + state *renameMentionsAllowTeamMembersState, +) (string, bool, bool) { + if state.safeOutputsChildIndent == "" { + state.safeOutputsChildIndent = indent + } + if strings.HasSuffix(trimmed, ":") { + state.inMentions = true + state.mentionsIndent = indent + state.mentionsChildIndent = "" + return line, false, true + } + renameMentionsAllowTeamMembersResetMentions(state) + if strings.Contains(trimmed, "allow-team-members:") { + newLine := strings.Replace(line, "allow-team-members:", "allowed-collaborators:", 1) + mentionsAllowTeamMembersCodemodLog.Printf("Renamed allow-team-members to allowed-collaborators in safe-outputs.mentions on line %d", index+1) + return newLine, true, true + } + return line, false, true +} + +func renameMentionsAllowTeamMembersLine( + line string, + trimmed string, + indent string, + index int, + state *renameMentionsAllowTeamMembersState, +) (string, bool) { + if !state.inMentions || indent != state.mentionsChildIndent || !strings.HasPrefix(trimmed, "allow-team-members:") { + return line, false + } + newLine, replaced := findAndReplaceInLine(line, "allow-team-members", "allowed-collaborators") + if replaced { + mentionsAllowTeamMembersCodemodLog.Printf("Renamed allow-team-members to allowed-collaborators in safe-outputs.mentions on line %d", index+1) + } + return newLine, replaced +} diff --git a/pkg/cli/codemod_messages_effective_tokens_suffix_to_ai_credits_suffix.go b/pkg/cli/codemod_messages_effective_tokens_suffix_to_ai_credits_suffix.go index 43c7999eb58..89c6f208075 100644 --- a/pkg/cli/codemod_messages_effective_tokens_suffix_to_ai_credits_suffix.go +++ b/pkg/cli/codemod_messages_effective_tokens_suffix_to_ai_credits_suffix.go @@ -71,90 +71,44 @@ func migrateMessagesEffectiveTokensSuffixToAICreditsSuffix(lines []string) ([]st result := make([]string, 0, len(lines)) modified := false - inSafeOutputs := false - safeOutputsIndent := "" - safeOutputsChildIndent := "" - inMessages := false - messagesIndent := "" - messagesChildIndent := "" - inBlockScalar := false - blockScalarIndent := "" + state := migrateMessagesEffectiveTokensSuffixToAICreditsSuffixState{} for _, line := range lines { trimmed := strings.TrimSpace(line) indent := getIndentation(line) - if inBlockScalar { - if trimmed == "" || len(indent) > len(blockScalarIndent) { - updated := strings.ReplaceAll(line, effectiveTokensSuffixPlaceholder, aiCreditsSuffixPlaceholder) - if updated != line { - modified = true - } - result = append(result, updated) - continue + if updated, changed, handled := migrateMessagesEffectiveTokensSuffixToAICreditsSuffixBlockScalarLine(line, trimmed, indent, &state); handled { + if changed { + modified = true } - inBlockScalar = false - blockScalarIndent = "" + result = append(result, updated) + continue } - if !strings.HasPrefix(trimmed, "#") { - if inMessages && hasExitedBlock(line, messagesIndent) { - inMessages = false - messagesIndent = "" - messagesChildIndent = "" - } - if inSafeOutputs && hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputs = false - safeOutputsIndent = "" - safeOutputsChildIndent = "" - inMessages = false - messagesIndent = "" - messagesChildIndent = "" - } - } + migrateMessagesEffectiveTokensSuffixToAICreditsSuffixExitBlocks(line, trimmed, &state) if strings.HasPrefix(trimmed, "safe-outputs:") { - inSafeOutputs = true - safeOutputsIndent = indent - safeOutputsChildIndent = "" - inMessages = false - messagesIndent = "" - messagesChildIndent = "" + state.inSafeOutputs = true + state.safeOutputsIndent = indent + state.safeOutputsChildIndent = "" + state.inMessages = false + state.messagesIndent = "" + state.messagesChildIndent = "" result = append(result, line) continue } - if inSafeOutputs && isDescendant(indent, safeOutputsIndent) && strings.HasSuffix(trimmed, ":") && !strings.HasPrefix(trimmed, "#") { - if safeOutputsChildIndent == "" { - safeOutputsChildIndent = indent - } - if indent == safeOutputsChildIndent && trimmed == "messages:" { - inMessages = true - messagesIndent = indent - messagesChildIndent = "" - } + if migrateMessagesEffectiveTokensSuffixToAICreditsSuffixEnterMessages(trimmed, indent, &state) { result = append(result, line) continue } - if inMessages && isDescendant(indent, messagesIndent) && trimmed != "" && !strings.HasPrefix(trimmed, "#") { - if messagesChildIndent == "" { - messagesChildIndent = indent - } - if indent == messagesChildIndent && strings.Contains(trimmed, ":") { - updated := strings.ReplaceAll(line, effectiveTokensSuffixPlaceholder, aiCreditsSuffixPlaceholder) - if updated != line { - modified = true - } - result = append(result, updated) - - parts := strings.SplitN(updated, ":", 2) - if len(parts) == 2 && isBlockScalarIndicator(parts[1]) { - inBlockScalar = true - blockScalarIndent = indent - } - continue + if updated, changed, handled := migrateMessagesEffectiveTokensSuffixToAICreditsSuffixMessageLine(line, trimmed, indent, &state); handled { + if changed { + modified = true } + result = append(result, updated) + continue } result = append(result, line) @@ -163,6 +117,101 @@ func migrateMessagesEffectiveTokensSuffixToAICreditsSuffix(lines []string) ([]st return result, modified } +type migrateMessagesEffectiveTokensSuffixToAICreditsSuffixState struct { + inSafeOutputs bool + safeOutputsIndent string + safeOutputsChildIndent string + inMessages bool + messagesIndent string + messagesChildIndent string + inBlockScalar bool + blockScalarIndent string +} + +func migrateMessagesEffectiveTokensSuffixToAICreditsSuffixBlockScalarLine( + line string, + trimmed string, + indent string, + state *migrateMessagesEffectiveTokensSuffixToAICreditsSuffixState, +) (string, bool, bool) { + if !state.inBlockScalar { + return line, false, false + } + if trimmed == "" || len(indent) > len(state.blockScalarIndent) { + updated := strings.ReplaceAll(line, effectiveTokensSuffixPlaceholder, aiCreditsSuffixPlaceholder) + return updated, updated != line, true + } + state.inBlockScalar = false + state.blockScalarIndent = "" + return line, false, false +} + +func migrateMessagesEffectiveTokensSuffixToAICreditsSuffixExitBlocks( + line string, + trimmed string, + state *migrateMessagesEffectiveTokensSuffixToAICreditsSuffixState, +) { + if strings.HasPrefix(trimmed, "#") { + return + } + if state.inMessages && hasExitedBlock(line, state.messagesIndent) { + state.inMessages = false + state.messagesIndent = "" + state.messagesChildIndent = "" + } + if state.inSafeOutputs && hasExitedBlock(line, state.safeOutputsIndent) { + state.inSafeOutputs = false + state.safeOutputsIndent = "" + state.safeOutputsChildIndent = "" + state.inMessages = false + state.messagesIndent = "" + state.messagesChildIndent = "" + } +} + +func migrateMessagesEffectiveTokensSuffixToAICreditsSuffixEnterMessages( + trimmed string, + indent string, + state *migrateMessagesEffectiveTokensSuffixToAICreditsSuffixState, +) bool { + if !state.inSafeOutputs || !isDescendant(indent, state.safeOutputsIndent) || !strings.HasSuffix(trimmed, ":") || strings.HasPrefix(trimmed, "#") { + return false + } + if state.safeOutputsChildIndent == "" { + state.safeOutputsChildIndent = indent + } + if indent == state.safeOutputsChildIndent && trimmed == "messages:" { + state.inMessages = true + state.messagesIndent = indent + state.messagesChildIndent = "" + } + return true +} + +func migrateMessagesEffectiveTokensSuffixToAICreditsSuffixMessageLine( + line string, + trimmed string, + indent string, + state *migrateMessagesEffectiveTokensSuffixToAICreditsSuffixState, +) (string, bool, bool) { + if !state.inMessages || !isDescendant(indent, state.messagesIndent) || trimmed == "" || strings.HasPrefix(trimmed, "#") { + return line, false, false + } + if state.messagesChildIndent == "" { + state.messagesChildIndent = indent + } + if indent != state.messagesChildIndent || !strings.Contains(trimmed, ":") { + return line, false, false + } + updated := strings.ReplaceAll(line, effectiveTokensSuffixPlaceholder, aiCreditsSuffixPlaceholder) + parts := strings.SplitN(updated, ":", 2) + if len(parts) == 2 && isBlockScalarIndicator(parts[1]) { + state.inBlockScalar = true + state.blockScalarIndent = indent + } + return updated, updated != line, true +} + func isBlockScalarIndicator(valueSegment string) bool { valueWithoutComment := valueSegment if before, _, found := strings.Cut(valueSegment, "#"); found { diff --git a/pkg/cli/codemod_network_firewall.go b/pkg/cli/codemod_network_firewall.go index d587f03d87a..352440cc88f 100644 --- a/pkg/cli/codemod_network_firewall.go +++ b/pkg/cli/codemod_network_firewall.go @@ -158,36 +158,16 @@ func mergeFirewallIntoExistingSandbox(lines []string, fieldValue any) ([]string, return lines, false } - sandboxIdx := -1 - for i, line := range lines { - trimmed := strings.TrimSpace(line) - if isTopLevelKey(line) && strings.HasPrefix(trimmed, "sandbox:") { - sandboxIdx = i - break - } - } + sandboxIdx := mergeFirewallIntoExistingSandboxFindSandbox(lines) if sandboxIdx == -1 { return lines, false } sandboxIndent := getIndentation(lines[sandboxIdx]) agentIndent := sandboxIndent + " " - sandboxEnd := len(lines) - for i := sandboxIdx + 1; i < len(lines); i++ { - if isTopLevelKey(lines[i]) { - sandboxEnd = i - break - } - } + sandboxEnd := mergeFirewallIntoExistingSandboxFindSandboxEnd(lines, sandboxIdx) - agentStart := -1 - for i := sandboxIdx + 1; i < sandboxEnd; i++ { - trimmed := strings.TrimSpace(lines[i]) - if strings.HasPrefix(trimmed, "agent:") && getIndentation(lines[i]) == agentIndent { - agentStart = i - break - } - } + agentStart := mergeFirewallIntoExistingSandboxFindAgent(lines, sandboxIdx, sandboxEnd, agentIndent) indentedAgentLines := indentLines(agentLines, agentIndent) if agentStart == -1 { @@ -198,6 +178,45 @@ func mergeFirewallIntoExistingSandbox(lines []string, fieldValue any) ([]string, return newLines, true } + agentEnd := mergeFirewallIntoExistingSandboxFindAgentEnd(lines, agentStart, sandboxEnd) + + newLines := make([]string, 0, len(lines)-((agentEnd-agentStart)-len(indentedAgentLines))) + newLines = append(newLines, lines[:agentStart]...) + newLines = append(newLines, indentedAgentLines...) + newLines = append(newLines, lines[agentEnd:]...) + return newLines, true +} + +func mergeFirewallIntoExistingSandboxFindSandbox(lines []string) int { + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if isTopLevelKey(line) && strings.HasPrefix(trimmed, "sandbox:") { + return i + } + } + return -1 +} + +func mergeFirewallIntoExistingSandboxFindSandboxEnd(lines []string, sandboxIdx int) int { + for i := sandboxIdx + 1; i < len(lines); i++ { + if isTopLevelKey(lines[i]) { + return i + } + } + return len(lines) +} + +func mergeFirewallIntoExistingSandboxFindAgent(lines []string, sandboxIdx int, sandboxEnd int, agentIndent string) int { + for i := sandboxIdx + 1; i < sandboxEnd; i++ { + trimmed := strings.TrimSpace(lines[i]) + if strings.HasPrefix(trimmed, "agent:") && getIndentation(lines[i]) == agentIndent { + return i + } + } + return -1 +} + +func mergeFirewallIntoExistingSandboxFindAgentEnd(lines []string, agentStart int, sandboxEnd int) int { agentEnd := agentStart + 1 agentFieldIndent := getIndentation(lines[agentStart]) for agentEnd < sandboxEnd { @@ -219,12 +238,7 @@ func mergeFirewallIntoExistingSandbox(lines []string, fieldValue any) ([]string, } break } - - newLines := make([]string, 0, len(lines)-((agentEnd-agentStart)-len(indentedAgentLines))) - newLines = append(newLines, lines[:agentStart]...) - newLines = append(newLines, indentedAgentLines...) - newLines = append(newLines, lines[agentEnd:]...) - return newLines, true + return agentEnd } func sandboxAgentLinesForExistingSandbox(fieldValue any) []string { diff --git a/pkg/cli/codemod_permissions_read.go b/pkg/cli/codemod_permissions_read.go index fac8a78fb8a..3dc7254ed8c 100644 --- a/pkg/cli/codemod_permissions_read.go +++ b/pkg/cli/codemod_permissions_read.go @@ -34,38 +34,7 @@ func getExpandPermissionsShorthandCodemod() Codemod { return content, false, nil } - newContent, applied, err := applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - var modified bool - result := make([]string, len(lines)) - for i, line := range lines { - trimmedLine := strings.TrimSpace(line) - - // Check for permissions line with shorthand - if strings.HasPrefix(trimmedLine, "permissions:") { - // Handle shorthand on same line: "permissions: read" or "permissions: write" - if strings.Contains(trimmedLine, ": read") && !strings.Contains(trimmedLine, "read-all") && !strings.Contains(trimmedLine, ": read\n") { - // Make sure it's "permissions: read" and not "contents: read" - if strings.TrimSpace(strings.Split(line, ":")[0]) == "permissions" { - result[i] = strings.Replace(line, ": read", ": read-all", 1) - modified = true - permissionsReadCodemodLog.Printf("Replaced 'permissions: read' with 'permissions: read-all' on line %d", i+1) - continue - } - } else if strings.Contains(trimmedLine, ": write") && !strings.Contains(trimmedLine, "write-all") { - // Make sure it's "permissions: write" and not "contents: write" - if strings.TrimSpace(strings.Split(line, ":")[0]) == "permissions" { - result[i] = strings.Replace(line, ": write", ": write-all", 1) - modified = true - permissionsReadCodemodLog.Printf("Replaced 'permissions: write' with 'permissions: write-all' on line %d", i+1) - continue - } - } - } - - result[i] = line - } - return result, modified - }) + newContent, applied, err := applyFrontmatterLineTransform(content, getExpandPermissionsShorthandCodemodLines) if applied { permissionsReadCodemodLog.Print("Applied permissions read/write to read-all/write-all migration") } @@ -73,3 +42,38 @@ func getExpandPermissionsShorthandCodemod() Codemod { }, } } + +func getExpandPermissionsShorthandCodemodLines(lines []string) ([]string, bool) { + var modified bool + result := make([]string, len(lines)) + for i, line := range lines { + updatedLine, changed := getExpandPermissionsShorthandCodemodLine(line, i) + if changed { + modified = true + } + result[i] = updatedLine + } + return result, modified +} + +func getExpandPermissionsShorthandCodemodLine(line string, index int) (string, bool) { + trimmedLine := strings.TrimSpace(line) + if !strings.HasPrefix(trimmedLine, "permissions:") { + return line, false + } + + if strings.Contains(trimmedLine, ": read") && !strings.Contains(trimmedLine, "read-all") && !strings.Contains(trimmedLine, ": read\n") { + // Make sure it's "permissions: read" and not "contents: read" + if strings.TrimSpace(strings.Split(line, ":")[0]) == "permissions" { + permissionsReadCodemodLog.Printf("Replaced 'permissions: read' with 'permissions: read-all' on line %d", index+1) + return strings.Replace(line, ": read", ": read-all", 1), true + } + } else if strings.Contains(trimmedLine, ": write") && !strings.Contains(trimmedLine, "write-all") { + // Make sure it's "permissions: write" and not "contents: write" + if strings.TrimSpace(strings.Split(line, ":")[0]) == "permissions" { + permissionsReadCodemodLog.Printf("Replaced 'permissions: write' with 'permissions: write-all' on line %d", index+1) + return strings.Replace(line, ": write", ": write-all", 1), true + } + } + return line, false +} diff --git a/pkg/cli/codemod_permissions_write.go b/pkg/cli/codemod_permissions_write.go index 3ebd0161ed3..12b42792220 100644 --- a/pkg/cli/codemod_permissions_write.go +++ b/pkg/cli/codemod_permissions_write.go @@ -23,115 +23,119 @@ func getMigrateWritePermissionsToReadCodemod() Codemod { Name: "Convert write permissions to read", Description: "Converts all write permissions to read permissions to comply with the new security policy", IntroducedIn: "0.4.0", - Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Check if permissions exist - permissionsValue, hasPermissions := frontmatter["permissions"] - if !hasPermissions { - return content, false, nil - } + Apply: getMigrateWritePermissionsToReadCodemodApply, + } +} - // Check if any write permissions exist - hasWritePermissions := false +func getMigrateWritePermissionsToReadCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + // Check if permissions exist + permissionsValue, hasPermissions := frontmatter["permissions"] + if !hasPermissions { + return content, false, nil + } + if !getMigrateWritePermissionsToReadCodemodHasWritePermissions(permissionsValue) { + return content, false, nil + } - // Handle string shorthand (write-all, write) - if strValue, ok := permissionsValue.(string); ok { - if strValue == "write-all" || strValue == "write" { - hasWritePermissions = true - } - } + newContent, applied, err := applyFrontmatterLineTransform(content, getMigrateWritePermissionsToReadCodemodLines) + if applied { + writePermissionsCodemodLog.Print("Applied write permissions to read migration") + } + return newContent, applied, err +} - // Handle map format - if mapValue, ok := permissionsValue.(map[string]any); ok { - for key, value := range mapValue { - // Skip write-only permissions (e.g. id-token, copilot-requests) since - // "read" is not a valid value for them — they only accept "write" or "none" - if writeOnlyPermissions[key] { - continue - } - if strValue, ok := value.(string); ok && strValue == "write" { - hasWritePermissions = true - break - } - } - } +func getMigrateWritePermissionsToReadCodemodHasWritePermissions(permissionsValue any) bool { + // Handle string shorthand (write-all, write) + if strValue, ok := permissionsValue.(string); ok { + return strValue == "write-all" || strValue == "write" + } - if !hasWritePermissions { - return content, false, nil + // Handle map format + if mapValue, ok := permissionsValue.(map[string]any); ok { + for key, value := range mapValue { + // Skip write-only permissions (e.g. id-token, copilot-requests) since + // "read" is not a valid value for them — they only accept "write" or "none" + if writeOnlyPermissions[key] { + continue } - - newContent, applied, err := applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - var modified bool - var inPermissionsBlock bool - var permissionsIndent string - result := make([]string, len(lines)) - for i, line := range lines { - trimmedLine := strings.TrimSpace(line) - - // Track if we're in the permissions block - if strings.HasPrefix(trimmedLine, "permissions:") { - inPermissionsBlock = true - permissionsIndent = getIndentation(line) - - // Handle shorthand on same line: "permissions: write-all" or "permissions: write" - if strings.Contains(trimmedLine, ": write-all") { - result[i] = strings.Replace(line, ": write-all", ": read-all", 1) - modified = true - writePermissionsCodemodLog.Printf("Replaced permissions: write-all with permissions: read-all on line %d", i+1) - continue - } else if strings.Contains(trimmedLine, ": write") && !strings.Contains(trimmedLine, "write-all") { - result[i] = strings.Replace(line, ": write", ": read", 1) - modified = true - writePermissionsCodemodLog.Printf("Replaced permissions: write with permissions: read on line %d", i+1) - continue - } - - result[i] = line - continue - } - - // Check if we've left the permissions block - if inPermissionsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, permissionsIndent) { - inPermissionsBlock = false - } - } - - // Replace write with read if in permissions block - if inPermissionsBlock && strings.Contains(trimmedLine, ": write") { - // Preserve indentation and everything else - // Extract the key, value, and any trailing comment - parts := strings.SplitN(line, ":", 2) - if len(parts) >= 2 { - key := parts[0] - permKey := strings.TrimSpace(key) - valueAndComment := parts[1] - - // Skip write-only permissions (e.g. id-token, copilot-requests) since - // "read" is not a valid value for them — they only accept "write" or "none" - if writeOnlyPermissions[permKey] { - result[i] = line - writePermissionsCodemodLog.Printf("Skipping write-only permission %q on line %d", permKey, i+1) - continue - } - - // Replace "write" with "read" in the value part - newValueAndComment := strings.Replace(valueAndComment, " write", " read", 1) - result[i] = fmt.Sprintf("%s:%s", key, newValueAndComment) - modified = true - writePermissionsCodemodLog.Printf("Replaced write with read on line %d", i+1) - } else { - result[i] = line - } - } else { - result[i] = line - } - } - return result, modified - }) - if applied { - writePermissionsCodemodLog.Print("Applied write permissions to read migration") + if strValue, ok := value.(string); ok && strValue == "write" { + return true } - return newContent, applied, err - }, + } + } + return false +} + +func getMigrateWritePermissionsToReadCodemodLines(lines []string) ([]string, bool) { + var modified bool + var inPermissionsBlock bool + var permissionsIndent string + result := make([]string, len(lines)) + for i, line := range lines { + updatedLine, changed := getMigrateWritePermissionsToReadCodemodLine(line, i, &inPermissionsBlock, &permissionsIndent) + if changed { + modified = true + } + result[i] = updatedLine + } + return result, modified +} + +func getMigrateWritePermissionsToReadCodemodLine(line string, index int, inPermissionsBlock *bool, permissionsIndent *string) (string, bool) { + trimmedLine := strings.TrimSpace(line) + + // Track if we're in the permissions block + if strings.HasPrefix(trimmedLine, "permissions:") { + *inPermissionsBlock = true + *permissionsIndent = getIndentation(line) + return getMigrateWritePermissionsToReadCodemodPermissionsLine(line, trimmedLine, index) + } + + // Check if we've left the permissions block + if *inPermissionsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + if hasExitedBlock(line, *permissionsIndent) { + *inPermissionsBlock = false + } + } + + if *inPermissionsBlock && strings.Contains(trimmedLine, ": write") { + return getMigrateWritePermissionsToReadCodemodNestedLine(line, index) } + return line, false +} + +func getMigrateWritePermissionsToReadCodemodPermissionsLine(line string, trimmedLine string, index int) (string, bool) { + // Handle shorthand on same line: "permissions: write-all" or "permissions: write" + if strings.Contains(trimmedLine, ": write-all") { + writePermissionsCodemodLog.Printf("Replaced permissions: write-all with permissions: read-all on line %d", index+1) + return strings.Replace(line, ": write-all", ": read-all", 1), true + } else if strings.Contains(trimmedLine, ": write") && !strings.Contains(trimmedLine, "write-all") { + writePermissionsCodemodLog.Printf("Replaced permissions: write with permissions: read on line %d", index+1) + return strings.Replace(line, ": write", ": read", 1), true + } + return line, false +} + +func getMigrateWritePermissionsToReadCodemodNestedLine(line string, index int) (string, bool) { + // Preserve indentation and everything else + // Extract the key, value, and any trailing comment + parts := strings.SplitN(line, ":", 2) + if len(parts) < 2 { + return line, false + } + key := parts[0] + permKey := strings.TrimSpace(key) + valueAndComment := parts[1] + + // Skip write-only permissions (e.g. id-token, copilot-requests) since + // "read" is not a valid value for them — they only accept "write" or "none" + if writeOnlyPermissions[permKey] { + writePermissionsCodemodLog.Printf("Skipping write-only permission %q on line %d", permKey, index+1) + return line, false + } + + // Replace "write" with "read" in the value part + newValueAndComment := strings.Replace(valueAndComment, " write", " read", 1) + writePermissionsCodemodLog.Printf("Replaced write with read on line %d", index+1) + return fmt.Sprintf("%s:%s", key, newValueAndComment), true } diff --git a/pkg/cli/codemod_playwright_domains.go b/pkg/cli/codemod_playwright_domains.go index 7ab2b62b56a..ca69d402fec 100644 --- a/pkg/cli/codemod_playwright_domains.go +++ b/pkg/cli/codemod_playwright_domains.go @@ -17,169 +17,199 @@ func getPlaywrightDomainsToNetworkAllowedCodemod() Codemod { Name: "Migrate playwright allowed_domains to network.allowed", Description: "Moves 'tools.playwright.allowed_domains' to top-level 'network.allowed'. Playwright egress is now controlled by the firewall.", IntroducedIn: "0.9.0", - Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Check if tools.playwright.allowed_domains exists - toolsValue, hasTools := frontmatter["tools"] - if !hasTools { - return content, false, nil - } + Apply: getPlaywrightDomainsToNetworkAllowedCodemodApply, + } +} - toolsMap, ok := toolsValue.(map[string]any) - if !ok { - return content, false, nil - } +func getPlaywrightDomainsToNetworkAllowedCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + playwrightMap, ok := getPlaywrightDomainsToNetworkAllowedCodemodPlaywrightMap(frontmatter) + if !ok { + return content, false, nil + } + allowedDomainsValue, hasAllowedDomains := playwrightMap["allowed_domains"] + if !hasAllowedDomains { + return content, false, nil + } - playwrightValue, hasPlaywright := toolsMap["playwright"] - if !hasPlaywright { - return content, false, nil - } + domains := getPlaywrightDomainsToNetworkAllowedCodemodDomains(allowedDomainsValue) + existingAllowed, hasTopLevelNetwork := getPlaywrightDomainsToNetworkAllowedCodemodExistingAllowed(frontmatter) + mergedDomains := sliceutil.Deduplicate(append(existingAllowed, domains...)) - playwrightMap, ok := playwrightValue.(map[string]any) - if !ok { - return content, false, nil - } + return applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { + return getPlaywrightDomainsToNetworkAllowedCodemodTransform(lines, domains, mergedDomains, hasTopLevelNetwork) + }) +} - allowedDomainsValue, hasAllowedDomains := playwrightMap["allowed_domains"] - if !hasAllowedDomains { - return content, false, nil - } +func getPlaywrightDomainsToNetworkAllowedCodemodPlaywrightMap(frontmatter map[string]any) (map[string]any, bool) { + toolsValue, hasTools := frontmatter["tools"] + if !hasTools { + return nil, false + } + toolsMap, ok := toolsValue.(map[string]any) + if !ok { + return nil, false + } + playwrightValue, hasPlaywright := toolsMap["playwright"] + if !hasPlaywright { + return nil, false + } + playwrightMap, ok := playwrightValue.(map[string]any) + return playwrightMap, ok +} - // Extract domains from allowed_domains - var domains []string - switch v := allowedDomainsValue.(type) { - case []any: - for _, item := range v { - if s, ok := item.(string); ok { - domains = append(domains, s) - } - } - case []string: - domains = v - case string: - domains = []string{v} +func getPlaywrightDomainsToNetworkAllowedCodemodDomains(allowedDomainsValue any) []string { + var domains []string + switch v := allowedDomainsValue.(type) { + case []any: + for _, item := range v { + if s, ok := item.(string); ok { + domains = append(domains, s) } + } + case []string: + domains = v + case string: + domains = []string{v} + } + return domains +} - // Merge with existing network.allowed - existingNetworkValue, hasTopLevelNetwork := frontmatter["network"] - var existingAllowed []string - if hasTopLevelNetwork { - if existingNetworkMap, ok := existingNetworkValue.(map[string]any); ok { - if existingAllowedValue, hasExistingAllowed := existingNetworkMap["allowed"]; hasExistingAllowed { - switch allowed := existingAllowedValue.(type) { - case []any: - for _, domain := range allowed { - if domainStr, ok := domain.(string); ok { - existingAllowed = append(existingAllowed, domainStr) - } - } - case []string: - existingAllowed = append(existingAllowed, allowed...) - } - } - } +func getPlaywrightDomainsToNetworkAllowedCodemodExistingAllowed(frontmatter map[string]any) ([]string, bool) { + existingNetworkValue, hasTopLevelNetwork := frontmatter["network"] + if !hasTopLevelNetwork { + return nil, false + } + existingNetworkMap, ok := existingNetworkValue.(map[string]any) + if !ok { + return nil, true + } + existingAllowedValue, hasExistingAllowed := existingNetworkMap["allowed"] + if !hasExistingAllowed { + return nil, true + } + var existingAllowed []string + switch allowed := existingAllowedValue.(type) { + case []any: + for _, domain := range allowed { + if domainStr, ok := domain.(string); ok { + existingAllowed = append(existingAllowed, domainStr) } + } + case []string: + existingAllowed = append(existingAllowed, allowed...) + } + return existingAllowed, true +} - mergedDomains := sliceutil.Deduplicate(append(existingAllowed, domains...)) - - return applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - // Remove allowed_domains from the tools.playwright block - result, modified := removeFieldFromPlaywright(lines, "allowed_domains") - if !modified { - return lines, false - } - - playwrightDomainsCodemodLog.Printf("Removed allowed_domains from tools.playwright (%d domain(s))", len(domains)) - - if hasTopLevelNetwork { - result = updateNetworkAllowed(result, mergedDomains) - playwrightDomainsCodemodLog.Printf("Updated top-level network.allowed with %d domain(s)", len(mergedDomains)) - } else { - result = addTopLevelNetwork(result, mergedDomains) - playwrightDomainsCodemodLog.Printf("Added top-level network.allowed with %d domain(s)", len(mergedDomains)) - } - - return result, true - }) - }, +func getPlaywrightDomainsToNetworkAllowedCodemodTransform( + lines []string, + domains []string, + mergedDomains []string, + hasTopLevelNetwork bool, +) ([]string, bool) { + result, modified := removeFieldFromPlaywright(lines, "allowed_domains") + if !modified { + return lines, false + } + playwrightDomainsCodemodLog.Printf("Removed allowed_domains from tools.playwright (%d domain(s))", len(domains)) + + if hasTopLevelNetwork { + result = updateNetworkAllowed(result, mergedDomains) + playwrightDomainsCodemodLog.Printf("Updated top-level network.allowed with %d domain(s)", len(mergedDomains)) + } else { + result = addTopLevelNetwork(result, mergedDomains) + playwrightDomainsCodemodLog.Printf("Added top-level network.allowed with %d domain(s)", len(mergedDomains)) } + return result, true } // removeFieldFromPlaywright removes a field from the tools.playwright block (two-level nesting) func removeFieldFromPlaywright(lines []string, fieldName string) ([]string, bool) { var result []string var modified bool - var inTools bool - var toolsIndent string - var inPlaywright bool - var playwrightIndent string - var inFieldBlock bool - var fieldIndent string + state := removeFieldFromPlaywrightState{} for i, line := range lines { - trimmed := strings.TrimSpace(line) - - // Track if we're in the tools block - if strings.HasPrefix(trimmed, "tools:") { - inTools = true - toolsIndent = getIndentation(line) + keepLine, lineModified := removeFieldFromPlaywrightKeepLine(line, i, fieldName, &state) + if lineModified { + modified = true + } + if keepLine { result = append(result, line) - continue } + } - // Check if we've left the tools block - if inTools && trimmed != "" && !strings.HasPrefix(trimmed, "#") { - if hasExitedBlock(line, toolsIndent) { - inTools = false - inPlaywright = false - } - } + return result, modified +} - // Track if we're in the playwright block within tools - if inTools && strings.HasPrefix(trimmed, "playwright:") { - inPlaywright = true - playwrightIndent = getIndentation(line) - result = append(result, line) - continue - } +type removeFieldFromPlaywrightState struct { + inTools bool + toolsIndent string + inPlaywright bool + playwrightIndent string + inFieldBlock bool + fieldIndent string +} - // Check if we've left the playwright block - if inPlaywright && trimmed != "" && !strings.HasPrefix(trimmed, "#") { - if hasExitedBlock(line, playwrightIndent) { - inPlaywright = false - } - } +func removeFieldFromPlaywrightKeepLine(line string, index int, fieldName string, state *removeFieldFromPlaywrightState) (bool, bool) { + trimmed := strings.TrimSpace(line) - // Remove the target field if in playwright block - if inPlaywright && strings.HasPrefix(trimmed, fieldName+":") { - modified = true - inFieldBlock = true - fieldIndent = getIndentation(line) - playwrightDomainsCodemodLog.Printf("Removed %s from tools.playwright on line %d", fieldName, i+1) - continue - } + if strings.HasPrefix(trimmed, "tools:") { + state.inTools = true + state.toolsIndent = getIndentation(line) + return true, false + } - // Skip nested properties under the removed field - if inFieldBlock { - if trimmed == "" { - continue - } - currentIndent := getIndentation(line) - if strings.HasPrefix(trimmed, "#") { - if len(currentIndent) > len(fieldIndent) { - continue - } - inFieldBlock = false - result = append(result, line) - continue - } - if len(currentIndent) > len(fieldIndent) { - continue - } - inFieldBlock = false - } + removeFieldFromPlaywrightExitBlocks(line, trimmed, state) - result = append(result, line) + if state.inTools && strings.HasPrefix(trimmed, "playwright:") { + state.inPlaywright = true + state.playwrightIndent = getIndentation(line) + return true, false } - return result, modified + if state.inPlaywright && strings.HasPrefix(trimmed, fieldName+":") { + state.inFieldBlock = true + state.fieldIndent = getIndentation(line) + playwrightDomainsCodemodLog.Printf("Removed %s from tools.playwright on line %d", fieldName, index+1) + return false, true + } + + if state.inFieldBlock { + return removeFieldFromPlaywrightKeepAfterRemovedField(line, trimmed, state), false + } + return true, false +} + +func removeFieldFromPlaywrightExitBlocks(line string, trimmed string, state *removeFieldFromPlaywrightState) { + if state.inTools && trimmed != "" && !strings.HasPrefix(trimmed, "#") { + if hasExitedBlock(line, state.toolsIndent) { + state.inTools = false + state.inPlaywright = false + } + } + if state.inPlaywright && trimmed != "" && !strings.HasPrefix(trimmed, "#") { + if hasExitedBlock(line, state.playwrightIndent) { + state.inPlaywright = false + } + } +} + +func removeFieldFromPlaywrightKeepAfterRemovedField(line string, trimmed string, state *removeFieldFromPlaywrightState) bool { + if trimmed == "" { + return false + } + currentIndent := getIndentation(line) + if strings.HasPrefix(trimmed, "#") { + if len(currentIndent) > len(state.fieldIndent) { + return false + } + state.inFieldBlock = false + return true + } + if len(currentIndent) > len(state.fieldIndent) { + return false + } + state.inFieldBlock = false + return true } diff --git a/pkg/cli/codemod_run_install_scripts.go b/pkg/cli/codemod_run_install_scripts.go index 2b57c47088e..aa74118a940 100644 --- a/pkg/cli/codemod_run_install_scripts.go +++ b/pkg/cli/codemod_run_install_scripts.go @@ -66,24 +66,13 @@ func getRunInstallScriptsToRuntimesNodeCodemod() Codemod { // run-install-scripts key and injecting it under runtimes.node. func migrateRunInstallScriptsLines(lines []string, risStr string, alreadyNested bool) ([]string, bool) { // Step 1: Find the top-level run-install-scripts line - topLevelIdx := -1 - for i, line := range lines { - if isTopLevelKey(line) && strings.HasPrefix(strings.TrimSpace(line), "run-install-scripts:") { - topLevelIdx = i - break - } - } + topLevelIdx := migrateRunInstallScriptsLinesFindTopLevel(lines) if topLevelIdx == -1 { return lines, false } // Remove the top-level line - result := make([]string, 0, len(lines)) - for i, line := range lines { - if i != topLevelIdx { - result = append(result, line) - } - } + result := migrateRunInstallScriptsLinesRemoveTopLevel(lines, topLevelIdx) runInstallScriptsCodemodLog.Printf("Removed top-level 'run-install-scripts' on line %d", topLevelIdx+1) if alreadyNested { @@ -96,13 +85,7 @@ func migrateRunInstallScriptsLines(lines []string, risStr string, alreadyNested indent := detectFrontmatterIndent(result) // Step 3: Find the runtimes: block (top-level key) - runtimesIdx := -1 - for i, line := range result { - if isTopLevelKey(line) && strings.HasPrefix(strings.TrimSpace(line), "runtimes:") { - runtimesIdx = i - break - } - } + runtimesIdx := migrateRunInstallScriptsLinesFindRuntimes(result) if runtimesIdx == -1 { // No runtimes block – append a new one at the end @@ -117,49 +100,87 @@ func migrateRunInstallScriptsLines(lines []string, risStr string, alreadyNested // Step 4: Find node: as a direct child of runtimes: runtimesIndent := getIndentation(result[runtimesIdx]) - nodeIdx := -1 - for i := runtimesIdx + 1; i < len(result); i++ { - line := result[i] + nodeIdx := migrateRunInstallScriptsLinesFindNode(result, runtimesIdx, runtimesIndent) + + if nodeIdx == -1 { + runInstallScriptsCodemodLog.Print("No 'node:' sub-block found – inserting new one under 'runtimes:'") + return migrateRunInstallScriptsLinesInsertNode(result, runtimesIdx, runtimesIndent, indent, risStr), true + } + + // Step 5: node: exists – inject run-install-scripts right after node: + newResult := migrateRunInstallScriptsLinesInsertField(result, nodeIdx, indent, risStr) + runInstallScriptsCodemodLog.Printf("Injected 'run-install-scripts: %s' under existing 'node:' sub-block", risStr) + return newResult, true +} + +func migrateRunInstallScriptsLinesFindTopLevel(lines []string) int { + for i, line := range lines { + if isTopLevelKey(line) && strings.HasPrefix(strings.TrimSpace(line), "run-install-scripts:") { + return i + } + } + return -1 +} + +func migrateRunInstallScriptsLinesRemoveTopLevel(lines []string, topLevelIdx int) []string { + result := make([]string, 0, len(lines)) + for i, line := range lines { + if i != topLevelIdx { + result = append(result, line) + } + } + return result +} + +func migrateRunInstallScriptsLinesFindRuntimes(lines []string) int { + for i, line := range lines { + if isTopLevelKey(line) && strings.HasPrefix(strings.TrimSpace(line), "runtimes:") { + return i + } + } + return -1 +} + +func migrateRunInstallScriptsLinesFindNode(lines []string, runtimesIdx int, runtimesIndent string) int { + for i := runtimesIdx + 1; i < len(lines); i++ { + line := lines[i] trimmed := strings.TrimSpace(line) if trimmed == "" { continue } lineIndent := getIndentation(line) if len(lineIndent) <= len(runtimesIndent) { - // Exited the runtimes block without finding node: - break + return -1 } if strings.HasPrefix(trimmed, "node:") { - nodeIdx = i runInstallScriptsCodemodLog.Printf("Found 'node:' sub-block at line %d", i+1) - break + return i } } + return -1 +} - if nodeIdx == -1 { - // No node: sub-block – insert one right after runtimes: - nodeIndent := runtimesIndent + indent - nodeLines := []string{ - nodeIndent + "node:", - fmt.Sprintf("%s%srun-install-scripts: %s", nodeIndent, indent, risStr), - } - newResult := make([]string, 0, len(result)+2) - newResult = append(newResult, result[:runtimesIdx+1]...) - newResult = append(newResult, nodeLines...) - newResult = append(newResult, result[runtimesIdx+1:]...) - runInstallScriptsCodemodLog.Print("No 'node:' sub-block found – inserting new one under 'runtimes:'") - return newResult, true +func migrateRunInstallScriptsLinesInsertNode(lines []string, runtimesIdx int, runtimesIndent string, indent string, risStr string) []string { + nodeIndent := runtimesIndent + indent + nodeLines := []string{ + nodeIndent + "node:", + fmt.Sprintf("%s%srun-install-scripts: %s", nodeIndent, indent, risStr), } + newResult := make([]string, 0, len(lines)+2) + newResult = append(newResult, lines[:runtimesIdx+1]...) + newResult = append(newResult, nodeLines...) + newResult = append(newResult, lines[runtimesIdx+1:]...) + return newResult +} - // Step 5: node: exists – inject run-install-scripts right after node: +func migrateRunInstallScriptsLinesInsertField(result []string, nodeIdx int, indent string, risStr string) []string { nodeIndent := getIndentation(result[nodeIdx]) fieldLine := fmt.Sprintf("%s%srun-install-scripts: %s", nodeIndent, indent, risStr) newResult := make([]string, 0, len(result)+1) newResult = append(newResult, result[:nodeIdx+1]...) newResult = append(newResult, fieldLine) newResult = append(newResult, result[nodeIdx+1:]...) - runInstallScriptsCodemodLog.Printf("Injected 'run-install-scripts: %s' under existing 'node:' sub-block", risStr) - return newResult, true + return newResult } // detectFrontmatterIndent returns the indentation unit used in the YAML frontmatter diff --git a/pkg/cli/codemod_safe_output_add_reviewer_allowlists.go b/pkg/cli/codemod_safe_output_add_reviewer_allowlists.go index c8b2c95d780..1d0a9723183 100644 --- a/pkg/cli/codemod_safe_output_add_reviewer_allowlists.go +++ b/pkg/cli/codemod_safe_output_add_reviewer_allowlists.go @@ -65,87 +65,31 @@ func renameAddReviewerAllowlists(lines []string) ([]string, bool) { result := make([]string, 0, len(lines)) modified := false - inSafeOutputs := false - safeOutputsIndent := "" - safeOutputsChildIndent := "" - inAddReviewer := false - addReviewerIndent := "" - addReviewerChildIndent := "" + state := renameAddReviewerAllowlistsState{} for i, line := range lines { trimmed := strings.TrimSpace(line) indent := getIndentation(line) - if !strings.HasPrefix(trimmed, "#") { - if inSafeOutputs && hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputs = false - safeOutputsChildIndent = "" - inAddReviewer = false - addReviewerIndent = "" - addReviewerChildIndent = "" - } - if inAddReviewer && hasExitedBlock(line, addReviewerIndent) { - inAddReviewer = false - addReviewerIndent = "" - addReviewerChildIndent = "" - } - } + renameAddReviewerAllowlistsUpdateBlockState(line, trimmed, &state) if strings.HasPrefix(trimmed, "safe-outputs:") { - inSafeOutputs = true - safeOutputsIndent = indent - safeOutputsChildIndent = "" - inAddReviewer = false - addReviewerIndent = "" - addReviewerChildIndent = "" + renameAddReviewerAllowlistsEnterSafeOutputs(indent, &state) result = append(result, line) continue } - if inSafeOutputs && isDescendant(indent, safeOutputsIndent) && strings.HasSuffix(trimmed, ":") && !strings.HasPrefix(trimmed, "#") { - if safeOutputsChildIndent == "" { - safeOutputsChildIndent = indent - } - if indent == safeOutputsChildIndent { - key := strings.TrimSuffix(trimmed, ":") - if key == "add-reviewer" { - inAddReviewer = true - addReviewerIndent = indent - addReviewerChildIndent = "" - } else { - inAddReviewer = false - addReviewerIndent = "" - addReviewerChildIndent = "" - } - } + if renameAddReviewerAllowlistsEnterHandler(trimmed, indent, &state) { result = append(result, line) continue } - if inAddReviewer && addReviewerChildIndent == "" && isDescendant(indent, addReviewerIndent) && trimmed != "" && !strings.HasPrefix(trimmed, "#") { - addReviewerChildIndent = indent - } + renameAddReviewerAllowlistsSetChildIndent(trimmed, indent, &state) - // Rename "reviewers:" but not "team-reviewers:" on the same pass — check full prefix - if inAddReviewer && indent == addReviewerChildIndent { - if strings.HasPrefix(trimmed, "reviewers:") && !strings.HasPrefix(trimmed, "team-reviewers:") { - newLine, replaced := findAndReplaceInLine(line, "reviewers", "allowed-reviewers") - if replaced { - result = append(result, newLine) - modified = true - safeOutputAddReviewerAllowlistsCodemodLog.Printf("Renamed reviewers to allowed-reviewers in safe-outputs.add-reviewer on line %d", i+1) - continue - } - } - if strings.HasPrefix(trimmed, "team-reviewers:") { - newLine, replaced := findAndReplaceInLine(line, "team-reviewers", "allowed-team-reviewers") - if replaced { - result = append(result, newLine) - modified = true - safeOutputAddReviewerAllowlistsCodemodLog.Printf("Renamed team-reviewers to allowed-team-reviewers in safe-outputs.add-reviewer on line %d", i+1) - continue - } - } + if newLine, replaced := renameAddReviewerAllowlistsReplaceLine(line, trimmed, indent, i, &state); replaced { + result = append(result, newLine) + modified = true + continue } result = append(result, line) @@ -153,3 +97,90 @@ func renameAddReviewerAllowlists(lines []string) ([]string, bool) { return result, modified } + +type renameAddReviewerAllowlistsState struct { + inSafeOutputs bool + safeOutputsIndent string + safeOutputsChildIndent string + inAddReviewer bool + addReviewerIndent string + addReviewerChildIndent string +} + +func renameAddReviewerAllowlistsUpdateBlockState(line, trimmed string, state *renameAddReviewerAllowlistsState) { + if strings.HasPrefix(trimmed, "#") { + return + } + if state.inSafeOutputs && hasExitedBlock(line, state.safeOutputsIndent) { + state.inSafeOutputs = false + state.safeOutputsChildIndent = "" + state.inAddReviewer = false + state.addReviewerIndent = "" + state.addReviewerChildIndent = "" + } + if state.inAddReviewer && hasExitedBlock(line, state.addReviewerIndent) { + state.inAddReviewer = false + state.addReviewerIndent = "" + state.addReviewerChildIndent = "" + } +} + +func renameAddReviewerAllowlistsEnterSafeOutputs(indent string, state *renameAddReviewerAllowlistsState) { + state.inSafeOutputs = true + state.safeOutputsIndent = indent + state.safeOutputsChildIndent = "" + state.inAddReviewer = false + state.addReviewerIndent = "" + state.addReviewerChildIndent = "" +} + +func renameAddReviewerAllowlistsEnterHandler(trimmed, indent string, state *renameAddReviewerAllowlistsState) bool { + if !state.inSafeOutputs || !isDescendant(indent, state.safeOutputsIndent) || + !strings.HasSuffix(trimmed, ":") || strings.HasPrefix(trimmed, "#") { + return false + } + if state.safeOutputsChildIndent == "" { + state.safeOutputsChildIndent = indent + } + if indent == state.safeOutputsChildIndent { + key := strings.TrimSuffix(trimmed, ":") + state.inAddReviewer = key == "add-reviewer" + if state.inAddReviewer { + state.addReviewerIndent = indent + state.addReviewerChildIndent = "" + } else { + state.addReviewerIndent = "" + state.addReviewerChildIndent = "" + } + } + return true +} + +func renameAddReviewerAllowlistsSetChildIndent(trimmed, indent string, state *renameAddReviewerAllowlistsState) { + if state.inAddReviewer && state.addReviewerChildIndent == "" && isDescendant(indent, state.addReviewerIndent) && + trimmed != "" && !strings.HasPrefix(trimmed, "#") { + state.addReviewerChildIndent = indent + } +} + +func renameAddReviewerAllowlistsReplaceLine(line, trimmed, indent string, lineIndex int, state *renameAddReviewerAllowlistsState) (string, bool) { + if !state.inAddReviewer || indent != state.addReviewerChildIndent { + return "", false + } + // Rename "reviewers:" but not "team-reviewers:" on the same pass — check full prefix + if strings.HasPrefix(trimmed, "reviewers:") && !strings.HasPrefix(trimmed, "team-reviewers:") { + newLine, replaced := findAndReplaceInLine(line, "reviewers", "allowed-reviewers") + if replaced { + safeOutputAddReviewerAllowlistsCodemodLog.Printf("Renamed reviewers to allowed-reviewers in safe-outputs.add-reviewer on line %d", lineIndex+1) + } + return newLine, replaced + } + if strings.HasPrefix(trimmed, "team-reviewers:") { + newLine, replaced := findAndReplaceInLine(line, "team-reviewers", "allowed-team-reviewers") + if replaced { + safeOutputAddReviewerAllowlistsCodemodLog.Printf("Renamed team-reviewers to allowed-team-reviewers in safe-outputs.add-reviewer on line %d", lineIndex+1) + } + return newLine, replaced + } + return "", false +} diff --git a/pkg/cli/codemod_safe_output_merge_pr_constraints.go b/pkg/cli/codemod_safe_output_merge_pr_constraints.go index 34b8e7babdd..8ab3389f47e 100644 --- a/pkg/cli/codemod_safe_output_merge_pr_constraints.go +++ b/pkg/cli/codemod_safe_output_merge_pr_constraints.go @@ -60,75 +60,31 @@ func renameMergePRConstraints(lines []string) ([]string, bool) { result := make([]string, 0, len(lines)) modified := false - inSafeOutputs := false - safeOutputsIndent := "" - safeOutputsChildIndent := "" - inMergePR := false - mergePRIndent := "" - mergePRChildIndent := "" + state := renameMergePRConstraintsState{} for i, line := range lines { trimmed := strings.TrimSpace(line) indent := getIndentation(line) - if !strings.HasPrefix(trimmed, "#") { - if inSafeOutputs && hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputs = false - safeOutputsChildIndent = "" - inMergePR = false - mergePRIndent = "" - mergePRChildIndent = "" - } - if inMergePR && hasExitedBlock(line, mergePRIndent) { - inMergePR = false - mergePRIndent = "" - mergePRChildIndent = "" - } - } + renameMergePRConstraintsUpdateBlockState(line, trimmed, &state) if strings.HasPrefix(trimmed, "safe-outputs:") { - inSafeOutputs = true - safeOutputsIndent = indent - safeOutputsChildIndent = "" - inMergePR = false - mergePRIndent = "" - mergePRChildIndent = "" + renameMergePRConstraintsEnterSafeOutputs(indent, &state) result = append(result, line) continue } - if inSafeOutputs && isDescendant(indent, safeOutputsIndent) && strings.HasSuffix(trimmed, ":") && !strings.HasPrefix(trimmed, "#") { - if safeOutputsChildIndent == "" { - safeOutputsChildIndent = indent - } - if indent == safeOutputsChildIndent { - key := strings.TrimSuffix(trimmed, ":") - if key == "merge-pull-request" { - inMergePR = true - mergePRIndent = indent - mergePRChildIndent = "" - } else { - inMergePR = false - mergePRIndent = "" - mergePRChildIndent = "" - } - } + if renameMergePRConstraintsEnterHandler(trimmed, indent, &state) { result = append(result, line) continue } - if inMergePR && mergePRChildIndent == "" && isDescendant(indent, mergePRIndent) && trimmed != "" && !strings.HasPrefix(trimmed, "#") { - mergePRChildIndent = indent - } + renameMergePRConstraintsSetChildIndent(trimmed, indent, &state) - if inMergePR && indent == mergePRChildIndent && strings.HasPrefix(trimmed, "allowed-labels:") { - newLine, replaced := findAndReplaceInLine(line, "allowed-labels", "required-labels") - if replaced { - result = append(result, newLine) - modified = true - safeOutputMergePRConstraintsCodemodLog.Printf("Renamed allowed-labels to required-labels in safe-outputs.merge-pull-request on line %d", i+1) - continue - } + if newLine, replaced := renameMergePRConstraintsReplaceLine(line, trimmed, indent, i, &state); replaced { + result = append(result, newLine) + modified = true + continue } result = append(result, line) @@ -136,3 +92,79 @@ func renameMergePRConstraints(lines []string) ([]string, bool) { return result, modified } + +type renameMergePRConstraintsState struct { + inSafeOutputs bool + safeOutputsIndent string + safeOutputsChildIndent string + inMergePR bool + mergePRIndent string + mergePRChildIndent string +} + +func renameMergePRConstraintsUpdateBlockState(line, trimmed string, state *renameMergePRConstraintsState) { + if strings.HasPrefix(trimmed, "#") { + return + } + if state.inSafeOutputs && hasExitedBlock(line, state.safeOutputsIndent) { + state.inSafeOutputs = false + state.safeOutputsChildIndent = "" + state.inMergePR = false + state.mergePRIndent = "" + state.mergePRChildIndent = "" + } + if state.inMergePR && hasExitedBlock(line, state.mergePRIndent) { + state.inMergePR = false + state.mergePRIndent = "" + state.mergePRChildIndent = "" + } +} + +func renameMergePRConstraintsEnterSafeOutputs(indent string, state *renameMergePRConstraintsState) { + state.inSafeOutputs = true + state.safeOutputsIndent = indent + state.safeOutputsChildIndent = "" + state.inMergePR = false + state.mergePRIndent = "" + state.mergePRChildIndent = "" +} + +func renameMergePRConstraintsEnterHandler(trimmed, indent string, state *renameMergePRConstraintsState) bool { + if !state.inSafeOutputs || !isDescendant(indent, state.safeOutputsIndent) || + !strings.HasSuffix(trimmed, ":") || strings.HasPrefix(trimmed, "#") { + return false + } + if state.safeOutputsChildIndent == "" { + state.safeOutputsChildIndent = indent + } + if indent == state.safeOutputsChildIndent { + key := strings.TrimSuffix(trimmed, ":") + state.inMergePR = key == "merge-pull-request" + if state.inMergePR { + state.mergePRIndent = indent + state.mergePRChildIndent = "" + } else { + state.mergePRIndent = "" + state.mergePRChildIndent = "" + } + } + return true +} + +func renameMergePRConstraintsSetChildIndent(trimmed, indent string, state *renameMergePRConstraintsState) { + if state.inMergePR && state.mergePRChildIndent == "" && isDescendant(indent, state.mergePRIndent) && + trimmed != "" && !strings.HasPrefix(trimmed, "#") { + state.mergePRChildIndent = indent + } +} + +func renameMergePRConstraintsReplaceLine(line, trimmed, indent string, lineIndex int, state *renameMergePRConstraintsState) (string, bool) { + if !state.inMergePR || indent != state.mergePRChildIndent || !strings.HasPrefix(trimmed, "allowed-labels:") { + return "", false + } + newLine, replaced := findAndReplaceInLine(line, "allowed-labels", "required-labels") + if replaced { + safeOutputMergePRConstraintsCodemodLog.Printf("Renamed allowed-labels to required-labels in safe-outputs.merge-pull-request on line %d", lineIndex+1) + } + return newLine, replaced +} diff --git a/pkg/cli/codemod_safe_output_require_title_prefix.go b/pkg/cli/codemod_safe_output_require_title_prefix.go index 27c673fb93d..77cd24a98a6 100644 --- a/pkg/cli/codemod_safe_output_require_title_prefix.go +++ b/pkg/cli/codemod_safe_output_require_title_prefix.go @@ -92,89 +92,31 @@ func renameSafeOutputTitlePrefixConstraints(lines []string, handlersToRename map result := make([]string, 0, len(lines)) modified := false - inSafeOutputs := false - safeOutputsIndent := "" - safeOutputsChildIndent := "" - activeHandler := "" - activeHandlerIndent := "" - handlerChildIndent := "" + state := renameSafeOutputTitlePrefixConstraintsState{} for i, line := range lines { trimmed := strings.TrimSpace(line) indent := getIndentation(line) - if !strings.HasPrefix(trimmed, "#") { - if inSafeOutputs && hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputs = false - safeOutputsChildIndent = "" - activeHandler = "" - activeHandlerIndent = "" - handlerChildIndent = "" - } - if activeHandler != "" && hasExitedBlock(line, activeHandlerIndent) { - activeHandler = "" - activeHandlerIndent = "" - handlerChildIndent = "" - } - } + renameSafeOutputTitlePrefixConstraintsUpdateBlockState(line, trimmed, &state) if strings.HasPrefix(trimmed, "safe-outputs:") { - inSafeOutputs = true - safeOutputsIndent = indent - safeOutputsChildIndent = "" - activeHandler = "" - activeHandlerIndent = "" - handlerChildIndent = "" + renameSafeOutputTitlePrefixConstraintsEnterSafeOutputs(indent, &state) result = append(result, line) continue } - if inSafeOutputs && isDescendant(indent, safeOutputsIndent) && strings.HasSuffix(trimmed, ":") && !strings.HasPrefix(trimmed, "#") { - if activeHandler != "" && handlerChildIndent == "" && isDescendant(indent, activeHandlerIndent) { - handlerChildIndent = indent - } - if safeOutputsChildIndent == "" { - safeOutputsChildIndent = indent - } - if indent != safeOutputsChildIndent { - result = append(result, line) - continue - } - key := strings.TrimSuffix(trimmed, ":") - if setutil.Contains(handlersToRename, key) { - activeHandler = key - activeHandlerIndent = indent - handlerChildIndent = "" - } else { - activeHandler = "" - activeHandlerIndent = "" - handlerChildIndent = "" - } + if renameSafeOutputTitlePrefixConstraintsEnterHandler(trimmed, indent, handlersToRename, &state) { result = append(result, line) continue } - if activeHandler != "" && handlerChildIndent == "" && isDescendant(indent, activeHandlerIndent) && trimmed != "" && !strings.HasPrefix(trimmed, "#") && !strings.HasPrefix(trimmed, "- ") { - handlerChildIndent = indent - } + renameSafeOutputTitlePrefixConstraintsSetChildIndent(trimmed, indent, &state) - if activeHandler != "" && indent == handlerChildIndent && strings.HasPrefix(trimmed, "title-prefix:") { - newLine, replaced := findAndReplaceInLine(line, "title-prefix", "required-title-prefix") - if replaced { - result = append(result, newLine) - modified = true - safeOutputRequireTitlePrefixCodemodLog.Printf("Renamed title-prefix in safe-outputs.%s on line %d", activeHandler, i+1) - continue - } - } - if activeHandler == "push-to-pull-request-branch" && indent == handlerChildIndent && strings.HasPrefix(trimmed, "labels:") { - newLine, replaced := findAndReplaceInLine(line, "labels", "required-labels") - if replaced { - result = append(result, newLine) - modified = true - safeOutputRequireTitlePrefixCodemodLog.Printf("Renamed labels in safe-outputs.%s on line %d", activeHandler, i+1) - continue - } + if newLine, replaced := renameSafeOutputTitlePrefixConstraintsReplaceLine(line, trimmed, indent, i, &state); replaced { + result = append(result, newLine) + modified = true + continue } result = append(result, line) @@ -182,3 +124,92 @@ func renameSafeOutputTitlePrefixConstraints(lines []string, handlersToRename map return result, modified } + +type renameSafeOutputTitlePrefixConstraintsState struct { + inSafeOutputs bool + safeOutputsIndent string + safeOutputsChildIndent string + activeHandler string + activeHandlerIndent string + handlerChildIndent string +} + +func renameSafeOutputTitlePrefixConstraintsUpdateBlockState(line, trimmed string, state *renameSafeOutputTitlePrefixConstraintsState) { + if strings.HasPrefix(trimmed, "#") { + return + } + if state.inSafeOutputs && hasExitedBlock(line, state.safeOutputsIndent) { + state.inSafeOutputs = false + state.safeOutputsChildIndent = "" + state.activeHandler = "" + state.activeHandlerIndent = "" + state.handlerChildIndent = "" + } + if state.activeHandler != "" && hasExitedBlock(line, state.activeHandlerIndent) { + state.activeHandler = "" + state.activeHandlerIndent = "" + state.handlerChildIndent = "" + } +} + +func renameSafeOutputTitlePrefixConstraintsEnterSafeOutputs(indent string, state *renameSafeOutputTitlePrefixConstraintsState) { + state.inSafeOutputs = true + state.safeOutputsIndent = indent + state.safeOutputsChildIndent = "" + state.activeHandler = "" + state.activeHandlerIndent = "" + state.handlerChildIndent = "" +} + +func renameSafeOutputTitlePrefixConstraintsEnterHandler(trimmed, indent string, handlersToRename map[string]struct { +}, state *renameSafeOutputTitlePrefixConstraintsState) bool { + if !state.inSafeOutputs || !isDescendant(indent, state.safeOutputsIndent) || + !strings.HasSuffix(trimmed, ":") || strings.HasPrefix(trimmed, "#") { + return false + } + if state.activeHandler != "" && state.handlerChildIndent == "" && isDescendant(indent, state.activeHandlerIndent) { + state.handlerChildIndent = indent + } + if state.safeOutputsChildIndent == "" { + state.safeOutputsChildIndent = indent + } + if indent != state.safeOutputsChildIndent { + return true + } + key := strings.TrimSuffix(trimmed, ":") + if setutil.Contains(handlersToRename, key) { + state.activeHandler = key + state.activeHandlerIndent = indent + state.handlerChildIndent = "" + } else { + state.activeHandler = "" + state.activeHandlerIndent = "" + state.handlerChildIndent = "" + } + return true +} + +func renameSafeOutputTitlePrefixConstraintsSetChildIndent(trimmed, indent string, state *renameSafeOutputTitlePrefixConstraintsState) { + if state.activeHandler != "" && state.handlerChildIndent == "" && isDescendant(indent, state.activeHandlerIndent) && + trimmed != "" && !strings.HasPrefix(trimmed, "#") && !strings.HasPrefix(trimmed, "- ") { + state.handlerChildIndent = indent + } +} + +func renameSafeOutputTitlePrefixConstraintsReplaceLine(line, trimmed, indent string, lineIndex int, state *renameSafeOutputTitlePrefixConstraintsState) (string, bool) { + if state.activeHandler != "" && indent == state.handlerChildIndent && strings.HasPrefix(trimmed, "title-prefix:") { + newLine, replaced := findAndReplaceInLine(line, "title-prefix", "required-title-prefix") + if replaced { + safeOutputRequireTitlePrefixCodemodLog.Printf("Renamed title-prefix in safe-outputs.%s on line %d", state.activeHandler, lineIndex+1) + } + return newLine, replaced + } + if state.activeHandler == "push-to-pull-request-branch" && indent == state.handlerChildIndent && strings.HasPrefix(trimmed, "labels:") { + newLine, replaced := findAndReplaceInLine(line, "labels", "required-labels") + if replaced { + safeOutputRequireTitlePrefixCodemodLog.Printf("Renamed labels in safe-outputs.%s on line %d", state.activeHandler, lineIndex+1) + } + return newLine, replaced + } + return "", false +} diff --git a/pkg/cli/codemod_schedule.go b/pkg/cli/codemod_schedule.go index 82387cdb4bb..6c67ace2c82 100644 --- a/pkg/cli/codemod_schedule.go +++ b/pkg/cli/codemod_schedule.go @@ -16,117 +16,131 @@ func getScheduleAtToAroundCodemod() Codemod { Name: "Migrate schedule 'at' syntax to 'around' syntax", Description: "Converts deprecated 'daily at TIME', 'weekly on DAY at TIME', and 'monthly on N at TIME' to fuzzy schedules or standard cron", IntroducedIn: "0.5.0", - Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - return applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - var modified bool - result := make([]string, len(lines)) - - for i, line := range lines { - trimmedLine := strings.TrimSpace(line) - originalLine := line - - // Skip if not a cron or schedule line - if !strings.Contains(trimmedLine, "cron:") && !strings.Contains(trimmedLine, "schedule:") { - result[i] = originalLine - continue - } - - // Extract leading whitespace to preserve indentation - leadingSpace := getIndentation(line) - - // Check if this is a list item (starts with - after whitespace) - restAfterSpace := strings.TrimLeft(line, " \t") - var listMarker string - if strings.HasPrefix(restAfterSpace, "-") { - // This is a list item, preserve the dash - listMarker = "- " - } - - // Extract the schedule value (after "cron:" or "schedule:") - var scheduleValue string - var fieldName string - - if strings.Contains(trimmedLine, "cron:") { - parts := strings.SplitN(trimmedLine, "cron:", 2) - if len(parts) == 2 { - fieldName = "cron" - scheduleValue = strings.TrimSpace(parts[1]) - } - } else if strings.Contains(trimmedLine, "schedule:") { - parts := strings.SplitN(trimmedLine, "schedule:", 2) - if len(parts) == 2 { - fieldName = "schedule" - scheduleValue = strings.TrimSpace(parts[1]) - } - } - - if scheduleValue == "" { - result[i] = originalLine - continue - } - - // Remove quotes if present - scheduleValue = strings.Trim(scheduleValue, "\"'") - - // Pattern 1: daily at TIME (not "daily around" or "daily between") - if strings.HasPrefix(scheduleValue, "daily at") && !strings.Contains(scheduleValue, "around") && !strings.Contains(scheduleValue, "between") { - newSchedule := strings.Replace(scheduleValue, "daily at", "daily around", 1) - result[i] = fmt.Sprintf("%s%s%s: %s", leadingSpace, listMarker, fieldName, newSchedule) - modified = true - scheduleCodemodLog.Printf("Converted 'daily at' to 'daily around' on line %d: %s -> %s", i+1, scheduleValue, newSchedule) - continue - } - - // Pattern 2: weekly on DAY at TIME - if strings.Contains(scheduleValue, "weekly on") && strings.Contains(scheduleValue, " at ") && !strings.Contains(scheduleValue, "around") { - newSchedule := strings.Replace(scheduleValue, " at ", " around ", 1) - result[i] = fmt.Sprintf("%s%s%s: %s", leadingSpace, listMarker, fieldName, newSchedule) - modified = true - scheduleCodemodLog.Printf("Converted 'weekly on DAY at' to 'weekly on DAY around' on line %d: %s -> %s", i+1, scheduleValue, newSchedule) - continue - } - - // Pattern 3: monthly on N [at TIME] - convert to cron - if strings.HasPrefix(scheduleValue, "monthly on") { - // Extract day number - var day string - var cronExpr string - - monthlyParts := strings.Fields(scheduleValue) - for idx, part := range monthlyParts { - if part == "on" && idx+1 < len(monthlyParts) { - day = monthlyParts[idx+1] - break - } - } - - if day != "" { - // Check if there's a time specification - if strings.Contains(scheduleValue, " at ") { - // Has time - default to 09:00 as example since we can't parse arbitrary times in codemod - // The user should manually adjust the hour/minute if needed - cronExpr = fmt.Sprintf("0 9 %s * *", day) - } else { - // No time - suggest midnight - cronExpr = fmt.Sprintf("0 0 %s * *", day) - } - - // Replace with cron and add explanatory comment - result[i] = fmt.Sprintf("%s%s%s: \"%s\" # Converted from '%s' (adjust time as needed)", leadingSpace, listMarker, fieldName, cronExpr, scheduleValue) - modified = true - scheduleCodemodLog.Printf("Converted 'monthly on' to cron on line %d: %s -> %s", i+1, scheduleValue, cronExpr) - continue - } - } - - result[i] = originalLine - } - - if modified { - scheduleCodemodLog.Print("Applied schedule 'at' to 'around' migration") - } - return result, modified - }) - }, + Apply: getScheduleAtToAroundCodemodApply, } } + +func getScheduleAtToAroundCodemodApply(content string, _ map[string]any) (string, bool, error) { + return applyFrontmatterLineTransform(content, getScheduleAtToAroundCodemodTransform) +} + +func getScheduleAtToAroundCodemodTransform(lines []string) ([]string, bool) { + var modified bool + result := make([]string, len(lines)) + + for i, line := range lines { + newLine, changed := getScheduleAtToAroundCodemodTransformLine(line, i) + result[i] = newLine + modified = modified || changed + } + + if modified { + scheduleCodemodLog.Print("Applied schedule 'at' to 'around' migration") + } + return result, modified +} + +func getScheduleAtToAroundCodemodTransformLine(line string, lineIndex int) (string, bool) { + trimmedLine := strings.TrimSpace(line) + // Skip if not a cron or schedule line + if !strings.Contains(trimmedLine, "cron:") && !strings.Contains(trimmedLine, "schedule:") { + return line, false + } + + leadingSpace, listMarker, fieldName, scheduleValue := getScheduleAtToAroundCodemodLineParts(line, trimmedLine) + if scheduleValue == "" { + return line, false + } + + // Remove quotes if present + scheduleValue = strings.Trim(scheduleValue, "\"'") + if newLine, changed := getScheduleAtToAroundCodemodDailyLine(leadingSpace, listMarker, fieldName, scheduleValue, lineIndex); changed { + return newLine, true + } + if newLine, changed := getScheduleAtToAroundCodemodWeeklyLine(leadingSpace, listMarker, fieldName, scheduleValue, lineIndex); changed { + return newLine, true + } + if newLine, changed := getScheduleAtToAroundCodemodMonthlyLine(leadingSpace, listMarker, fieldName, scheduleValue, lineIndex); changed { + return newLine, true + } + return line, false +} + +func getScheduleAtToAroundCodemodLineParts(line, trimmedLine string) (string, string, string, string) { + // Extract leading whitespace to preserve indentation + leadingSpace := getIndentation(line) + + // Check if this is a list item (starts with - after whitespace) + restAfterSpace := strings.TrimLeft(line, " \t") + listMarker := "" + if strings.HasPrefix(restAfterSpace, "-") { + // This is a list item, preserve the dash + listMarker = "- " + } + + // Extract the schedule value (after "cron:" or "schedule:") + if strings.Contains(trimmedLine, "cron:") { + parts := strings.SplitN(trimmedLine, "cron:", 2) + if len(parts) == 2 { + return leadingSpace, listMarker, "cron", strings.TrimSpace(parts[1]) + } + } else if strings.Contains(trimmedLine, "schedule:") { + parts := strings.SplitN(trimmedLine, "schedule:", 2) + if len(parts) == 2 { + return leadingSpace, listMarker, "schedule", strings.TrimSpace(parts[1]) + } + } + return leadingSpace, listMarker, "", "" +} + +func getScheduleAtToAroundCodemodDailyLine(leadingSpace, listMarker, fieldName, scheduleValue string, lineIndex int) (string, bool) { + // Pattern 1: daily at TIME (not "daily around" or "daily between") + if !strings.HasPrefix(scheduleValue, "daily at") || strings.Contains(scheduleValue, "around") || strings.Contains(scheduleValue, "between") { + return "", false + } + newSchedule := strings.Replace(scheduleValue, "daily at", "daily around", 1) + scheduleCodemodLog.Printf("Converted 'daily at' to 'daily around' on line %d: %s -> %s", lineIndex+1, scheduleValue, newSchedule) + return fmt.Sprintf("%s%s%s: %s", leadingSpace, listMarker, fieldName, newSchedule), true +} + +func getScheduleAtToAroundCodemodWeeklyLine(leadingSpace, listMarker, fieldName, scheduleValue string, lineIndex int) (string, bool) { + // Pattern 2: weekly on DAY at TIME + if !strings.Contains(scheduleValue, "weekly on") || !strings.Contains(scheduleValue, " at ") || strings.Contains(scheduleValue, "around") { + return "", false + } + newSchedule := strings.Replace(scheduleValue, " at ", " around ", 1) + scheduleCodemodLog.Printf("Converted 'weekly on DAY at' to 'weekly on DAY around' on line %d: %s -> %s", lineIndex+1, scheduleValue, newSchedule) + return fmt.Sprintf("%s%s%s: %s", leadingSpace, listMarker, fieldName, newSchedule), true +} + +func getScheduleAtToAroundCodemodMonthlyLine(leadingSpace, listMarker, fieldName, scheduleValue string, lineIndex int) (string, bool) { + // Pattern 3: monthly on N [at TIME] - convert to cron + if !strings.HasPrefix(scheduleValue, "monthly on") { + return "", false + } + day := getScheduleAtToAroundCodemodMonthlyDay(scheduleValue) + if day == "" { + return "", false + } + cronExpr := fmt.Sprintf("0 0 %s * *", day) + if strings.Contains(scheduleValue, " at ") { + // Has time - default to 09:00 as example since we can't parse arbitrary times in codemod + // The user should manually adjust the hour/minute if needed + cronExpr = fmt.Sprintf("0 9 %s * *", day) + } + + // Replace with cron and add explanatory comment + scheduleCodemodLog.Printf("Converted 'monthly on' to cron on line %d: %s -> %s", lineIndex+1, scheduleValue, cronExpr) + return fmt.Sprintf("%s%s%s: \"%s\" # Converted from '%s' (adjust time as needed)", leadingSpace, listMarker, fieldName, cronExpr, scheduleValue), true +} + +func getScheduleAtToAroundCodemodMonthlyDay(scheduleValue string) string { + // Extract day number + monthlyParts := strings.Fields(scheduleValue) + for idx, part := range monthlyParts { + if part == "on" && idx+1 < len(monthlyParts) { + return monthlyParts[idx+1] + } + } + return "" +} diff --git a/pkg/cli/codemod_serena_import.go b/pkg/cli/codemod_serena_import.go index d2dc10d0c85..437b25cda03 100644 --- a/pkg/cli/codemod_serena_import.go +++ b/pkg/cli/codemod_serena_import.go @@ -343,21 +343,7 @@ func isSerenaImportPath(path string) bool { } func addSerenaImport(lines []string, languages []string) []string { - var langLine string - if len(languages) == 0 { - // No languages were specified in the original workflow. Emit a placeholder so - // the user knows what to fill in. The empty array is valid per the import-schema - // (the field is present); Serena simply won't analyse any language until updated. - langLine = ` languages: [] # TODO: specify languages, e.g. ["TypeScript", "JavaScript"]` - } else { - langLine = " languages: " + formatStringArrayInline(languages) - } - entry := []string{ - " - uses: shared/mcp/serena.md", - " with:", - langLine, - } - + entry := addSerenaImportEntry(languages) importsIdx := -1 importsEnd := len(lines) for i, line := range lines { @@ -407,6 +393,23 @@ func addSerenaImport(lines []string, languages []string) []string { return result } +func addSerenaImportEntry(languages []string) []string { + var langLine string + if len(languages) == 0 { + // No languages were specified in the original workflow. Emit a placeholder so + // the user knows what to fill in. The empty array is valid per the import-schema + // (the field is present); Serena simply won't analyse any language until updated. + langLine = ` languages: [] # TODO: specify languages, e.g. ["TypeScript", "JavaScript"]` + } else { + langLine = " languages: " + formatStringArrayInline(languages) + } + return []string{ + " - uses: shared/mcp/serena.md", + " with:", + langLine, + } +} + func formatStringArrayInline(values []string) string { quoted := make([]string, 0, len(values)) for _, value := range values { diff --git a/pkg/cli/codemod_slash_command.go b/pkg/cli/codemod_slash_command.go index 0e3581237e6..64bcf7a871d 100644 --- a/pkg/cli/codemod_slash_command.go +++ b/pkg/cli/codemod_slash_command.go @@ -15,67 +15,74 @@ func getCommandToSlashCommandCodemod() Codemod { Name: "Migrate on.command to on.slash_command", Description: "Replaces deprecated 'on.command' field with 'on.slash_command'", IntroducedIn: "0.2.0", - Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Check if on.command exists - onValue, hasOn := frontmatter["on"] - if !hasOn { - return content, false, nil - } + Apply: getCommandToSlashCommandCodemodApply, + } +} - onMap, ok := onValue.(map[string]any) - if !ok { - return content, false, nil - } +func getCommandToSlashCommandCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + if !getCommandToSlashCommandCodemodNeedsMigration(frontmatter) { + return content, false, nil + } - // Check if command field exists in on - _, hasCommand := onMap["command"] - if !hasCommand { - return content, false, nil - } + newContent, applied, err := applyFrontmatterLineTransform(content, getCommandToSlashCommandCodemodTransform) + if applied { + slashCommandCodemodLog.Print("Applied on.command to on.slash_command migration") + } + return newContent, applied, err +} - newContent, applied, err := applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - var modified bool - var inOnBlock bool - var onIndent string - result := make([]string, len(lines)) - for i, line := range lines { - trimmedLine := strings.TrimSpace(line) +func getCommandToSlashCommandCodemodNeedsMigration(frontmatter map[string]any) bool { + // Check if on.command exists + onValue, hasOn := frontmatter["on"] + if !hasOn { + return false + } - // Track if we're in the on block - if strings.HasPrefix(trimmedLine, "on:") { - inOnBlock = true - onIndent = getIndentation(line) - result[i] = line - continue - } + onMap, ok := onValue.(map[string]any) + if !ok { + return false + } - // Check if we've left the on block - if inOnBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, onIndent) { - inOnBlock = false - } - } + // Check if command field exists in on + _, hasCommand := onMap["command"] + return hasCommand +} - // Replace command with slash_command if in on block - if inOnBlock && strings.HasPrefix(trimmedLine, "command:") { - replacedLine, didReplace := findAndReplaceInLine(line, "command", "slash_command") - if didReplace { - result[i] = replacedLine - modified = true - slashCommandCodemodLog.Printf("Replaced on.command with on.slash_command on line %d", i+1) - } else { - result[i] = line - } - } else { - result[i] = line - } - } - return result, modified - }) - if applied { - slashCommandCodemodLog.Print("Applied on.command to on.slash_command migration") - } - return newContent, applied, err - }, +func getCommandToSlashCommandCodemodTransform(lines []string) ([]string, bool) { + var modified bool + var inOnBlock bool + var onIndent string + result := make([]string, len(lines)) + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) + inOnBlock, onIndent = getCommandToSlashCommandCodemodTrackOnBlock(line, trimmedLine, inOnBlock, onIndent) + result[i], modified = getCommandToSlashCommandCodemodReplaceLine(line, trimmedLine, inOnBlock, i, modified) + } + return result, modified +} + +func getCommandToSlashCommandCodemodTrackOnBlock(line, trimmedLine string, inOnBlock bool, onIndent string) (bool, string) { + // Track if we're in the on block + if strings.HasPrefix(trimmedLine, "on:") { + return true, getIndentation(line) + } + + // Check if we've left the on block + if inOnBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") && hasExitedBlock(line, onIndent) { + return false, onIndent + } + return inOnBlock, onIndent +} + +func getCommandToSlashCommandCodemodReplaceLine(line, trimmedLine string, inOnBlock bool, lineIndex int, modified bool) (string, bool) { + // Replace command with slash_command if in on block + if !inOnBlock || !strings.HasPrefix(trimmedLine, "command:") { + return line, modified + } + replacedLine, didReplace := findAndReplaceInLine(line, "command", "slash_command") + if didReplace { + slashCommandCodemodLog.Printf("Replaced on.command with on.slash_command on line %d", lineIndex+1) + return replacedLine, true } + return line, modified } diff --git a/pkg/cli/codemod_steps_run_secrets_env.go b/pkg/cli/codemod_steps_run_secrets_env.go index 4ab842a01f2..c8dd4e40e92 100644 --- a/pkg/cli/codemod_steps_run_secrets_env.go +++ b/pkg/cli/codemod_steps_run_secrets_env.go @@ -154,23 +154,55 @@ func transformStepsWithinSection(sectionLines []string, sectionIndent string) ([ func rewriteStepRunSecretsToEnv(stepLines []string, stepIndent string) ([]string, bool) { modified := false - seen := make(map[string]struct { - }) - orderedBindings := make([]string, 0) - bindingExprs := make(map[string]string) - firstRunLine := -1 - envStart := -1 - envEnd := -1 - envIndent := "" - var envKeyIndentLen int - existingEnvKeys := make(map[string]struct { - }) + state := rewriteStepRunSecretsToEnvState{ + seen: make(map[string]struct{}), + orderedBindings: make([]string, 0), + bindingExprs: make(map[string]string), + firstRunLine: -1, + envStart: -1, + envEnd: -1, + existingEnvKeys: make(map[string]struct{}), + } // First pass: detect shell type so PowerShell steps get $env:VARNAME syntax. // Restrict the scan to lines at the direct step-key indentation level so // that a run-block body line that happens to contain a literal substring // like "shell: pwsh" is not misclassified as PowerShell. - shellIsPowerShell := false + shellIsPowerShell := rewriteStepRunSecretsToEnvDetectPowerShell(stepLines, stepIndent) + + for i := 0; i < len(stepLines); i++ { + changed := rewriteStepRunSecretsToEnvProcessLine(stepLines, i, stepIndent, shellIsPowerShell, &state) + modified = modified || changed + } + + if len(state.orderedBindings) == 0 { + return stepLines, modified + } + + stepsRunSecretsEnvCodemodLog.Printf("Found %d unique run expression references in step run commands", len(state.orderedBindings)) + + missingBindings := rewriteStepRunSecretsToEnvMissingBindings(&state) + if len(missingBindings) == 0 { + return stepLines, true + } + + stepsRunSecretsEnvCodemodLog.Printf("Adding env bindings for %d missing expressions: %v", len(missingBindings), missingBindings) + return rewriteStepRunSecretsToEnvInsertMissingBindings(stepLines, stepIndent, missingBindings, modified, &state) +} + +type rewriteStepRunSecretsToEnvState struct { + seen map[string]struct{} + orderedBindings []string + bindingExprs map[string]string + firstRunLine int + envStart int + envEnd int + envIndent string + envKeyIndentLen int + existingEnvKeys map[string]struct{} +} + +func rewriteStepRunSecretsToEnvDetectPowerShell(stepLines []string, stepIndent string) bool { directKeyIndent := stepIndent + " " for _, line := range stepLines { trimmed := strings.TrimSpace(line) @@ -183,137 +215,132 @@ func rewriteStepRunSecretsToEnv(stepLines []string, stepIndent string) ([]string shellMatch, shellValue, _ := parseStepKeyLine(trimmed, indent, stepIndent, "shell") if shellMatch { v := strings.ToLower(strings.TrimSpace(shellValue)) - if v == "pwsh" || v == "powershell" { //nolint:tolowerequalfold - shellIsPowerShell = true - } - break + return v == "pwsh" || v == "powershell" //nolint:tolowerequalfold } } + return false +} - for i := 0; i < len(stepLines); i++ { - line := stepLines[i] - trimmed := strings.TrimSpace(line) - indent := getIndentation(line) +func rewriteStepRunSecretsToEnvProcessLine(stepLines []string, i int, stepIndent string, shellIsPowerShell bool, state *rewriteStepRunSecretsToEnvState) bool { + line := stepLines[i] + trimmed := strings.TrimSpace(line) + indent := getIndentation(line) - envMatch, envValue, currentEnvKeyIndentLen := parseStepKeyLine(trimmed, indent, stepIndent, "env") - if envMatch && envValue == "" { - envStart = i - envIndent = indent - envKeyIndentLen = currentEnvKeyIndentLen - envEnd = i - for j := i + 1; j < len(stepLines); j++ { - t := strings.TrimSpace(stepLines[j]) - if t == "" { - envEnd = j - continue - } - if effectiveStepLineIndentLen(t, getIndentation(stepLines[j]), stepIndent) <= envKeyIndentLen { - break - } - envEnd = j - key := parseYAMLMapKey(t) - if key != "" { - existingEnvKeys[key] = struct { - }{} - } - } - } + rewriteStepRunSecretsToEnvTrackEnvBlock(stepLines, i, stepIndent, trimmed, indent, state) + runMatch, runValue, runKeyIndentLen := parseStepKeyLine(trimmed, indent, stepIndent, "run") + if !runMatch { + return false + } + if state.firstRunLine == -1 { + state.firstRunLine = i + } + if runValue == "|" || runValue == "|-" || runValue == ">" || runValue == ">-" { + return rewriteStepRunSecretsToEnvProcessRunBlock(stepLines, i, stepIndent, runKeyIndentLen, shellIsPowerShell, state) + } - runMatch, runValue, runKeyIndentLen := parseStepKeyLine(trimmed, indent, stepIndent, "run") - if !runMatch { + newLine, bindings := replaceStepExpressionRefs(line, shellIsPowerShell, state.bindingExprs) + if len(bindings) == 0 { + return false + } + stepLines[i] = newLine + rewriteStepRunSecretsToEnvRegisterBindings(bindings, state) + return true +} + +func rewriteStepRunSecretsToEnvTrackEnvBlock(stepLines []string, i int, stepIndent, trimmed, indent string, state *rewriteStepRunSecretsToEnvState) { + envMatch, envValue, currentEnvKeyIndentLen := parseStepKeyLine(trimmed, indent, stepIndent, "env") + if !envMatch || envValue != "" { + return + } + state.envStart = i + state.envIndent = indent + state.envKeyIndentLen = currentEnvKeyIndentLen + state.envEnd = i + for j := i + 1; j < len(stepLines); j++ { + t := strings.TrimSpace(stepLines[j]) + if t == "" { + state.envEnd = j continue } - if firstRunLine == -1 { - firstRunLine = i + if effectiveStepLineIndentLen(t, getIndentation(stepLines[j]), stepIndent) <= state.envKeyIndentLen { + break } + state.envEnd = j + if key := parseYAMLMapKey(t); key != "" { + state.existingEnvKeys[key] = struct{}{} + } + } +} - if runValue == "|" || runValue == "|-" || runValue == ">" || runValue == ">-" { - for j := i + 1; j < len(stepLines); j++ { - t := strings.TrimSpace(stepLines[j]) - if t == "" { - continue - } - if effectiveStepLineIndentLen(t, getIndentation(stepLines[j]), stepIndent) <= runKeyIndentLen { - break - } - // Skip shell comment lines – expressions inside # comments are - // documentation-only and must not generate env bindings. - // NOTE: heredoc boundaries are not tracked; lines starting with - // '#' inside a heredoc body are also skipped (follow-up needed). - if strings.HasPrefix(t, "#") { - continue - } - updatedLine, bindings := replaceStepExpressionRefs(stepLines[j], shellIsPowerShell, bindingExprs) - if len(bindings) > 0 { - stepLines[j] = updatedLine - modified = true - } - for _, binding := range bindings { - if !setutil.Contains(seen, binding.Name) { - seen[binding.Name] = struct { - }{} - orderedBindings = append(orderedBindings, binding.Name) - bindingExprs[binding.Name] = binding.Expression - } - } - } +func rewriteStepRunSecretsToEnvProcessRunBlock(stepLines []string, i int, stepIndent string, runKeyIndentLen int, shellIsPowerShell bool, state *rewriteStepRunSecretsToEnvState) bool { + modified := false + for j := i + 1; j < len(stepLines); j++ { + t := strings.TrimSpace(stepLines[j]) + if t == "" { continue } - - newLine, bindings := replaceStepExpressionRefs(line, shellIsPowerShell, bindingExprs) + if effectiveStepLineIndentLen(t, getIndentation(stepLines[j]), stepIndent) <= runKeyIndentLen { + break + } + // Skip shell comment lines – expressions inside # comments are + // documentation-only and must not generate env bindings. + // NOTE: heredoc boundaries are not tracked; lines starting with + // '#' inside a heredoc body are also skipped (follow-up needed). + if strings.HasPrefix(t, "#") { + continue + } + updatedLine, bindings := replaceStepExpressionRefs(stepLines[j], shellIsPowerShell, state.bindingExprs) if len(bindings) > 0 { - stepLines[i] = newLine + stepLines[j] = updatedLine modified = true - } - for _, binding := range bindings { - if !setutil.Contains(seen, binding.Name) { - seen[binding.Name] = struct { - }{} - orderedBindings = append(orderedBindings, binding.Name) - bindingExprs[binding.Name] = binding.Expression - } + rewriteStepRunSecretsToEnvRegisterBindings(bindings, state) } } + return modified +} - if len(orderedBindings) == 0 { - return stepLines, modified +func rewriteStepRunSecretsToEnvRegisterBindings(bindings []stepExpressionBinding, state *rewriteStepRunSecretsToEnvState) { + for _, binding := range bindings { + if !setutil.Contains(state.seen, binding.Name) { + state.seen[binding.Name] = struct{}{} + state.orderedBindings = append(state.orderedBindings, binding.Name) + state.bindingExprs[binding.Name] = binding.Expression + } } +} - stepsRunSecretsEnvCodemodLog.Printf("Found %d unique run expression references in step run commands", len(orderedBindings)) - - missingBindings := make([]string, 0, len(orderedBindings)) - for _, name := range orderedBindings { - if !setutil.Contains(existingEnvKeys, name) { +func rewriteStepRunSecretsToEnvMissingBindings(state *rewriteStepRunSecretsToEnvState) []string { + missingBindings := make([]string, 0, len(state.orderedBindings)) + for _, name := range state.orderedBindings { + if !setutil.Contains(state.existingEnvKeys, name) { missingBindings = append(missingBindings, name) } } - if len(missingBindings) == 0 { - return stepLines, true - } - - stepsRunSecretsEnvCodemodLog.Printf("Adding env bindings for %d missing expressions: %v", len(missingBindings), missingBindings) + return missingBindings +} - if envStart != -1 { - insertAt := envEnd + 1 - envValueIndent := envIndent + " " +func rewriteStepRunSecretsToEnvInsertMissingBindings(stepLines []string, stepIndent string, missingBindings []string, modified bool, state *rewriteStepRunSecretsToEnvState) ([]string, bool) { + if state.envStart != -1 { + insertAt := state.envEnd + 1 + envValueIndent := state.envIndent + " " insertLines := make([]string, 0, len(missingBindings)) for _, name := range missingBindings { - insertLines = append(insertLines, fmt.Sprintf("%s%s: %s", envValueIndent, name, bindingExprs[name])) + insertLines = append(insertLines, fmt.Sprintf("%s%s: %s", envValueIndent, name, state.bindingExprs[name])) } stepLines = append(stepLines[:insertAt], append(insertLines, stepLines[insertAt:]...)...) return stepLines, true } - if firstRunLine == -1 { + if state.firstRunLine == -1 { return stepLines, modified } insertIndent := stepIndent + " " insertLines := []string{insertIndent + "env:"} for _, name := range missingBindings { - insertLines = append(insertLines, fmt.Sprintf("%s %s: %s", insertIndent, name, bindingExprs[name])) + insertLines = append(insertLines, fmt.Sprintf("%s %s: %s", insertIndent, name, state.bindingExprs[name])) } - stepLines = append(stepLines[:firstRunLine], append(insertLines, stepLines[firstRunLine:]...)...) + stepLines = append(stepLines[:state.firstRunLine], append(insertLines, stepLines[state.firstRunLine:]...)...) return stepLines, true } @@ -330,17 +357,12 @@ func replaceStepExpressionRefs(line string, shellIsPowerShell bool, existingBind var result strings.Builder last := 0 - // bodyToName maps expression body → assigned env-var name for same-body dedup - // within this line (avoids re-computing the name for repeated occurrences). - bodyToName := make(map[string]string) - // localNames maps env-var name → canonical expression for within-line - // collision detection (two different bodies that sanitize to the same name). - localNames := make(map[string]string) - // registeredNames tracks which names already appear in ordered, so we never - // add a duplicate binding entry. - registeredNames := make(map[string]struct { - }) - ordered := make([]stepExpressionBinding, 0, len(matches)) + state := replaceStepExpressionRefsState{ + bodyToName: make(map[string]string), + localNames: make(map[string]string), + registeredNames: make(map[string]struct{}), + ordered: make([]stepExpressionBinding, 0, len(matches)), + } for _, match := range matches { if len(match) < 4 { @@ -353,55 +375,77 @@ func replaceStepExpressionRefs(line string, shellIsPowerShell bool, existingBind result.WriteString(line[last:fullStart]) - // Same expression body already resolved in this line – reuse the name. - if cachedName, done := bodyToName[body]; done { - if shellIsPowerShell { - result.WriteString("$env:" + cachedName) - } else { - result.WriteString("$" + cachedName) - } + if cachedName, done := state.bodyToName[body]; done { + result.WriteString(replaceStepExpressionRefsVarRef(cachedName, shellIsPowerShell)) last = fullEnd continue } - envName, canonicalExpression, ok := mapRunExpressionToEnvBinding(body) + envName, canonicalExpression, ok := replaceStepExpressionRefsBinding(body, existingBindings, &state) if !ok { result.WriteString(fullExpression) last = fullEnd continue } - // Collision guard: if this env-var name is already bound to a *different* - // expression (from a previous line in this step via existingBindings, or - // from an earlier occurrence within this line via localNames), fall back - // to a hash-based name so both expressions receive unique bindings. - if crossLine := existingBindings[envName]; (crossLine != "" && crossLine != canonicalExpression) || - (localNames[envName] != "" && localNames[envName] != canonicalExpression) { - envName = hashedBindingName("EXPR", body) - canonicalExpression = fmt.Sprintf("${{ %s }}", body) - } - - bodyToName[body] = envName - localNames[envName] = canonicalExpression - - if shellIsPowerShell { - result.WriteString("$env:" + envName) - } else { - result.WriteString("$" + envName) - } - if !setutil.Contains(registeredNames, envName) { - registeredNames[envName] = struct { - }{} - ordered = append(ordered, stepExpressionBinding{ - Name: envName, - Expression: canonicalExpression, - }) - } + state.bodyToName[body] = envName + state.localNames[envName] = canonicalExpression + result.WriteString(replaceStepExpressionRefsVarRef(envName, shellIsPowerShell)) + replaceStepExpressionRefsRegisterBinding(envName, canonicalExpression, &state) last = fullEnd } result.WriteString(line[last:]) - return result.String(), ordered + return result.String(), state.ordered +} + +type replaceStepExpressionRefsState struct { + // bodyToName maps expression body → assigned env-var name for same-body dedup + // within this line (avoids re-computing the name for repeated occurrences). + bodyToName map[string]string + // localNames maps env-var name → canonical expression for within-line + // collision detection (two different bodies that sanitize to the same name). + localNames map[string]string + // registeredNames tracks which names already appear in ordered, so we never + // add a duplicate binding entry. + registeredNames map[string]struct{} + ordered []stepExpressionBinding +} + +func replaceStepExpressionRefsBinding(body string, existingBindings map[string]string, state *replaceStepExpressionRefsState) (string, string, bool) { + envName, canonicalExpression, ok := mapRunExpressionToEnvBinding(body) + if !ok { + return "", "", false + } + + // Collision guard: if this env-var name is already bound to a *different* + // expression (from a previous line in this step via existingBindings, or + // from an earlier occurrence within this line via localNames), fall back + // to a hash-based name so both expressions receive unique bindings. + if crossLine := existingBindings[envName]; (crossLine != "" && crossLine != canonicalExpression) || + (state.localNames[envName] != "" && state.localNames[envName] != canonicalExpression) { + envName = hashedBindingName("EXPR", body) + canonicalExpression = fmt.Sprintf("${{ %s }}", body) + } + return envName, canonicalExpression, true +} + +func replaceStepExpressionRefsVarRef(envName string, shellIsPowerShell bool) string { + if shellIsPowerShell { + return "$env:" + envName + } + return "$" + envName +} + +func replaceStepExpressionRefsRegisterBinding(envName, canonicalExpression string, state *replaceStepExpressionRefsState) { + if setutil.Contains(state.registeredNames, envName) { + return + } + state.registeredNames[envName] = struct{}{} + state.ordered = append(state.ordered, stepExpressionBinding{ + Name: envName, + Expression: canonicalExpression, + }) } func mapRunExpressionToEnvBinding(body string) (string, string, bool) { diff --git a/pkg/cli/codemod_upload_assets.go b/pkg/cli/codemod_upload_assets.go index 1d8922de291..84b959140aa 100644 --- a/pkg/cli/codemod_upload_assets.go +++ b/pkg/cli/codemod_upload_assets.go @@ -15,67 +15,74 @@ func getUploadAssetsCodemod() Codemod { Name: "Migrate upload-assets to upload-asset", Description: "Replaces deprecated 'safe-outputs.upload-assets' field with 'safe-outputs.upload-asset' (plural to singular)", IntroducedIn: "0.3.0", - Apply: func(content string, frontmatter map[string]any) (string, bool, error) { - // Check if safe-outputs.upload-assets exists - safeOutputsValue, hasSafeOutputs := frontmatter["safe-outputs"] - if !hasSafeOutputs { - return content, false, nil - } + Apply: getUploadAssetsCodemodApply, + } +} - safeOutputsMap, ok := safeOutputsValue.(map[string]any) - if !ok { - return content, false, nil - } +func getUploadAssetsCodemodApply(content string, frontmatter map[string]any) (string, bool, error) { + if !getUploadAssetsCodemodNeedsMigration(frontmatter) { + return content, false, nil + } - // Check if upload-assets field exists in safe-outputs (plural is deprecated) - _, hasUploadAssets := safeOutputsMap["upload-assets"] - if !hasUploadAssets { - return content, false, nil - } + newContent, applied, err := applyFrontmatterLineTransform(content, getUploadAssetsCodemodTransform) + if applied { + uploadAssetsCodemodLog.Print("Applied upload-assets to upload-asset migration") + } + return newContent, applied, err +} - newContent, applied, err := applyFrontmatterLineTransform(content, func(lines []string) ([]string, bool) { - var modified bool - var inSafeOutputsBlock bool - var safeOutputsIndent string - result := make([]string, len(lines)) - for i, line := range lines { - trimmedLine := strings.TrimSpace(line) +func getUploadAssetsCodemodNeedsMigration(frontmatter map[string]any) bool { + // Check if safe-outputs.upload-assets exists + safeOutputsValue, hasSafeOutputs := frontmatter["safe-outputs"] + if !hasSafeOutputs { + return false + } - // Track if we're in the safe-outputs block - if strings.HasPrefix(trimmedLine, "safe-outputs:") { - inSafeOutputsBlock = true - safeOutputsIndent = getIndentation(line) - result[i] = line - continue - } + safeOutputsMap, ok := safeOutputsValue.(map[string]any) + if !ok { + return false + } - // Check if we've left the safe-outputs block - if inSafeOutputsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, safeOutputsIndent) { - inSafeOutputsBlock = false - } - } + // Check if upload-assets field exists in safe-outputs (plural is deprecated) + _, hasUploadAssets := safeOutputsMap["upload-assets"] + return hasUploadAssets +} - // Replace upload-assets with upload-asset if in safe-outputs block - if inSafeOutputsBlock && strings.HasPrefix(trimmedLine, "upload-assets:") { - replacedLine, didReplace := findAndReplaceInLine(line, "upload-assets", "upload-asset") - if didReplace { - result[i] = replacedLine - modified = true - uploadAssetsCodemodLog.Printf("Replaced safe-outputs.upload-assets with safe-outputs.upload-asset on line %d", i+1) - } else { - result[i] = line - } - } else { - result[i] = line - } - } - return result, modified - }) - if applied { - uploadAssetsCodemodLog.Print("Applied upload-assets to upload-asset migration") - } - return newContent, applied, err - }, +func getUploadAssetsCodemodTransform(lines []string) ([]string, bool) { + var modified bool + var inSafeOutputsBlock bool + var safeOutputsIndent string + result := make([]string, len(lines)) + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) + inSafeOutputsBlock, safeOutputsIndent = getUploadAssetsCodemodTrackSafeOutputsBlock(line, trimmedLine, inSafeOutputsBlock, safeOutputsIndent) + result[i], modified = getUploadAssetsCodemodReplaceLine(line, trimmedLine, inSafeOutputsBlock, i, modified) + } + return result, modified +} + +func getUploadAssetsCodemodTrackSafeOutputsBlock(line, trimmedLine string, inSafeOutputsBlock bool, safeOutputsIndent string) (bool, string) { + // Track if we're in the safe-outputs block + if strings.HasPrefix(trimmedLine, "safe-outputs:") { + return true, getIndentation(line) + } + + // Check if we've left the safe-outputs block + if inSafeOutputsBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") && hasExitedBlock(line, safeOutputsIndent) { + return false, safeOutputsIndent + } + return inSafeOutputsBlock, safeOutputsIndent +} + +func getUploadAssetsCodemodReplaceLine(line, trimmedLine string, inSafeOutputsBlock bool, lineIndex int, modified bool) (string, bool) { + // Replace upload-assets with upload-asset if in safe-outputs block + if !inSafeOutputsBlock || !strings.HasPrefix(trimmedLine, "upload-assets:") { + return line, modified + } + replacedLine, didReplace := findAndReplaceInLine(line, "upload-assets", "upload-asset") + if didReplace { + uploadAssetsCodemodLog.Printf("Replaced safe-outputs.upload-assets with safe-outputs.upload-asset on line %d", lineIndex+1) + return replacedLine, true } + return line, modified } diff --git a/pkg/cli/codemod_user_rate_limit.go b/pkg/cli/codemod_user_rate_limit.go index 2afbcc9f93f..a8f0db4715a 100644 --- a/pkg/cli/codemod_user_rate_limit.go +++ b/pkg/cli/codemod_user_rate_limit.go @@ -38,10 +38,7 @@ func getRateLimitToUserRateLimitCodemod() Codemod { func renameRateLimitToUserRateLimit(lines []string) ([]string, bool) { var result []string modified := false - - inUserRateLimit := false - userRateLimitIndent := "" - userRateLimitChildIndent := "" + state := renameRateLimitToUserRateLimitState{} for i, line := range lines { trimmed := strings.TrimSpace(line) @@ -51,54 +48,26 @@ func renameRateLimitToUserRateLimit(lines []string) ([]string, bool) { continue } - if !strings.HasPrefix(trimmed, "#") && inUserRateLimit && hasExitedBlock(line, userRateLimitIndent) { - inUserRateLimit = false - userRateLimitChildIndent = "" - } + renameRateLimitToUserRateLimitUpdateBlockState(line, trimmed, &state) if isTopLevelKey(line) && strings.HasPrefix(trimmed, "rate-limit:") { - lineIndent := getIndentation(line) - newLine, replaced := findAndReplaceInLine(line, "rate-limit", "user-rate-limit") - if replaced { + if newLine, replaced := renameRateLimitToUserRateLimitTopLevel(line, i, &state); replaced { result = append(result, newLine) modified = true - inUserRateLimit = true - userRateLimitIndent = lineIndent - userRateLimitChildIndent = "" - userRateLimitCodemodLog.Printf("Renamed 'rate-limit' to 'user-rate-limit' on line %d", i+1) continue } } if isTopLevelKey(line) && strings.HasPrefix(trimmed, "user-rate-limit:") { - inUserRateLimit = true - userRateLimitIndent = getIndentation(line) - userRateLimitChildIndent = "" + renameRateLimitToUserRateLimitEnterBlock(getIndentation(line), &state) result = append(result, line) continue } - if inUserRateLimit { - lineIndent := getIndentation(line) - if isDescendant(lineIndent, userRateLimitIndent) { - if trimmed != "" && !strings.HasPrefix(trimmed, "#") && userRateLimitChildIndent == "" { - userRateLimitChildIndent = lineIndent - } - if userRateLimitChildIndent != "" && lineIndent != userRateLimitChildIndent { - result = append(result, line) - continue - } - newLine, replaced := findAndReplaceInLine(line, "max-runs", "max-runs-per-window") - if !replaced { - newLine, replaced = findAndReplaceInLine(line, "max", "max-runs-per-window") - } - if replaced { - result = append(result, newLine) - modified = true - userRateLimitCodemodLog.Printf("Renamed max field to 'max-runs-per-window' on line %d", i+1) - continue - } - } + if newLine, replaced := renameRateLimitToUserRateLimitNestedLine(line, trimmed, i, &state); replaced { + result = append(result, newLine) + modified = true + continue } result = append(result, line) @@ -106,3 +75,55 @@ func renameRateLimitToUserRateLimit(lines []string) ([]string, bool) { return result, modified } + +type renameRateLimitToUserRateLimitState struct { + inUserRateLimit bool + userRateLimitIndent string + userRateLimitChildIndent string +} + +func renameRateLimitToUserRateLimitUpdateBlockState(line, trimmed string, state *renameRateLimitToUserRateLimitState) { + if !strings.HasPrefix(trimmed, "#") && state.inUserRateLimit && hasExitedBlock(line, state.userRateLimitIndent) { + state.inUserRateLimit = false + state.userRateLimitChildIndent = "" + } +} + +func renameRateLimitToUserRateLimitEnterBlock(indent string, state *renameRateLimitToUserRateLimitState) { + state.inUserRateLimit = true + state.userRateLimitIndent = indent + state.userRateLimitChildIndent = "" +} + +func renameRateLimitToUserRateLimitTopLevel(line string, lineIndex int, state *renameRateLimitToUserRateLimitState) (string, bool) { + newLine, replaced := findAndReplaceInLine(line, "rate-limit", "user-rate-limit") + if replaced { + renameRateLimitToUserRateLimitEnterBlock(getIndentation(line), state) + userRateLimitCodemodLog.Printf("Renamed 'rate-limit' to 'user-rate-limit' on line %d", lineIndex+1) + } + return newLine, replaced +} + +func renameRateLimitToUserRateLimitNestedLine(line, trimmed string, lineIndex int, state *renameRateLimitToUserRateLimitState) (string, bool) { + if !state.inUserRateLimit { + return "", false + } + lineIndent := getIndentation(line) + if !isDescendant(lineIndent, state.userRateLimitIndent) { + return "", false + } + if trimmed != "" && !strings.HasPrefix(trimmed, "#") && state.userRateLimitChildIndent == "" { + state.userRateLimitChildIndent = lineIndent + } + if state.userRateLimitChildIndent != "" && lineIndent != state.userRateLimitChildIndent { + return "", false + } + newLine, replaced := findAndReplaceInLine(line, "max-runs", "max-runs-per-window") + if !replaced { + newLine, replaced = findAndReplaceInLine(line, "max", "max-runs-per-window") + } + if replaced { + userRateLimitCodemodLog.Printf("Renamed max field to 'max-runs-per-window' on line %d", lineIndex+1) + } + return newLine, replaced +} diff --git a/pkg/cli/codemod_workflow_dispatch_required.go b/pkg/cli/codemod_workflow_dispatch_required.go index d1cfc24dbd2..13b17227451 100644 --- a/pkg/cli/codemod_workflow_dispatch_required.go +++ b/pkg/cli/codemod_workflow_dispatch_required.go @@ -77,15 +77,7 @@ func getWorkflowDispatchRequiredFalseCodemod() Codemod { func rewriteWorkflowDispatchRequiredFalse(lines []string) ([]string, bool) { result := make([]string, 0, len(lines)) modified := false - - inOn := false - onIndent := "" - inWD := false - wdIndent := "" - inInputs := false - inputsIndent := "" - inInputEntry := false - inputEntryIndent := "" + state := rewriteWorkflowDispatchRequiredFalseState{} for _, line := range lines { trimmed := strings.TrimSpace(line) @@ -97,93 +89,43 @@ func rewriteWorkflowDispatchRequiredFalse(lines []string) ([]string, bool) { continue } - // Exit deeper states first (order matters: deepest → shallowest). - if inInputEntry && len(indent) <= len(inputEntryIndent) { - inInputEntry = false - } - if inInputs && len(indent) <= len(inputsIndent) { - inInputs = false - inInputEntry = false - } - if inWD && len(indent) <= len(wdIndent) { - inWD = false - inInputs = false - inInputEntry = false - } - if inOn && len(indent) <= len(onIndent) { - inOn = false - inWD = false - inInputs = false - inInputEntry = false - } + rewriteWorkflowDispatchRequiredFalseExitBlocks(indent, &state) // Enter on: block. if isTopLevelKey(line) && strings.HasPrefix(trimmed, "on:") { - inOn = true - onIndent = indent + state.inOn = true + state.onIndent = indent result = append(result, line) continue } - // Enter workflow_dispatch: within on:. - if inOn && !inWD && strings.HasPrefix(trimmed, "workflow_dispatch:") { - inWD = true - wdIndent = indent + if rewriteWorkflowDispatchRequiredFalseEnterWorkflowDispatch(trimmed, indent, &state) { result = append(result, line) continue } - // Enter inputs: within workflow_dispatch: (allow trailing comments). - if inWD && !inInputs && strings.HasPrefix(trimmed, "inputs:") { - remainder := strings.TrimSpace(strings.TrimPrefix(trimmed, "inputs:")) - if remainder == "" || strings.HasPrefix(remainder, "#") { - inInputs = true - inputsIndent = indent - result = append(result, line) - continue - } + if rewriteWorkflowDispatchRequiredFalseEnterInputs(trimmed, indent, &state) { + result = append(result, line) + continue } - // Handle inline inputs maps (for example: "inputs: { pr_number: { required: true } }"). - if inWD && strings.HasPrefix(trimmed, "inputs:") && strings.Contains(trimmed, "required: true") { - newLine := strings.ReplaceAll(line, "required: true", "required: false") - if newLine != line { - result = append(result, newLine) - modified = true - workflowDispatchRequiredLog.Print("Rewrote inline workflow_dispatch input required: true to required: false") - continue - } + if newLine, replaced := rewriteWorkflowDispatchRequiredFalseInlineInputs(line, trimmed, &state); replaced { + result = append(result, newLine) + modified = true + continue } - // Enter an individual input entry and handle inline input maps - // (for example: "pr_number: { required: true }"). - if inInputs && !inInputEntry && len(indent) > len(inputsIndent) { - if strings.Contains(trimmed, "{") && strings.Contains(trimmed, "required: true") { - newLine := strings.ReplaceAll(line, "required: true", "required: false") - if newLine != line { - result = append(result, newLine) - modified = true - workflowDispatchRequiredLog.Print("Rewrote inline input entry required: true to required: false") - continue - } - } - inInputEntry = true - inputEntryIndent = indent - result = append(result, line) + newLine, handled, changed := rewriteWorkflowDispatchRequiredFalseInputEntry(line, trimmed, indent, &state) + if handled { + result = append(result, newLine) + modified = modified || changed continue } - // Within an input entry's properties: rewrite "required: true" → "required: false". - if inInputEntry && len(indent) > len(inputEntryIndent) { - if strings.HasPrefix(trimmed, "required: true") { - newLine := strings.Replace(line, "required: true", "required: false", 1) - if newLine != line { - result = append(result, newLine) - modified = true - workflowDispatchRequiredLog.Print("Rewrote workflow_dispatch input required: true to required: false") - continue - } - } + if newLine, replaced := rewriteWorkflowDispatchRequiredFalseInputRequired(line, trimmed, indent, &state); replaced { + result = append(result, newLine) + modified = true + continue } result = append(result, line) @@ -191,3 +133,104 @@ func rewriteWorkflowDispatchRequiredFalse(lines []string) ([]string, bool) { return result, modified } + +type rewriteWorkflowDispatchRequiredFalseState struct { + inOn bool + onIndent string + inWD bool + wdIndent string + inInputs bool + inputsIndent string + inInputEntry bool + inputEntryIndent string +} + +func rewriteWorkflowDispatchRequiredFalseExitBlocks(indent string, state *rewriteWorkflowDispatchRequiredFalseState) { + // Exit deeper states first (order matters: deepest → shallowest). + if state.inInputEntry && len(indent) <= len(state.inputEntryIndent) { + state.inInputEntry = false + } + if state.inInputs && len(indent) <= len(state.inputsIndent) { + state.inInputs = false + state.inInputEntry = false + } + if state.inWD && len(indent) <= len(state.wdIndent) { + state.inWD = false + state.inInputs = false + state.inInputEntry = false + } + if state.inOn && len(indent) <= len(state.onIndent) { + state.inOn = false + state.inWD = false + state.inInputs = false + state.inInputEntry = false + } +} + +func rewriteWorkflowDispatchRequiredFalseEnterWorkflowDispatch(trimmed, indent string, state *rewriteWorkflowDispatchRequiredFalseState) bool { + // Enter workflow_dispatch: within on:. + if !state.inOn || state.inWD || !strings.HasPrefix(trimmed, "workflow_dispatch:") { + return false + } + state.inWD = true + state.wdIndent = indent + return true +} + +func rewriteWorkflowDispatchRequiredFalseEnterInputs(trimmed, indent string, state *rewriteWorkflowDispatchRequiredFalseState) bool { + // Enter inputs: within workflow_dispatch: (allow trailing comments). + if !state.inWD || state.inInputs || !strings.HasPrefix(trimmed, "inputs:") { + return false + } + remainder := strings.TrimSpace(strings.TrimPrefix(trimmed, "inputs:")) + if remainder != "" && !strings.HasPrefix(remainder, "#") { + return false + } + state.inInputs = true + state.inputsIndent = indent + return true +} + +func rewriteWorkflowDispatchRequiredFalseInlineInputs(line, trimmed string, state *rewriteWorkflowDispatchRequiredFalseState) (string, bool) { + // Handle inline inputs maps (for example: "inputs: { pr_number: { required: true } }"). + if !state.inWD || !strings.HasPrefix(trimmed, "inputs:") || !strings.Contains(trimmed, "required: true") { + return "", false + } + newLine := strings.ReplaceAll(line, "required: true", "required: false") + if newLine == line { + return "", false + } + workflowDispatchRequiredLog.Print("Rewrote inline workflow_dispatch input required: true to required: false") + return newLine, true +} + +func rewriteWorkflowDispatchRequiredFalseInputEntry(line, trimmed, indent string, state *rewriteWorkflowDispatchRequiredFalseState) (string, bool, bool) { + // Enter an individual input entry and handle inline input maps + // (for example: "pr_number: { required: true }"). + if !state.inInputs || state.inInputEntry || len(indent) <= len(state.inputsIndent) { + return "", false, false + } + if strings.Contains(trimmed, "{") && strings.Contains(trimmed, "required: true") { + newLine := strings.ReplaceAll(line, "required: true", "required: false") + if newLine != line { + workflowDispatchRequiredLog.Print("Rewrote inline input entry required: true to required: false") + return newLine, true, true + } + } + state.inInputEntry = true + state.inputEntryIndent = indent + return line, true, false +} + +func rewriteWorkflowDispatchRequiredFalseInputRequired(line, trimmed, indent string, state *rewriteWorkflowDispatchRequiredFalseState) (string, bool) { + // Within an input entry's properties: rewrite "required: true" → "required: false". + if !state.inInputEntry || len(indent) <= len(state.inputEntryIndent) || !strings.HasPrefix(trimmed, "required: true") { + return "", false + } + newLine := strings.Replace(line, "required: true", "required: false", 1) + if newLine == line { + return "", false + } + workflowDispatchRequiredLog.Print("Rewrote workflow_dispatch input required: true to required: false") + return newLine, true +} diff --git a/pkg/cli/codemod_workflow_run_branches.go b/pkg/cli/codemod_workflow_run_branches.go index ce10f85a8e6..7a6c3b35847 100644 --- a/pkg/cli/codemod_workflow_run_branches.go +++ b/pkg/cli/codemod_workflow_run_branches.go @@ -88,6 +88,37 @@ func getWorkflowRunBranchesCodemodWithDeps(deps workflowRunBranchesCodemodDeps) } func addWorkflowRunBranches(lines []string, branches []string) ([]string, bool) { + onIdx, onIndent, onEnd := addWorkflowRunBranchesFindOnBlock(lines) + if onIdx == -1 { + return lines, false + } + + workflowRunIdx, workflowRunIndent, workflowRunEnd := addWorkflowRunBranchesFindWorkflowRunBlock(lines, onIdx, onIndent, onEnd) + if workflowRunIdx == -1 { + return lines, false + } + + for i := workflowRunIdx + 1; i < workflowRunEnd; i++ { + if strings.HasPrefix(strings.TrimSpace(lines[i]), "branches:") { + return lines, false + } + } + + branchIndent := workflowRunIndent + " " + entries := make([]string, 0, len(branches)+1) + entries = append(entries, branchIndent+"branches:") + for _, branch := range branches { + entries = append(entries, branchIndent+" - "+branch) + } + + result := make([]string, 0, len(lines)+len(entries)) + result = append(result, lines[:workflowRunEnd]...) + result = append(result, entries...) + result = append(result, lines[workflowRunEnd:]...) + return result, true +} + +func addWorkflowRunBranchesFindOnBlock(lines []string) (int, string, int) { onIdx := -1 onIndent := "" onEnd := len(lines) @@ -104,10 +135,10 @@ func addWorkflowRunBranches(lines []string, branches []string) ([]string, bool) break } } - if onIdx == -1 { - return lines, false - } + return onIdx, onIndent, onEnd +} +func addWorkflowRunBranchesFindWorkflowRunBlock(lines []string, onIdx int, onIndent string, onEnd int) (int, string, int) { workflowRunIdx := -1 workflowRunIndent := "" workflowRunEnd := onEnd @@ -116,53 +147,35 @@ func addWorkflowRunBranches(lines []string, branches []string) ([]string, bool) if trimmed == "" || strings.HasPrefix(trimmed, "#") { continue } - if len(getIndentation(lines[i])) <= len(onIndent) { break } - if strings.HasPrefix(trimmed, "workflow_run:") { if strings.Contains(trimmed, "{") { - return lines, false + return -1, "", onEnd } workflowRunIdx = i workflowRunIndent = getIndentation(lines[i]) - for j := i + 1; j < onEnd; j++ { - innerTrimmed := strings.TrimSpace(lines[j]) - if innerTrimmed == "" || strings.HasPrefix(innerTrimmed, "#") { - continue - } - if len(getIndentation(lines[j])) <= len(workflowRunIndent) { - workflowRunEnd = j - break - } - } + workflowRunEnd = addWorkflowRunBranchesFindWorkflowRunEnd(lines, i, workflowRunIndent, onEnd) break } } + return workflowRunIdx, workflowRunIndent, workflowRunEnd +} - if workflowRunIdx == -1 { - return lines, false - } - - for i := workflowRunIdx + 1; i < workflowRunEnd; i++ { - if strings.HasPrefix(strings.TrimSpace(lines[i]), "branches:") { - return lines, false +func addWorkflowRunBranchesFindWorkflowRunEnd(lines []string, workflowRunIdx int, workflowRunIndent string, onEnd int) int { + workflowRunEnd := onEnd + for j := workflowRunIdx + 1; j < onEnd; j++ { + innerTrimmed := strings.TrimSpace(lines[j]) + if innerTrimmed == "" || strings.HasPrefix(innerTrimmed, "#") { + continue + } + if len(getIndentation(lines[j])) <= len(workflowRunIndent) { + workflowRunEnd = j + break } } - - branchIndent := workflowRunIndent + " " - entries := make([]string, 0, len(branches)+1) - entries = append(entries, branchIndent+"branches:") - for _, branch := range branches { - entries = append(entries, branchIndent+" - "+branch) - } - - result := make([]string, 0, len(lines)+len(entries)) - result = append(result, lines[:workflowRunEnd]...) - result = append(result, entries...) - result = append(result, lines[workflowRunEnd:]...) - return result, true + return workflowRunEnd } func normalizeWorkflowRunBranches(branches []string) []string { diff --git a/pkg/cli/commands.go b/pkg/cli/commands.go index 9e7d12407f0..61b9ea1aeec 100644 --- a/pkg/cli/commands.go +++ b/pkg/cli/commands.go @@ -67,10 +67,7 @@ func CreateWorkflowMarkdownFile(workflowName string, verbose bool, force bool, e } // Create the template content - template := createWorkflowTemplate(workflowName, engine) - - // Write the template to file with restrictive permissions (owner-only) - if err := os.WriteFile(destFile, []byte(template), constants.FilePermSensitive); err != nil { + if err := createWorkflowMarkdownFileWriteTemplate(destFile, workflowName, engine); err != nil { return fmt.Errorf("failed to write workflow file '%s': %w", destFile, err) } @@ -80,6 +77,12 @@ func CreateWorkflowMarkdownFile(workflowName string, verbose bool, force bool, e return nil } +func createWorkflowMarkdownFileWriteTemplate(destFile, workflowName, engine string) error { + template := createWorkflowTemplate(workflowName, engine) + // Write the template to file with restrictive permissions (owner-only) + return os.WriteFile(destFile, []byte(template), constants.FilePermSensitive) +} + // createWorkflowTemplate generates a concise workflow template with essential options func createWorkflowTemplate(workflowName string, engine string) string { engineLine := "" diff --git a/pkg/cli/compile_compiler_setup.go b/pkg/cli/compile_compiler_setup.go index 66491b6a3e2..7ad9fb610a6 100644 --- a/pkg/cli/compile_compiler_setup.go +++ b/pkg/cli/compile_compiler_setup.go @@ -131,6 +131,14 @@ func createAndConfigureCompiler(config CompileConfig) *workflow.Compiler { func configureCompilerFlags(compiler *workflow.Compiler, config CompileConfig) { compileCompilerSetupLog.Print("Configuring compiler flags") + configureCompilerFlagsValidation(compiler, config) + configureCompilerFlagsExecutionModes(compiler, config) + configureCompilerFlagsRefreshAndApprove(compiler, config) + configureCompilerFlagsImageAndGHES(compiler, config) + configureCompilerFlagsPriorManifest(compiler, config) +} + +func configureCompilerFlagsValidation(compiler *workflow.Compiler, config CompileConfig) { // Set validation based on the validate flag (false by default for compatibility) compiler.SetSkipValidation(!config.Validate) compileCompilerSetupLog.Printf("Validation enabled: %v", config.Validate) @@ -145,7 +153,9 @@ func configureCompilerFlags(compiler *workflow.Compiler, config CompileConfig) { compiler.SetStrictMode(config.Strict) compiler.SetAllowActionRefs(config.AllowActionRefs) compiler.SetForceStaged(config.Staged) +} +func configureCompilerFlagsExecutionModes(compiler *workflow.Compiler, config CompileConfig) { // Set trial mode if specified if config.TrialMode { compileCompilerSetupLog.Printf("Enabling trial mode: repoSlug=%s", config.TrialLogicalRepoSlug) @@ -160,7 +170,9 @@ func configureCompilerFlags(compiler *workflow.Compiler, config CompileConfig) { compileCompilerSetupLog.Print("Enabling --use-samples: agentic step will be replaced by a deterministic replay driver") compiler.SetUseSamples(true) } +} +func configureCompilerFlagsRefreshAndApprove(compiler *workflow.Compiler, config CompileConfig) { // Set refresh stop time flag compiler.SetRefreshStopTime(config.RefreshStopTime) if config.RefreshStopTime { @@ -179,7 +191,9 @@ func configureCompilerFlags(compiler *workflow.Compiler, config CompileConfig) { if config.Approve { compileCompilerSetupLog.Print("Safe update changes approved via --approve flag: skipping safe update enforcement for new restricted secrets or unapproved action additions/removals") } +} +func configureCompilerFlagsImageAndGHES(compiler *workflow.Compiler, config CompileConfig) { // Set require docker flag: when set, container image validation fails instead of // silently skipping when Docker is not available. compiler.SetRequireDocker(config.ValidateImages) @@ -193,7 +207,9 @@ func configureCompilerFlags(compiler *workflow.Compiler, config CompileConfig) { if config.GHESCompat { compileCompilerSetupLog.Print("GHES compatibility mode enabled via --ghes flag: artifact actions will use latest non-v3 pins") } +} +func configureCompilerFlagsPriorManifest(compiler *workflow.Compiler, config CompileConfig) { // Load pre-cached manifests from file (written by MCP server at startup). // These take precedence over git HEAD / filesystem reads for safe update enforcement. if config.PriorManifestFile != "" { diff --git a/pkg/cli/compile_file_operations.go b/pkg/cli/compile_file_operations.go index bf0fa1c42ff..222ccd3fb1e 100644 --- a/pkg/cli/compile_file_operations.go +++ b/pkg/cli/compile_file_operations.go @@ -148,6 +148,33 @@ func compileModifiedFilesWithDependencies(ctx context.Context, compiler *workflo console.ClearScreen() // Use dependency graph to determine what needs to be recompiled + workflowsToCompile := compileModifiedFilesWithDependenciesAffectedWorkflows(depGraph, compiler, files) + + fmt.Fprintln(os.Stderr, console.FormatProgressMessage("Watching for file changes")) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Recompiling %d workflow(s) affected by %d change(s)...", len(workflowsToCompile), len(files)))) + } + + // Reset warning count before compilation + compiler.ResetWarningCount() + + // Track compilation statistics + stats := &CompilationStats{} + + for _, file := range workflowsToCompile { + compileSingleFile(ctx, compiler, file, stats, verbose, true) + } + + // Get warning count from compiler + stats.Warnings = compiler.GetWarningCount() + + compileModifiedFilesWithDependenciesSaveArtifacts(compiler, stats, verbose) + + // Print summary instead of just "Recompiled" + printCompilationSummary(stats, false) +} + +func compileModifiedFilesWithDependenciesAffectedWorkflows(depGraph *DependencyGraph, compiler *workflow.Compiler, files []string) []string { var workflowsToCompile []string uniqueWorkflows := make(map[string]struct { }) @@ -174,24 +201,10 @@ func compileModifiedFilesWithDependencies(ctx context.Context, compiler *workflo } } - fmt.Fprintln(os.Stderr, console.FormatProgressMessage("Watching for file changes")) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Recompiling %d workflow(s) affected by %d change(s)...", len(workflowsToCompile), len(files)))) - } - - // Reset warning count before compilation - compiler.ResetWarningCount() - - // Track compilation statistics - stats := &CompilationStats{} - - for _, file := range workflowsToCompile { - compileSingleFile(ctx, compiler, file, stats, verbose, true) - } - - // Get warning count from compiler - stats.Warnings = compiler.GetWarningCount() + return workflowsToCompile +} +func compileModifiedFilesWithDependenciesSaveArtifacts(compiler *workflow.Compiler, stats *CompilationStats, verbose bool) { // Save the action cache after compilations actionCache := compiler.GetSharedActionCache() hasActionCacheEntries := actionCache != nil && len(actionCache.Entries) > 0 @@ -220,9 +233,6 @@ func compileModifiedFilesWithDependencies(ctx context.Context, compiler *workflo } else { compileHelpersLog.Print("Skipping .gitattributes update (no compiled workflows and no action cache entries)") } - - // Print summary instead of just "Recompiled" - printCompilationSummary(stats, false) } // handleFileDeleted handles the deletion of a markdown file by removing its corresponding lock file diff --git a/pkg/cli/compile_orchestrator.go b/pkg/cli/compile_orchestrator.go index 411afcae16e..83172e5d53a 100644 --- a/pkg/cli/compile_orchestrator.go +++ b/pkg/cli/compile_orchestrator.go @@ -19,37 +19,16 @@ func CompileWorkflows(ctx context.Context, config CompileConfig) ([]*workflow.Wo compileOrchestratorLog.Printf("Starting workflow compilation: files=%d, validate=%v, watch=%v, noEmit=%v", len(config.MarkdownFiles), config.Validate, config.Watch, config.NoEmit) - // Check context cancellation at the start - select { - case <-ctx.Done(): - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) - return nil, ctx.Err() - default: - } - - if os.Getenv("GH_HOST") == "" { //nolint:osgetenvlibrary - if detectedHost := getHostFromOriginRemote(); detectedHost != "github.com" && detectedHost != "" { - compileOrchestratorLog.Printf("Auto-detected GHES host from git remote: %s", detectedHost) - workflow.SetDefaultGHHost(detectedHost) - } else if detectedHost == "github.com" { - workflow.SetDefaultGHHost("") - } - } - - // Validate configuration - if err := validateCompileConfig(config); err != nil { + if err := compileWorkflowsCheckContext(ctx); err != nil { return nil, err } + compileWorkflowsSetDefaultGHHost() - // Validate action mode if specified - if err := validateActionModeConfig(config.ActionMode); err != nil { + if err := compileWorkflowsValidateConfig(config); err != nil { return nil, err } - // Initialize actionlint statistics if actionlint is enabled - if config.Actionlint && !config.NoEmit { - initActionlintStats() - } + compileWorkflowsInitActionlint(config) // Track compilation statistics stats := &CompilationStats{} @@ -58,22 +37,13 @@ func CompileWorkflows(ctx context.Context, config CompileConfig) ([]*workflow.Wo var validationResults []ValidationResult // Set up workflow directory (using default if not specified) - workflowDir := config.WorkflowDir - if workflowDir == "" { - workflowDir = constants.GetWorkflowDir() - compileOrchestratorLog.Printf("Using default workflow directory: %s", workflowDir) - } else { - workflowDir = filepath.Clean(workflowDir) - compileOrchestratorLog.Printf("Using custom workflow directory: %s", workflowDir) - } + workflowDir := compileWorkflowsWorkflowDir(config.WorkflowDir) // Preprocess args: expand directory paths and GitHub URLs to constituent workflow files - if len(config.MarkdownFiles) > 0 { - expandedFiles, err := resolveCompileArgs(config.MarkdownFiles, config.Verbose) - if err != nil { - return nil, err - } - config.MarkdownFiles = expandedFiles + var err error + config.MarkdownFiles, err = compileWorkflowsResolveArgs(config.MarkdownFiles, config.Verbose) + if err != nil { + return nil, err } // Create and configure compiler @@ -91,22 +61,7 @@ func CompileWorkflows(ctx context.Context, config CompileConfig) ([]*workflow.Wo // Handle watch mode (early return) if config.Watch { - // Watch mode: watch for file changes and recompile automatically - // For watch mode, we only support a single file for now - var markdownFile string - if len(config.MarkdownFiles) > 0 { - if len(config.MarkdownFiles) > 1 { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Watch mode only supports a single file, using the first one")) - } - // Resolve the workflow file to get the full path - resolvedFile, err := resolveWorkflowFile(config.MarkdownFiles[0], config.Verbose) - if err != nil { - // Return error directly without wrapping - it already contains formatted message with suggestions - return nil, err - } - markdownFile = resolvedFile - } - return nil, watchAndCompileWorkflows(ctx, markdownFile, compiler, config.Verbose) + return compileWorkflowsWatch(ctx, config, compiler) } // Compile specific files or all files in directory @@ -118,3 +73,81 @@ func CompileWorkflows(ctx context.Context, config CompileConfig) ([]*workflow.Wo // Compile all workflow files in directory return compileAllFilesInDirectory(ctx, compiler, config, workflowDir, stats, &validationResults) } + +func compileWorkflowsCheckContext(ctx context.Context) error { + // Check context cancellation at the start + select { + case <-ctx.Done(): + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) + return ctx.Err() + default: + return nil + } +} + +func compileWorkflowsSetDefaultGHHost() { + if os.Getenv("GH_HOST") != "" { //nolint:osgetenvlibrary + return + } + if detectedHost := getHostFromOriginRemote(); detectedHost != "github.com" && detectedHost != "" { + compileOrchestratorLog.Printf("Auto-detected GHES host from git remote: %s", detectedHost) + workflow.SetDefaultGHHost(detectedHost) + } else if detectedHost == "github.com" { + workflow.SetDefaultGHHost("") + } +} + +func compileWorkflowsValidateConfig(config CompileConfig) error { + // Validate configuration + if err := validateCompileConfig(config); err != nil { + return err + } + + // Validate action mode if specified + return validateActionModeConfig(config.ActionMode) +} + +func compileWorkflowsInitActionlint(config CompileConfig) { + // Initialize actionlint statistics if actionlint is enabled + if config.Actionlint && !config.NoEmit { + initActionlintStats() + } +} + +func compileWorkflowsWorkflowDir(workflowDir string) string { + if workflowDir == "" { + workflowDir = constants.GetWorkflowDir() + compileOrchestratorLog.Printf("Using default workflow directory: %s", workflowDir) + return workflowDir + } + workflowDir = filepath.Clean(workflowDir) + compileOrchestratorLog.Printf("Using custom workflow directory: %s", workflowDir) + return workflowDir +} + +func compileWorkflowsResolveArgs(markdownFiles []string, verbose bool) ([]string, error) { + // Preprocess args: expand directory paths and GitHub URLs to constituent workflow files + if len(markdownFiles) == 0 { + return markdownFiles, nil + } + return resolveCompileArgs(markdownFiles, verbose) +} + +func compileWorkflowsWatch(ctx context.Context, config CompileConfig, compiler *workflow.Compiler) ([]*workflow.WorkflowData, error) { + // Watch mode: watch for file changes and recompile automatically + // For watch mode, we only support a single file for now + var markdownFile string + if len(config.MarkdownFiles) > 0 { + if len(config.MarkdownFiles) > 1 { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Watch mode only supports a single file, using the first one")) + } + // Resolve the workflow file to get the full path + resolvedFile, err := resolveWorkflowFile(config.MarkdownFiles[0], config.Verbose) + if err != nil { + // Return error directly without wrapping - it already contains formatted message with suggestions + return nil, err + } + markdownFile = resolvedFile + } + return nil, watchAndCompileWorkflows(ctx, markdownFile, compiler, config.Verbose) +} diff --git a/pkg/cli/compile_pipeline.go b/pkg/cli/compile_pipeline.go index 84396ccf4d7..f6e9728bb19 100644 --- a/pkg/cli/compile_pipeline.go +++ b/pkg/cli/compile_pipeline.go @@ -40,6 +40,24 @@ var compileOrchestrationLog = logger.New("cli:compile_pipeline") const fallbackCompilationErrorMessage = "compilation failed (no detailed error message available)" +type compileSpecificFilesState struct { + workflowDataList []*workflow.WorkflowData + compiledCount int + errorCount int + lockFilesForActionlint []string + lockFilesForZizmor []string + lockFilesForDirTools []string +} + +type compileAllFilesInDirectoryState struct { + workflowDataList []*workflow.WorkflowData + successCount int + errorCount int + lockFilesForActionlint []string + lockFilesForZizmor []string + lockFilesForDirTools []string +} + // compileSpecificFiles compiles a specific list of workflow files func compileSpecificFiles( ctx context.Context, @@ -57,402 +75,466 @@ func compileSpecificFiles( compileOrchestrationLog.Print("Automatically enabling action SHA validation due to --force-refresh-action-pins") } - var workflowDataList []*workflow.WorkflowData - var compiledCount int - var errorCount int - var lockFilesForActionlint []string - var lockFilesForZizmor []string - var lockFilesForDirTools []string // lock files for directory-based tools (poutine, runner-guard) + state := &compileSpecificFilesState{} + if err := compileSpecificFilesProcessAll(ctx, compiler, config, shouldValidate, stats, validationResults, state); err != nil { + return state.workflowDataList, err + } + if err := compileSpecificFilesRunBatchTools(ctx, config, state); err != nil { + return state.workflowDataList, err + } + + // Get warning count from compiler + stats.Warnings = compiler.GetWarningCount() - // Compile each specified file + // Display schedule warnings + displayScheduleWarnings(compiler, config.JSONOutput) + + // Display safe update warnings (emitted as prompts for the calling agent) + displaySafeUpdateWarnings(compiler, config.JSONOutput) + + // Post-processing + if err := runPostProcessing(compiler, state.workflowDataList, config, state.compiledCount); err != nil { + return state.workflowDataList, err + } + + // Output results + if err := outputResults(stats, validationResults, config); err != nil { + return state.workflowDataList, err + } + + // Return error if any compilations failed + // Don't return the detailed error message here since it's already printed in the summary + // Returning a simple error prevents duplication in the output + if state.errorCount > 0 { + return state.workflowDataList, errors.New("compilation failed") + } + + return state.workflowDataList, nil +} + +func compileSpecificFilesProcessAll( + ctx context.Context, + compiler *workflow.Compiler, + config CompileConfig, + shouldValidate bool, + stats *CompilationStats, + validationResults *[]ValidationResult, + state *compileSpecificFilesState, +) error { for _, markdownFile := range config.MarkdownFiles { - // Respect context cancellation between files (e.g. Ctrl+C) - select { - case <-ctx.Done(): + if err := ctx.Err(); err != nil { fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) - return workflowDataList, ctx.Err() - default: + return err } + compileSpecificFilesProcessOne(ctx, compiler, config, shouldValidate, stats, validationResults, state, markdownFile) + } + return nil +} - stats.Total++ - - // Initialize validation result - result := ValidationResult{ - Workflow: markdownFile, - Valid: true, - Errors: []CompileValidationError{}, - Warnings: []CompileValidationError{}, - } +func compileSpecificFilesProcessOne( + ctx context.Context, + compiler *workflow.Compiler, + config CompileConfig, + shouldValidate bool, + stats *CompilationStats, + validationResults *[]ValidationResult, + state *compileSpecificFilesState, + markdownFile string, +) { + stats.Total++ + resolvedFile, ok := compileSpecificFilesResolveFile(config, stats, validationResults, state, markdownFile) + if !ok { + return + } + fileResult := compileWorkflowFile(ctx, compiler, resolvedFile, compileWorkflowFileOptions{ + verbose: config.Verbose, + jsonOutput: config.JSONOutput, + noEmit: config.NoEmit, + strict: config.Strict, + validate: shouldValidate, + }) + compileSpecificFilesApplyResult(config, stats, state, resolvedFile, fileResult) + *validationResults = append(*validationResults, fileResult.validationResult) +} - // Resolve workflow ID or file path to actual file path - compileOrchestrationLog.Printf("Resolving workflow file: %s", markdownFile) - resolvedFile, err := resolveWorkflowFile(markdownFile, config.Verbose) - if err != nil { - // Don't print error here - it will be displayed in the compilation summary - // The error is stored in ValidationResult for JSON output and returned for main to display - errorCount++ - stats.Errors++ - trackWorkflowFailure(stats, markdownFile, 1, []string{err.Error()}) - result.Valid = false - result.Errors = append(result.Errors, CompileValidationError{ - Type: "resolution_error", - Message: err.Error(), - }) - *validationResults = append(*validationResults, result) - continue - } +func compileSpecificFilesResolveFile( + config CompileConfig, + stats *CompilationStats, + validationResults *[]ValidationResult, + state *compileSpecificFilesState, + markdownFile string, +) (string, bool) { + result := ValidationResult{Workflow: markdownFile, Valid: true, Errors: []CompileValidationError{}, Warnings: []CompileValidationError{}} + compileOrchestrationLog.Printf("Resolving workflow file: %s", markdownFile) + resolvedFile, err := resolveWorkflowFile(markdownFile, config.Verbose) + if err == nil { compileOrchestrationLog.Printf("Resolved to: %s", resolvedFile) + return resolvedFile, true + } + state.errorCount++ + stats.Errors++ + trackWorkflowFailure(stats, markdownFile, 1, []string{err.Error()}) + result.Valid = false + result.Errors = append(result.Errors, CompileValidationError{Type: "resolution_error", Message: err.Error()}) + *validationResults = append(*validationResults, result) + return "", false +} - // Update result with resolved file name - result.Workflow = filepath.Base(resolvedFile) - - // Compile regular workflow file (disable per-file security tools) - fileResult := compileWorkflowFile( - ctx, compiler, resolvedFile, compileWorkflowFileOptions{ - verbose: config.Verbose, - jsonOutput: config.JSONOutput, - noEmit: config.NoEmit, - strict: config.Strict, - validate: shouldValidate, - // zizmor, poutine, actionlint disabled per-file (batched instead) - }, - ) - - if !fileResult.success { - // Collect error messages from validation result for display in summary - var errMsgs []string - for _, verr := range fileResult.validationResult.Errors { - errMsgs = append(errMsgs, verr.Message) - } - if len(errMsgs) == 0 { - errMsgs = []string{fallbackCompilationErrorMessage} - } - errorCount++ - stats.Errors += len(errMsgs) - trackWorkflowFailure(stats, resolvedFile, len(errMsgs), errMsgs) - } else { - compiledCount++ - if fileResult.workflowData != nil { - workflowDataList = append(workflowDataList, fileResult.workflowData) - } +func compileSpecificFilesApplyResult(config CompileConfig, stats *CompilationStats, state *compileSpecificFilesState, resolvedFile string, fileResult compileWorkflowFileResult) { + if !fileResult.success { + errMsgs := compileSpecificFilesErrorMessages(fileResult) + state.errorCount++ + stats.Errors += len(errMsgs) + trackWorkflowFailure(stats, resolvedFile, len(errMsgs), errMsgs) + return + } + state.compiledCount++ + if fileResult.workflowData != nil { + state.workflowDataList = append(state.workflowDataList, fileResult.workflowData) + } + compileSpecificFilesCollectLockFiles(config, state, fileResult.lockFile) +} - // Collect lock files for batch security tools - if !config.NoEmit && fileResult.lockFile != "" { - if _, err := os.Stat(fileResult.lockFile); err == nil { - if config.Actionlint { - lockFilesForActionlint = append(lockFilesForActionlint, fileResult.lockFile) - } - if config.Zizmor { - lockFilesForZizmor = append(lockFilesForZizmor, fileResult.lockFile) - } - if config.Poutine || config.RunnerGuard { - lockFilesForDirTools = append(lockFilesForDirTools, fileResult.lockFile) - } - } - } - } +func compileSpecificFilesErrorMessages(fileResult compileWorkflowFileResult) []string { + var errMsgs []string + for _, verr := range fileResult.validationResult.Errors { + errMsgs = append(errMsgs, verr.Message) + } + if len(errMsgs) == 0 { + errMsgs = []string{fallbackCompilationErrorMessage} + } + return errMsgs +} - *validationResults = append(*validationResults, fileResult.validationResult) +func compileSpecificFilesCollectLockFiles(config CompileConfig, state *compileSpecificFilesState, lockFile string) { + if config.NoEmit || lockFile == "" { + return + } + if _, err := os.Stat(lockFile); err != nil { + return + } + if config.Actionlint { + state.lockFilesForActionlint = append(state.lockFilesForActionlint, lockFile) + } + if config.Zizmor { + state.lockFilesForZizmor = append(state.lockFilesForZizmor, lockFile) + } + if config.Poutine || config.RunnerGuard { + state.lockFilesForDirTools = append(state.lockFilesForDirTools, lockFile) } +} - // Run batch actionlint on all collected lock files - if config.Actionlint && !config.NoEmit && len(lockFilesForActionlint) > 0 { +func compileSpecificFilesRunBatchTools(ctx context.Context, config CompileConfig, state *compileSpecificFilesState) error { + if config.Actionlint && !config.NoEmit && len(state.lockFilesForActionlint) > 0 { if err := ctx.Err(); err != nil { - return workflowDataList, err + return err } - if err := RunActionlintOnFiles(ctx, lockFilesForActionlint, config.Verbose && !config.JSONOutput, config.Strict); err != nil { - if config.Strict { - return workflowDataList, err - } + if err := RunActionlintOnFiles(ctx, state.lockFilesForActionlint, config.Verbose && !config.JSONOutput, config.Strict); err != nil && config.Strict { + return err } } - - // Run batch zizmor on all collected lock files - if config.Zizmor && !config.NoEmit && len(lockFilesForZizmor) > 0 { + if config.Zizmor && !config.NoEmit && len(state.lockFilesForZizmor) > 0 { if err := ctx.Err(); err != nil { - return workflowDataList, err + return err } - if err := RunZizmorOnFiles(lockFilesForZizmor, config.Verbose && !config.JSONOutput, config.Strict); err != nil { - if config.Strict { - return workflowDataList, err - } + if err := RunZizmorOnFiles(state.lockFilesForZizmor, config.Verbose && !config.JSONOutput, config.Strict); err != nil && config.Strict { + return err } } + return compileSpecificFilesRunDirectoryTools(ctx, config, state) +} - // Run batch poutine once on the workflow directory - // Get the directory from the first lock file (all should be in same directory) - if config.Poutine && !config.NoEmit && len(lockFilesForDirTools) > 0 { +func compileSpecificFilesRunDirectoryTools(ctx context.Context, config CompileConfig, state *compileSpecificFilesState) error { + if len(state.lockFilesForDirTools) == 0 || config.NoEmit { + return nil + } + workflowDir := filepath.Dir(state.lockFilesForDirTools[0]) + if config.Poutine { if err := ctx.Err(); err != nil { - return workflowDataList, err + return err } - workflowDir := filepath.Dir(lockFilesForDirTools[0]) - if err := runBatchDirectoryTool("poutine", workflowDir, config.Verbose && !config.JSONOutput, config.Strict, RunPoutineOnDirectory); err != nil { - if config.Strict { - return workflowDataList, err - } + if err := runBatchDirectoryTool("poutine", workflowDir, config.Verbose && !config.JSONOutput, config.Strict, RunPoutineOnDirectory); err != nil && config.Strict { + return err } } - - // Run batch runner-guard once on the workflow directory - // Get the directory from the first lock file (all should be in same directory) - if config.RunnerGuard && !config.NoEmit && len(lockFilesForDirTools) > 0 { + if config.RunnerGuard { if err := ctx.Err(); err != nil { - return workflowDataList, err + return err } - workflowDir := filepath.Dir(lockFilesForDirTools[0]) - if err := runBatchDirectoryTool("runner-guard", workflowDir, config.Verbose && !config.JSONOutput, config.Strict, RunRunnerGuardOnDirectory); err != nil { - if config.Strict { - return workflowDataList, err - } + if err := runBatchDirectoryTool("runner-guard", workflowDir, config.Verbose && !config.JSONOutput, config.Strict, RunRunnerGuardOnDirectory); err != nil && config.Strict { + return err } } + return nil +} - // Get warning count from compiler - stats.Warnings = compiler.GetWarningCount() - - // Display schedule warnings - displayScheduleWarnings(compiler, config.JSONOutput) - - // Display safe update warnings (emitted as prompts for the calling agent) - displaySafeUpdateWarnings(compiler, config.JSONOutput) +// compileAllFilesInDirectory compiles all workflow files in a directory +func compileAllFilesInDirectory( + ctx context.Context, + compiler *workflow.Compiler, + config CompileConfig, + workflowDir string, + stats *CompilationStats, + validationResults *[]ValidationResult, +) ([]*workflow.WorkflowData, error) { + gitRoot, workflowsDir, mdFiles, err := compileAllFilesInDirectoryInputs(config, workflowDir) + if err != nil { + return nil, err + } - // Post-processing - if err := runPostProcessing(compiler, workflowDataList, config, compiledCount); err != nil { - return workflowDataList, err + compileOrchestrationLog.Printf("Found %d markdown files to compile", len(mdFiles)) + if config.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d markdown files to compile", len(mdFiles)))) } - // Output results - if err := outputResults(stats, validationResults, config); err != nil { - return workflowDataList, err + // Handle purge logic: collect existing files before compilation + var purgeData *purgeTrackingData + if config.Purge { + purgeData = collectPurgeData(workflowsDir, mdFiles, config.Verbose) } - // Return error if any compilations failed - // Don't return the detailed error message here since it's already printed in the summary - // Returning a simple error prevents duplication in the output - if errorCount > 0 { - return workflowDataList, errors.New("compilation failed") + shouldValidate := compileAllFilesInDirectoryShouldValidate(config) + state := &compileAllFilesInDirectoryState{} + if err := compileAllFilesInDirectoryProcessAll(ctx, compiler, config, shouldValidate, mdFiles, stats, validationResults, state); err != nil { + return state.workflowDataList, err + } + if err := compileAllFilesInDirectoryRunBatchTools(ctx, config, workflowsDir, state); err != nil { + return state.workflowDataList, err } - return workflowDataList, nil + return compileAllFilesInDirectoryFinalize(compileAllFilesInDirectoryFinalizeParams{ + Ctx: ctx, + Compiler: compiler, + Config: config, + WorkflowsDir: workflowsDir, + GitRoot: gitRoot, + MDFiles: mdFiles, + PurgeData: purgeData, + Stats: stats, + ValidationResults: validationResults, + State: state, + }) } -// compileAllFilesInDirectory compiles all workflow files in a directory -func compileAllFilesInDirectory( - ctx context.Context, - compiler *workflow.Compiler, +type compileAllFilesInDirectoryFinalizeParams struct { + Ctx context.Context + Compiler *workflow.Compiler + Config CompileConfig + WorkflowsDir string + GitRoot string + MDFiles []string + PurgeData *purgeTrackingData + Stats *CompilationStats + ValidationResults *[]ValidationResult + State *compileAllFilesInDirectoryState +} + +func compileAllFilesInDirectoryFinalize(p compileAllFilesInDirectoryFinalizeParams) ([]*workflow.WorkflowData, error) { + // Emit recommendation when many slash commands are present without centralized strategy. + displayCentralizedSlashCommandRecommendation(p.Compiler, p.State.workflowDataList, p.Config.JSONOutput) + p.Stats.Warnings = p.Compiler.GetWarningCount() + displayScheduleWarnings(p.Compiler, p.Config.JSONOutput) + displaySafeUpdateWarnings(p.Compiler, p.Config.JSONOutput) + if p.Config.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Successfully compiled %d out of %d workflow files", p.State.successCount, len(p.MDFiles)))) + } + if p.Config.Purge && p.PurgeData != nil { + runPurgeOperations(p.WorkflowsDir, p.PurgeData, p.Config.Verbose) + } + if err := runPostProcessingForDirectory(p.Ctx, p.Compiler, p.State.workflowDataList, p.Config, p.WorkflowsDir, p.GitRoot, p.State.successCount, p.State.errorCount); err != nil { + return p.State.workflowDataList, err + } + return compileAllFilesInDirectoryOutput(p.Config, p.MDFiles, p.Stats, p.ValidationResults, p.State) +} + +func compileAllFilesInDirectoryOutput( config CompileConfig, - workflowDir string, + mdFiles []string, stats *CompilationStats, validationResults *[]ValidationResult, + state *compileAllFilesInDirectoryState, ) ([]*workflow.WorkflowData, error) { - // Find git root for consistent behavior + // Output results. + // Populate MarkdownFiles so that outputResults can collect per-workflow stats + // (e.g. schedule heatmap) even when the caller did not specify explicit files. + if config.Stats && len(config.MarkdownFiles) == 0 { + config.MarkdownFiles = mdFiles + } + if err := outputResults(stats, validationResults, config); err != nil { + return state.workflowDataList, err + } + + // Return error if any compilations failed + if state.errorCount > 0 { + return state.workflowDataList, errors.New("compilation failed") + } + + return state.workflowDataList, nil +} + +func compileAllFilesInDirectoryInputs(config CompileConfig, workflowDir string) (string, string, []string, error) { gitRoot, err := gitutil.FindGitRoot() if err != nil { - return nil, fmt.Errorf("compile without arguments requires being in a git repository: %w", err) + return "", "", nil, fmt.Errorf("compile without arguments requires being in a git repository: %w", err) } compileOrchestrationLog.Printf("Found git root: %s", gitRoot) - - // Compile all markdown files in the specified workflow directory workflowsDir := filepath.Join(gitRoot, workflowDir) if _, err := os.Stat(workflowsDir); os.IsNotExist(err) { - return nil, fmt.Errorf("the %s directory does not exist in git root (%s)", workflowDir, gitRoot) + return "", "", nil, fmt.Errorf("the %s directory does not exist in git root (%s)", workflowDir, gitRoot) } - compileOrchestrationLog.Printf("Scanning for markdown files in %s", workflowsDir) if config.Verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Scanning for markdown files in "+workflowsDir)) } - - // Find and filter markdown files (shared helper keeps logic in one place) mdFiles, err := getMarkdownWorkflowFiles(workflowsDir) if err != nil { - return nil, fmt.Errorf("failed to find markdown files: %w", err) + return "", "", nil, fmt.Errorf("failed to find markdown files: %w", err) } mdFiles, err = filterMarkdownFilesWithFrontmatter(mdFiles) if err != nil { - return nil, fmt.Errorf("failed to filter markdown files: %w", err) + return "", "", nil, fmt.Errorf("failed to filter markdown files: %w", err) } - if len(mdFiles) == 0 { - return nil, fmt.Errorf("no workflow markdown files found in %s (workflow files must start with a frontmatter opener on the first line)", workflowsDir) - } - - compileOrchestrationLog.Printf("Found %d markdown files to compile", len(mdFiles)) - if config.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d markdown files to compile", len(mdFiles)))) - } - - // Handle purge logic: collect existing files before compilation - var purgeData *purgeTrackingData - if config.Purge { - purgeData = collectPurgeData(workflowsDir, mdFiles, config.Verbose) + return "", "", nil, fmt.Errorf("no workflow markdown files found in %s (workflow files must start with a frontmatter opener on the first line)", workflowsDir) } + return gitRoot, workflowsDir, mdFiles, nil +} - // Enable validation automatically when force-refresh-action-pins is used - // to verify all resolved action SHAs are valid +func compileAllFilesInDirectoryShouldValidate(config CompileConfig) bool { shouldValidate := config.Validate || config.ForceRefreshActionPins if config.ForceRefreshActionPins && !config.Validate { compileOrchestrationLog.Print("Automatically enabling action SHA validation due to --force-refresh-action-pins") } + return shouldValidate +} - // Compile each file - var workflowDataList []*workflow.WorkflowData - var successCount int - var errorCount int - var lockFilesForActionlint []string - var lockFilesForZizmor []string - var lockFilesForDirTools []string // lock files for directory-based tools (poutine, runner-guard) - +func compileAllFilesInDirectoryProcessAll( + ctx context.Context, + compiler *workflow.Compiler, + config CompileConfig, + shouldValidate bool, + mdFiles []string, + stats *CompilationStats, + validationResults *[]ValidationResult, + state *compileAllFilesInDirectoryState, +) error { for _, file := range mdFiles { - // Respect context cancellation between files (e.g. Ctrl+C) - select { - case <-ctx.Done(): + if err := ctx.Err(); err != nil { fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) - return workflowDataList, ctx.Err() - default: + return err } + compileAllFilesInDirectoryProcessOne(ctx, compiler, config, shouldValidate, stats, validationResults, state, file) + } + return nil +} - stats.Total++ - - // Compile regular workflow file (disable per-file security tools) - fileResult := compileWorkflowFile( - ctx, compiler, file, compileWorkflowFileOptions{ - verbose: config.Verbose, - jsonOutput: config.JSONOutput, - noEmit: config.NoEmit, - strict: config.Strict, - validate: shouldValidate, - // zizmor, poutine, actionlint disabled per-file (batched instead) - }, - ) - - if !fileResult.success { - // Collect error messages from validation result - var errMsgs []string - for _, verr := range fileResult.validationResult.Errors { - errMsgs = append(errMsgs, verr.Message) - } - if len(errMsgs) == 0 { - errMsgs = []string{fallbackCompilationErrorMessage} - } - errorCount++ - stats.Errors += len(errMsgs) - trackWorkflowFailure(stats, file, len(errMsgs), errMsgs) - } else { - successCount++ - if fileResult.workflowData != nil { - workflowDataList = append(workflowDataList, fileResult.workflowData) - } +func compileAllFilesInDirectoryProcessOne( + ctx context.Context, + compiler *workflow.Compiler, + config CompileConfig, + shouldValidate bool, + stats *CompilationStats, + validationResults *[]ValidationResult, + state *compileAllFilesInDirectoryState, + file string, +) { + stats.Total++ + fileResult := compileWorkflowFile(ctx, compiler, file, compileWorkflowFileOptions{ + verbose: config.Verbose, + jsonOutput: config.JSONOutput, + noEmit: config.NoEmit, + strict: config.Strict, + validate: shouldValidate, + }) + compileAllFilesInDirectoryApplyResult(config, stats, state, file, fileResult) + *validationResults = append(*validationResults, fileResult.validationResult) +} - // Collect lock files for batch security tools - if !config.NoEmit && fileResult.lockFile != "" { - if _, err := os.Stat(fileResult.lockFile); err == nil { - if config.Actionlint { - lockFilesForActionlint = append(lockFilesForActionlint, fileResult.lockFile) - } - if config.Zizmor { - lockFilesForZizmor = append(lockFilesForZizmor, fileResult.lockFile) - } - if config.Poutine || config.RunnerGuard { - lockFilesForDirTools = append(lockFilesForDirTools, fileResult.lockFile) - } - } - } - } +func compileAllFilesInDirectoryApplyResult(config CompileConfig, stats *CompilationStats, state *compileAllFilesInDirectoryState, file string, fileResult compileWorkflowFileResult) { + if !fileResult.success { + errMsgs := compileAllFilesInDirectoryErrorMessages(fileResult) + state.errorCount++ + stats.Errors += len(errMsgs) + trackWorkflowFailure(stats, file, len(errMsgs), errMsgs) + return + } + state.successCount++ + if fileResult.workflowData != nil { + state.workflowDataList = append(state.workflowDataList, fileResult.workflowData) + } + compileAllFilesInDirectoryCollectLockFiles(config, state, fileResult.lockFile) +} - *validationResults = append(*validationResults, fileResult.validationResult) +func compileAllFilesInDirectoryErrorMessages(fileResult compileWorkflowFileResult) []string { + var errMsgs []string + for _, verr := range fileResult.validationResult.Errors { + errMsgs = append(errMsgs, verr.Message) + } + if len(errMsgs) == 0 { + errMsgs = []string{fallbackCompilationErrorMessage} + } + return errMsgs +} + +func compileAllFilesInDirectoryCollectLockFiles(config CompileConfig, state *compileAllFilesInDirectoryState, lockFile string) { + if config.NoEmit || lockFile == "" { + return + } + if _, err := os.Stat(lockFile); err != nil { + return + } + if config.Actionlint { + state.lockFilesForActionlint = append(state.lockFilesForActionlint, lockFile) + } + if config.Zizmor { + state.lockFilesForZizmor = append(state.lockFilesForZizmor, lockFile) } + if config.Poutine || config.RunnerGuard { + state.lockFilesForDirTools = append(state.lockFilesForDirTools, lockFile) + } +} - // Run batch actionlint - if config.Actionlint && !config.NoEmit && len(lockFilesForActionlint) > 0 { +func compileAllFilesInDirectoryRunBatchTools(ctx context.Context, config CompileConfig, workflowsDir string, state *compileAllFilesInDirectoryState) error { + if config.Actionlint && !config.NoEmit && len(state.lockFilesForActionlint) > 0 { if err := ctx.Err(); err != nil { - return workflowDataList, err + return err } - if err := RunActionlintOnFiles(ctx, lockFilesForActionlint, config.Verbose && !config.JSONOutput, config.Strict); err != nil { - if config.Strict { - return workflowDataList, err - } + if err := RunActionlintOnFiles(ctx, state.lockFilesForActionlint, config.Verbose && !config.JSONOutput, config.Strict); err != nil && config.Strict { + return err } } - - // Run batch zizmor - if config.Zizmor && !config.NoEmit && len(lockFilesForZizmor) > 0 { + if config.Zizmor && !config.NoEmit && len(state.lockFilesForZizmor) > 0 { if err := ctx.Err(); err != nil { - return workflowDataList, err + return err } - if err := RunZizmorOnFiles(lockFilesForZizmor, config.Verbose && !config.JSONOutput, config.Strict); err != nil { - if config.Strict { - return workflowDataList, err - } + if err := RunZizmorOnFiles(state.lockFilesForZizmor, config.Verbose && !config.JSONOutput, config.Strict); err != nil && config.Strict { + return err } } + return compileAllFilesInDirectoryRunDirectoryTools(ctx, config, workflowsDir, state) +} - // Run batch poutine once on the workflow directory - if config.Poutine && !config.NoEmit && len(lockFilesForDirTools) > 0 { +func compileAllFilesInDirectoryRunDirectoryTools(ctx context.Context, config CompileConfig, workflowsDir string, state *compileAllFilesInDirectoryState) error { + if config.NoEmit || len(state.lockFilesForDirTools) == 0 { + return nil + } + if config.Poutine { if err := ctx.Err(); err != nil { - return workflowDataList, err + return err } - if err := runBatchDirectoryTool("poutine", workflowsDir, config.Verbose && !config.JSONOutput, config.Strict, RunPoutineOnDirectory); err != nil { - if config.Strict { - return workflowDataList, err - } + if err := runBatchDirectoryTool("poutine", workflowsDir, config.Verbose && !config.JSONOutput, config.Strict, RunPoutineOnDirectory); err != nil && config.Strict { + return err } } - - // Run batch runner-guard once on the workflow directory - if config.RunnerGuard && !config.NoEmit && len(lockFilesForDirTools) > 0 { + if config.RunnerGuard { if err := ctx.Err(); err != nil { - return workflowDataList, err + return err } - if err := runBatchDirectoryTool("runner-guard", workflowsDir, config.Verbose && !config.JSONOutput, config.Strict, RunRunnerGuardOnDirectory); err != nil { - if config.Strict { - return workflowDataList, err - } + if err := runBatchDirectoryTool("runner-guard", workflowsDir, config.Verbose && !config.JSONOutput, config.Strict, RunRunnerGuardOnDirectory); err != nil && config.Strict { + return err } } - - // Emit recommendation when many slash commands are present without centralized strategy. - displayCentralizedSlashCommandRecommendation(compiler, workflowDataList, config.JSONOutput) - - // Get warning count from compiler - stats.Warnings = compiler.GetWarningCount() - - // Display schedule warnings - displayScheduleWarnings(compiler, config.JSONOutput) - - // Display safe update warnings (emitted as prompts for the calling agent) - displaySafeUpdateWarnings(compiler, config.JSONOutput) - - if config.Verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Successfully compiled %d out of %d workflow files", successCount, len(mdFiles)))) - } - - // Handle purge logic if requested - if config.Purge && purgeData != nil { - runPurgeOperations(workflowsDir, purgeData, config.Verbose) - } - - // Post-processing - if err := runPostProcessingForDirectory(ctx, compiler, workflowDataList, config, workflowsDir, gitRoot, successCount, errorCount); err != nil { - return workflowDataList, err - } - - // Output results. - // Populate MarkdownFiles so that outputResults can collect per-workflow stats - // (e.g. schedule heatmap) even when the caller did not specify explicit files. - if config.Stats && len(config.MarkdownFiles) == 0 { - config.MarkdownFiles = mdFiles - } - if err := outputResults(stats, validationResults, config); err != nil { - return workflowDataList, err - } - - // Return error if any compilations failed - if errorCount > 0 { - return workflowDataList, errors.New("compilation failed") - } - - return workflowDataList, nil + return nil } // purgeTrackingData holds data needed for purge operations @@ -559,41 +641,11 @@ func runPostProcessingForDirectory( // Update .gitattributes (errors are non-fatal) _ = updateGitAttributes(successCount, actionCache, config.Verbose) - // Generate Dependabot manifests if requested - if config.Dependabot && !config.NoEmit { - absWorkflowDir := getAbsoluteWorkflowDir(workflowsDir, gitRoot) - if err := generateDependabotManifestsWrapper(compiler, workflowDataList, absWorkflowDir, config.ForceOverwrite, config.Strict); err != nil { - if config.Strict { - return err - } - } + if err := runPostProcessingForDirectoryDependabot(compiler, workflowDataList, config, workflowsDir, gitRoot); err != nil { + return err } - - // Reconcile compiler-managed Dependabot ignore entries for compiler-emitted action refs. - if config.Dependabot && !config.NoEmit { - if err := compiler.ReconcileManagedDependabotIgnoresInRepo(gitRoot); err != nil { - if config.Strict { - return err - } - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to reconcile compiler-managed Dependabot ignore entries: %v", err))) - } - } - - // Generate maintenance workflow if needed. - // Skip maintenance workflow generation when using custom --dir option. - // Keep invoking generators for empty workflowDataList so stale generated files are cleaned up. - if !config.NoEmit && config.WorkflowDir == "" { - absWorkflowDir := getAbsoluteWorkflowDir(workflowsDir, gitRoot) - if err := generateMaintenanceWorkflowWrapper(ctx, compiler, workflowDataList, absWorkflowDir, gitRoot, config.Verbose, config.Strict); err != nil { - if config.Strict { - return err - } - } - if err := generateCentralSlashCommandWorkflowWrapper(ctx, workflowDataList, absWorkflowDir, gitRoot, config.Strict); err != nil { - if config.Strict { - return err - } - } + if err := runPostProcessingForDirectoryMaintenance(ctx, compiler, workflowDataList, config, workflowsDir, gitRoot); err != nil { + return err } // Prune stale gh-aw-actions entries before saving @@ -610,6 +662,50 @@ func runPostProcessingForDirectory( return nil } +func runPostProcessingForDirectoryDependabot( + compiler *workflow.Compiler, + workflowDataList []*workflow.WorkflowData, + config CompileConfig, + workflowsDir string, + gitRoot string, +) error { + if !config.Dependabot || config.NoEmit { + return nil + } + absWorkflowDir := getAbsoluteWorkflowDir(workflowsDir, gitRoot) + if err := generateDependabotManifestsWrapper(compiler, workflowDataList, absWorkflowDir, config.ForceOverwrite, config.Strict); err != nil && config.Strict { + return err + } + if err := compiler.ReconcileManagedDependabotIgnoresInRepo(gitRoot); err != nil { + if config.Strict { + return err + } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to reconcile compiler-managed Dependabot ignore entries: %v", err))) + } + return nil +} + +func runPostProcessingForDirectoryMaintenance( + ctx context.Context, + compiler *workflow.Compiler, + workflowDataList []*workflow.WorkflowData, + config CompileConfig, + workflowsDir string, + gitRoot string, +) error { + if config.NoEmit || config.WorkflowDir != "" { + return nil + } + absWorkflowDir := getAbsoluteWorkflowDir(workflowsDir, gitRoot) + if err := generateMaintenanceWorkflowWrapper(ctx, compiler, workflowDataList, absWorkflowDir, gitRoot, config.Verbose, config.Strict); err != nil && config.Strict { + return err + } + if err := generateCentralSlashCommandWorkflowWrapper(ctx, workflowDataList, absWorkflowDir, gitRoot, config.Strict); err != nil && config.Strict { + return err + } + return nil +} + // outputResults outputs compilation results in the requested format func outputResults( stats *CompilationStats, diff --git a/pkg/cli/compile_repository_manifest.go b/pkg/cli/compile_repository_manifest.go index de14b272b8d..3cfff00b933 100644 --- a/pkg/cli/compile_repository_manifest.go +++ b/pkg/cli/compile_repository_manifest.go @@ -19,40 +19,61 @@ var findGitRootForManifestValidation = gitutil.FindGitRoot func validateRepositoryManifestForCompilation(config CompileConfig, stats *CompilationStats, validationResults *[]ValidationResult) error { compileRepositoryManifestLog.Print("Validating repository manifest for compilation") + manifestPath, err := validateRepositoryManifestForCompilationFindManifest() + if err != nil || manifestPath == "" { + return err + } + + warnings, parseErr, err := validateRepositoryManifestForCompilationReadAndValidate(manifestPath) + if err != nil { + return err + } + compileRepositoryManifestLog.Printf("Manifest parse result: warnings=%d, error=%v", len(warnings), parseErr) + + if len(warnings) > 0 { + stats.Warnings += len(warnings) + } + + result := validateRepositoryManifestForCompilationResult(manifestPath, warnings, parseErr) + return validateRepositoryManifestForCompilationHandleResult(config, warnings, parseErr, result, validationResults) +} + +func validateRepositoryManifestForCompilationFindManifest() (string, error) { gitRoot, err := findGitRootForManifestValidation() if err != nil { if errors.Is(err, gitutil.ErrNotGitRepository) { compileRepositoryManifestLog.Print("Not in a git repository, skipping manifest validation") - return nil + return "", nil } - return fmt.Errorf("failed to find git root for manifest validation: %w", err) + return "", fmt.Errorf("failed to find git root for manifest validation: %w", err) } manifestPath, err := findLocalRepositoryPackageManifest(gitRoot) if err != nil { - return err + return "", err } if manifestPath == "" { compileRepositoryManifestLog.Printf("No repository manifest found in %s", gitRoot) - return nil + return "", nil } - compileRepositoryManifestLog.Printf("Found repository manifest at %s", manifestPath) + return manifestPath, nil +} + +func validateRepositoryManifestForCompilationReadAndValidate(manifestPath string) ([]string, error, error) { content, err := os.ReadFile(manifestPath) if err != nil { - return fmt.Errorf("failed to read Agentic Workflow manifest %q: %w", manifestPath, err) + return nil, nil, fmt.Errorf("failed to read Agentic Workflow manifest %q: %w", manifestPath, err) } _, warnings, parseErr := parseRepositoryPackageManifest(manifestPath, content) if parseErr == nil { parseErr = validateLocalRepositoryPackageContents(manifestPath) } - compileRepositoryManifestLog.Printf("Manifest parse result: warnings=%d, error=%v", len(warnings), parseErr) - - if len(warnings) > 0 { - stats.Warnings += len(warnings) - } + return warnings, parseErr, nil +} +func validateRepositoryManifestForCompilationResult(manifestPath string, warnings []string, parseErr error) ValidationResult { result := ValidationResult{ Workflow: filepath.Base(manifestPath), Valid: parseErr == nil, @@ -63,7 +84,16 @@ func validateRepositoryManifestForCompilation(config CompileConfig, stats *Compi Message: warning, }) } + return result +} +func validateRepositoryManifestForCompilationHandleResult( + config CompileConfig, + warnings []string, + parseErr error, + result ValidationResult, + validationResults *[]ValidationResult, +) error { if parseErr != nil { result.Errors = append(result.Errors, CompileValidationError{ Type: "manifest_error", diff --git a/pkg/cli/compile_schedule_calendar.go b/pkg/cli/compile_schedule_calendar.go index b7a4d805ffd..4bfe22c556a 100644 --- a/pkg/cli/compile_schedule_calendar.go +++ b/pkg/cli/compile_schedule_calendar.go @@ -231,15 +231,21 @@ func displayScheduleCalendar(statsList []*WorkflowStats) { } isTerminal := tty.IsStderrTerminal() + displayScheduleCalendarTitle() + displayScheduleCalendarHeader() + displayScheduleCalendarRows(grid, isTerminal) + displayScheduleCalendarLegend(isTerminal) +} - // Title +func displayScheduleCalendarTitle() { fmt.Fprintln(os.Stderr) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Schedule Heatmap (UTC)")) fmt.Fprintln(os.Stderr) +} +func displayScheduleCalendarHeader() { // Hour header row: each cell is 3 chars wide ("XX "). // The day-label column is 5 chars: "Mon " (3 + 2 spaces). - const cellWidth = 3 const dayLabelWidth = 5 var headerBuf strings.Builder @@ -249,7 +255,11 @@ func displayScheduleCalendar(statsList []*WorkflowStats) { } header := headerBuf.String() fmt.Fprintln(os.Stderr, console.FormatTableHeaderStderr(header)) +} +func displayScheduleCalendarRows(grid *scheduleGrid, isTerminal bool) { + const cellWidth = 3 + const dayLabelWidth = 5 // One row per day, ordered Mon through Sun. for calendarIndex, dayLabel := range calendarDayNames { cronDay := calendarDayIndex[calendarIndex] @@ -268,7 +278,9 @@ func displayScheduleCalendar(statsList []*WorkflowStats) { fmt.Fprintln(os.Stderr, row.String()) } +} +func displayScheduleCalendarLegend(isTerminal bool) { // Legend fmt.Fprintln(os.Stderr) type legendEntry struct { diff --git a/pkg/cli/compile_stats.go b/pkg/cli/compile_stats.go index a0c67304c67..e7351853bd7 100644 --- a/pkg/cli/compile_stats.go +++ b/pkg/cli/compile_stats.go @@ -71,6 +71,15 @@ func collectWorkflowStats(lockFilePath string) (*WorkflowStats, error) { FileSize: fileInfo.Size(), } + collectWorkflowStatsCounts(stats, workflowYAML) + collectWorkflowStatsSchedules(stats, workflowYAML) + + compileStatsLog.Printf("Stats collected: jobs=%d, steps=%d, scripts=%d, size=%d bytes, schedules=%d", + stats.Jobs, stats.Steps, stats.ScriptCount, stats.FileSize, len(stats.Schedules)) + return stats, nil +} + +func collectWorkflowStatsCounts(stats *WorkflowStats, workflowYAML map[string]any) { // Count jobs and steps if jobs, ok := workflowYAML["jobs"].(map[string]any); ok { stats.Jobs = len(jobs) @@ -102,7 +111,9 @@ func collectWorkflowStats(lockFilePath string) (*WorkflowStats, error) { } } } +} +func collectWorkflowStatsSchedules(stats *WorkflowStats, workflowYAML map[string]any) { // Extract cron expressions from on.schedule[*].cron if onSection, ok := workflowYAML["on"].(map[string]any); ok { if schedules, ok := onSection["schedule"].([]any); ok { @@ -115,10 +126,6 @@ func collectWorkflowStats(lockFilePath string) (*WorkflowStats, error) { } } } - - compileStatsLog.Printf("Stats collected: jobs=%d, steps=%d, scripts=%d, size=%d bytes, schedules=%d", - stats.Jobs, stats.Steps, stats.ScriptCount, stats.FileSize, len(stats.Schedules)) - return stats, nil } // trackWorkflowFailure adds a workflow failure to the compilation statistics @@ -154,74 +161,91 @@ func printCompilationSummary(stats *CompilationStats, showAllErrors bool) { // Use different formatting based on whether there were errors if stats.Errors > 0 { fmt.Fprintln(os.Stderr, console.FormatErrorMessage(summary)) + printCompilationSummaryFailures(stats, showAllErrors) + } else if stats.Warnings > 0 { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(summary)) + } else { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(summary)) + } +} - // Show agent-friendly list of failed workflow IDs first - if len(stats.FailureDetails) > 0 { - fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, console.FormatErrorMessage("Failed workflows:")) - for _, failure := range stats.FailureDetails { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(filepath.Base(failure.Path))) - } - fmt.Fprintln(os.Stderr) +func printCompilationSummaryFailures(stats *CompilationStats, showAllErrors bool) { + // Show agent-friendly list of failed workflow IDs first + if len(stats.FailureDetails) > 0 { + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, console.FormatErrorMessage("Failed workflows:")) + for _, failure := range stats.FailureDetails { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(filepath.Base(failure.Path))) + } + fmt.Fprintln(os.Stderr) - // Display the actual error messages for each failed workflow - for _, failure := range stats.FailureDetails { - report := workflow.BuildPrioritizedErrorReportFromMessages(failure.ErrorMessages, showAllErrors) - if report.TotalCount == 0 { - continue - } + // Display the actual error messages for each failed workflow + for _, failure := range stats.FailureDetails { + printCompilationSummaryFailureDetail(failure, showAllErrors) + } + } else if len(stats.FailedWorkflows) > 0 { + printCompilationSummaryLegacyFailures(stats.FailedWorkflows) + } +} - header := fmt.Sprintf("%s (%d error(s)", filepath.Base(failure.Path), report.TotalCount) - if !showAllErrors && report.HiddenCount > 0 { - header += fmt.Sprintf(", showing top %d", len(report.DisplayedErrors)) - } - header += "):" - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(header)) - - lastHeading := "" - for i, prioritized := range report.DisplayedErrors { - heading := prioritized.Severity.Heading() - if heading != lastHeading { - fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("%s %s:", prioritized.Severity.Icon(), heading))) - lastHeading = heading - } - fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("%d. %s", i+1, prioritized.Message))) - if prioritized.Suggestion != "" { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("→ "+prioritized.Suggestion)) - } - } +func printCompilationSummaryFailureDetail(failure WorkflowFailure, showAllErrors bool) { + report := workflow.BuildPrioritizedErrorReportFromMessages(failure.ErrorMessages, showAllErrors) + if report.TotalCount == 0 { + return + } - if report.RecoveryPlan != nil && len(report.RecoveryPlan.Steps) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("💡 Recovery plan:")) - for i, step := range report.RecoveryPlan.Steps { - fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("%d. %s", i+1, step))) - } - } + header := fmt.Sprintf("%s (%d error(s)", filepath.Base(failure.Path), report.TotalCount) + if !showAllErrors && report.HiddenCount > 0 { + header += fmt.Sprintf(", showing top %d", len(report.DisplayedErrors)) + } + header += "):" + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(header)) - if report.SuppressedCount > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Suppressed %d cascading error(s) until the root cause is fixed.", report.SuppressedCount))) - } + printCompilationSummaryPrioritizedErrors(report) + printCompilationSummaryRecoveryPlan(report) - if !showAllErrors && report.HiddenCount > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run 'gh aw compile --show-all' to see all %d prioritized error(s).", report.TotalCount))) - } - } - } else if len(stats.FailedWorkflows) > 0 { - // Fallback for backward compatibility if FailureDetails is not populated - fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, console.FormatErrorMessage("Failed workflows:")) - for _, wf := range stats.FailedWorkflows { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(wf)) - } - fmt.Fprintln(os.Stderr) + if report.SuppressedCount > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Suppressed %d cascading error(s) until the root cause is fixed.", report.SuppressedCount))) + } + if !showAllErrors && report.HiddenCount > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run 'gh aw compile --show-all' to see all %d prioritized error(s).", report.TotalCount))) + } +} + +func printCompilationSummaryPrioritizedErrors(report workflow.PrioritizedErrorReport) { + lastHeading := "" + for i, prioritized := range report.DisplayedErrors { + heading := prioritized.Severity.Heading() + if heading != lastHeading { + fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("%s %s:", prioritized.Severity.Icon(), heading))) + lastHeading = heading + } + fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("%d. %s", i+1, prioritized.Message))) + if prioritized.Suggestion != "" { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("→ "+prioritized.Suggestion)) } - } else if stats.Warnings > 0 { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(summary)) - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(summary)) } } +func printCompilationSummaryRecoveryPlan(report workflow.PrioritizedErrorReport) { + if report.RecoveryPlan != nil && len(report.RecoveryPlan.Steps) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("💡 Recovery plan:")) + for i, step := range report.RecoveryPlan.Steps { + fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("%d. %s", i+1, step))) + } + } +} + +func printCompilationSummaryLegacyFailures(failedWorkflows []string) { + // Fallback for backward compatibility if FailureDetails is not populated + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, console.FormatErrorMessage("Failed workflows:")) + for _, wf := range failedWorkflows { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(wf)) + } + fmt.Fprintln(os.Stderr) +} + // collectWorkflowStatisticsWrapper collects and returns workflow statistics func collectWorkflowStatisticsWrapper(markdownFiles []string) []*WorkflowStats { compileStatsLog.Printf("Collecting workflow statistics for %d files", len(markdownFiles)) @@ -261,20 +285,7 @@ func displayStatsTable(statsList []*WorkflowStats) { return 0 }) - // Calculate totals - totalSize := int64(0) - totalJobs := 0 - totalSteps := 0 - totalScripts := 0 - totalScriptSize := 0 - - for _, stats := range statsList { - totalSize += stats.FileSize - totalJobs += stats.Jobs - totalSteps += stats.Steps - totalScripts += stats.ScriptCount - totalScriptSize += stats.ScriptSize - } + totals := displayStatsTableTotals(statsList) // Limit display to top 10 workflows by size displayCount := len(statsList) @@ -283,7 +294,41 @@ func displayStatsTable(statsList []*WorkflowStats) { displayCount = maxDisplay } - // Build table rows + rows := displayStatsTableRows(statsList, displayCount, maxDisplay) + tableConfig := console.TableConfig{ + Title: "", + Headers: []string{"WORKFLOW", "FILE SIZE", "JOBS", "STEPS", "SCRIPTS"}, + Rows: rows, + } + + // Render and print table + fmt.Fprint(os.Stderr, console.RenderTable(tableConfig)) + + // Print summary + displayStatsTableSummary(statsList, maxDisplay, totals) +} + +type displayStatsTableTotalValues struct { + totalSize int64 + totalJobs int + totalSteps int + totalScripts int + totalScriptSize int +} + +func displayStatsTableTotals(statsList []*WorkflowStats) displayStatsTableTotalValues { + var totals displayStatsTableTotalValues + for _, stats := range statsList { + totals.totalSize += stats.FileSize + totals.totalJobs += stats.Jobs + totals.totalSteps += stats.Steps + totals.totalScripts += stats.ScriptCount + totals.totalScriptSize += stats.ScriptSize + } + return totals +} + +func displayStatsTableRows(statsList []*WorkflowStats, displayCount int, maxDisplay int) [][]string { rows := make([][]string, 0, displayCount) for i, stats := range statsList { if i >= maxDisplay { @@ -307,25 +352,17 @@ func displayStatsTable(statsList []*WorkflowStats) { strconv.Itoa(stats.ScriptCount), }) } + return rows +} - // Create table config - tableConfig := console.TableConfig{ - Title: "", - Headers: []string{"WORKFLOW", "FILE SIZE", "JOBS", "STEPS", "SCRIPTS"}, - Rows: rows, - } - - // Render and print table - fmt.Fprint(os.Stderr, console.RenderTable(tableConfig)) - - // Print summary +func displayStatsTableSummary(statsList []*WorkflowStats, maxDisplay int, totals displayStatsTableTotalValues) { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Summary:")) if len(statsList) > maxDisplay { fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("Showing top %d of %d workflows (sorted by size)", maxDisplay, len(statsList)))) } fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("Total workflows: %d", len(statsList)))) - fmt.Fprintln(os.Stderr, console.FormatListItem("Total size: "+console.FormatFileSize(totalSize))) - fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("Total jobs: %d", totalJobs))) - fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("Total steps: %d", totalSteps))) - fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("Total scripts: %d (%s)", totalScripts, console.FormatFileSize(int64(totalScriptSize))))) + fmt.Fprintln(os.Stderr, console.FormatListItem("Total size: "+console.FormatFileSize(totals.totalSize))) + fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("Total jobs: %d", totals.totalJobs))) + fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("Total steps: %d", totals.totalSteps))) + fmt.Fprintln(os.Stderr, console.FormatListItem(fmt.Sprintf("Total scripts: %d (%s)", totals.totalScripts, console.FormatFileSize(int64(totals.totalScriptSize))))) } diff --git a/pkg/cli/compile_validation.go b/pkg/cli/compile_validation.go index fe79d0f991b..6152a533488 100644 --- a/pkg/cli/compile_validation.go +++ b/pkg/cli/compile_validation.go @@ -32,7 +32,34 @@ func CompileWorkflowWithValidation(ctx context.Context, compiler *workflow.Compi compileValidationLog.Printf("Compiling workflow with validation: file=%s, strict=%v, validateSHAs=%v", filePath, opts.Strict, opts.ValidateActionSHAs) compiler.SetContext(ctx) + compileWorkflowWithValidationSetWorkflowMetadata(compiler, filePath) + // Compile the workflow first + if err := compiler.CompileWorkflow(filePath); err != nil { + compileValidationLog.Printf("Workflow compilation failed: %v", err) + return err + } + + return compileWorkflowWithValidationValidateGeneratedLock(ctx, compiler, filePath, opts) +} + +// CompileWorkflowDataWithValidation compiles from already-parsed WorkflowData with validation +// This avoids re-parsing when the workflow data has already been parsed +func CompileWorkflowDataWithValidation(ctx context.Context, compiler *workflow.Compiler, workflowData *workflow.WorkflowData, filePath string, opts CompileValidationOptions) error { + compileValidationLog.Printf("Compiling from parsed WorkflowData: file=%s", filePath) + + compiler.SetContext(ctx) + + // Compile the workflow using already-parsed data + if err := compiler.CompileWorkflowData(workflowData, filePath); err != nil { + compileValidationLog.Printf("WorkflowData compilation failed: %v", err) + return err + } + + return compileWorkflowDataWithValidationValidateGeneratedLock(ctx, compiler, filePath, opts) +} + +func compileWorkflowWithValidationSetWorkflowMetadata(compiler *workflow.Compiler, filePath string) { // Set workflow identifier for schedule scattering (use repository-relative path for stability) relPath, err := getRepositoryRelativePath(filePath) if err != nil { @@ -53,137 +80,115 @@ func CompileWorkflowWithValidation(ctx context.Context, compiler *workflow.Compi compileValidationLog.Printf("Repository slug for file set: %s", fileRepoSlug) } } +} - // Compile the workflow first - if err := compiler.CompileWorkflow(filePath); err != nil { - compileValidationLog.Printf("Workflow compilation failed: %v", err) +func compileWorkflowWithValidationValidateGeneratedLock(ctx context.Context, compiler *workflow.Compiler, filePath string, opts CompileValidationOptions) error { + lockFile, err := compileWorkflowWithValidationReadGeneratedLock(filePath) + if err != nil || lockFile == "" { return err } + return compileWorkflowWithValidationRunGeneratedLockChecks(ctx, compiler, lockFile, opts) +} +func compileWorkflowWithValidationReadGeneratedLock(filePath string) (string, error) { // Always validate that the generated lock file is valid YAML (CLI requirement) lockFile := stringutil.MarkdownToLockFile(filePath) if _, err := os.Stat(lockFile); err != nil { compileValidationLog.Print("Lock file not found, skipping validation (likely no-emit mode)") // Lock file doesn't exist (likely due to no-emit), skip YAML validation - return nil + return "", nil } compileValidationLog.Print("Validating generated lock file YAML syntax") - lockContent, err := os.ReadFile(lockFile) if err != nil { - return fmt.Errorf("failed to read generated lock file for validation: %w", err) + return "", fmt.Errorf("failed to read generated lock file for validation: %w", err) } - - // Validate the lock file is valid YAML var yamlValidationTest any if err := yaml.Unmarshal(lockContent, &yamlValidationTest); err != nil { - return fmt.Errorf("generated lock file is not valid YAML: %w", err) + return "", fmt.Errorf("generated lock file is not valid YAML: %w", err) } + return lockFile, nil +} - // Validate action SHAs if requested +func compileWorkflowWithValidationRunGeneratedLockChecks(ctx context.Context, compiler *workflow.Compiler, lockFile string, opts CompileValidationOptions) error { if opts.ValidateActionSHAs { compileValidationLog.Print("Validating action SHAs in lock file") - // Use the compiler's shared action cache to benefit from cached resolutions actionCache := compiler.GetSharedActionCache() if err := workflow.ValidateActionSHAsInLockFile(ctx, lockFile, actionCache, opts.Verbose); err != nil { - // Action SHA validation warnings are non-fatal compileValidationLog.Printf("Action SHA validation completed with warnings: %v", err) } } - - // Run zizmor on the generated lock file if requested if opts.RunZizmorPerFile { if err := runZizmorOnFile(lockFile, opts.Verbose, opts.Strict); err != nil { return fmt.Errorf("zizmor security scan failed: %w", err) } } - - // Run poutine on the generated lock file if requested if opts.RunPoutinePerFile { if err := runPoutineOnFile(lockFile, opts.Verbose, opts.Strict); err != nil { return fmt.Errorf("poutine security scan failed: %w", err) } } - - // Run actionlint on the generated lock file if requested - // Note: For batch processing, use RunActionlintOnFiles instead if opts.RunActionlintPerFile { if err := runActionlintOnFiles(ctx, []string{lockFile}, opts.Verbose, opts.Strict); err != nil { return fmt.Errorf("actionlint linter failed: %w", err) } } - return nil } -// CompileWorkflowDataWithValidation compiles from already-parsed WorkflowData with validation -// This avoids re-parsing when the workflow data has already been parsed -func CompileWorkflowDataWithValidation(ctx context.Context, compiler *workflow.Compiler, workflowData *workflow.WorkflowData, filePath string, opts CompileValidationOptions) error { - compileValidationLog.Printf("Compiling from parsed WorkflowData: file=%s", filePath) - - compiler.SetContext(ctx) - - // Compile the workflow using already-parsed data - if err := compiler.CompileWorkflowData(workflowData, filePath); err != nil { - compileValidationLog.Printf("WorkflowData compilation failed: %v", err) +func compileWorkflowDataWithValidationValidateGeneratedLock(ctx context.Context, compiler *workflow.Compiler, filePath string, opts CompileValidationOptions) error { + lockFile, err := compileWorkflowDataWithValidationReadGeneratedLock(filePath) + if err != nil || lockFile == "" { return err } + return compileWorkflowDataWithValidationRunGeneratedLockChecks(ctx, compiler, lockFile, opts) +} +func compileWorkflowDataWithValidationReadGeneratedLock(filePath string) (string, error) { // Always validate that the generated lock file is valid YAML (CLI requirement) lockFile := stringutil.MarkdownToLockFile(filePath) if _, err := os.Stat(lockFile); err != nil { compileValidationLog.Print("Lock file not found, skipping validation (likely no-emit mode)") // Lock file doesn't exist (likely due to no-emit), skip YAML validation - return nil + return "", nil } compileValidationLog.Print("Validating generated lock file YAML syntax") - lockContent, err := os.ReadFile(lockFile) if err != nil { - return fmt.Errorf("failed to read generated lock file for validation: %w", err) + return "", fmt.Errorf("failed to read generated lock file for validation: %w", err) } - - // Validate the lock file is valid YAML var yamlValidationTest any if err := yaml.Unmarshal(lockContent, &yamlValidationTest); err != nil { - return fmt.Errorf("generated lock file is not valid YAML: %w", err) + return "", fmt.Errorf("generated lock file is not valid YAML: %w", err) } + return lockFile, nil +} - // Validate action SHAs if requested +func compileWorkflowDataWithValidationRunGeneratedLockChecks(ctx context.Context, compiler *workflow.Compiler, lockFile string, opts CompileValidationOptions) error { if opts.ValidateActionSHAs { compileValidationLog.Print("Validating action SHAs in lock file") - // Use the compiler's shared action cache to benefit from cached resolutions actionCache := compiler.GetSharedActionCache() if err := workflow.ValidateActionSHAsInLockFile(ctx, lockFile, actionCache, opts.Verbose); err != nil { - // Action SHA validation warnings are non-fatal compileValidationLog.Printf("Action SHA validation completed with warnings: %v", err) } } - - // Run zizmor on the generated lock file if requested if opts.RunZizmorPerFile { if err := runZizmorOnFile(lockFile, opts.Verbose, opts.Strict); err != nil { return fmt.Errorf("zizmor security scan failed: %w", err) } } - - // Run poutine on the generated lock file if requested if opts.RunPoutinePerFile { if err := runPoutineOnFile(lockFile, opts.Verbose, opts.Strict); err != nil { return fmt.Errorf("poutine security scan failed: %w", err) } } - - // Run actionlint on the generated lock file if requested - // Note: For batch processing, use RunActionlintOnFiles instead if opts.RunActionlintPerFile { if err := runActionlintOnFiles(ctx, []string{lockFile}, opts.Verbose, opts.Strict); err != nil { return fmt.Errorf("actionlint linter failed: %w", err) } } - return nil } diff --git a/pkg/cli/compile_watch.go b/pkg/cli/compile_watch.go index 6f85b782a93..a321b96a456 100644 --- a/pkg/cli/compile_watch.go +++ b/pkg/cli/compile_watch.go @@ -23,28 +23,44 @@ var compileWatchLog = logger.New("cli:compile_watch") // watchAndCompileWorkflows watches for changes to workflow files and recompiles them automatically func watchAndCompileWorkflows(ctx context.Context, markdownFile string, compiler *workflow.Compiler, verbose bool) error { - // Find git root for consistent behavior - gitRoot, err := gitutil.FindGitRoot() + _, workflowsDir, markdownFile, err := watchAndCompileWorkflowsInputs(markdownFile) + if err != nil { + return err + } + + depGraph := watchAndCompileWorkflowsBuildDependencyGraph(workflowsDir, compiler, verbose) + watcher, err := watchAndCompileWorkflowsCreateWatcher(workflowsDir) if err != nil { - return fmt.Errorf("watch mode requires being in a git repository: %w", err) + return err } + defer watcher.Close() + + watchAndCompileWorkflowsPrintBegin(markdownFile, workflowsDir, verbose) + watchAndCompileWorkflowsInitial(ctx, markdownFile, compiler, workflowsDir, verbose) + return watchAndCompileWorkflowsLoop(ctx, markdownFile, compiler, verbose, watcher, depGraph) +} +func watchAndCompileWorkflowsInputs(markdownFile string) (string, string, string, error) { + gitRoot, err := gitutil.FindGitRoot() + if err != nil { + return "", "", "", fmt.Errorf("watch mode requires being in a git repository: %w", err) + } workflowsDir := filepath.Join(gitRoot, constants.GetWorkflowDir()) if _, err := os.Stat(workflowsDir); os.IsNotExist(err) { - return fmt.Errorf("the %s directory does not exist in git root (%s)", constants.GetWorkflowDir(), gitRoot) + return "", "", "", fmt.Errorf("the %s directory does not exist in git root (%s)", constants.GetWorkflowDir(), gitRoot) } - - // If a specific file is provided, watch only that file and its directory if markdownFile != "" { if !filepath.IsAbs(markdownFile) { markdownFile = filepath.Join(workflowsDir, markdownFile) } if _, err := os.Stat(markdownFile); os.IsNotExist(err) { - return fmt.Errorf("specified markdown file does not exist: %s", markdownFile) + return "", "", "", fmt.Errorf("specified markdown file does not exist: %s", markdownFile) } } + return gitRoot, workflowsDir, markdownFile, nil +} - // Build dependency graph for intelligent recompilation +func watchAndCompileWorkflowsBuildDependencyGraph(workflowsDir string, compiler *workflow.Compiler, verbose bool) *DependencyGraph { depGraph := NewDependencyGraph(workflowsDir) compileWatchLog.Print("Building dependency graph for watch mode...") if err := depGraph.BuildGraph(compiler); err != nil { @@ -56,36 +72,38 @@ func watchAndCompileWorkflows(ctx context.Context, markdownFile string, compiler fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Dependency graph built: %d workflows", len(depGraph.nodes)))) } } + return depGraph +} - // Set up file system watcher with buffered events for better handling of burst activity +func watchAndCompileWorkflowsCreateWatcher(workflowsDir string) (*fsnotify.Watcher, error) { watcher, err := fsnotify.NewBufferedWatcher(100) if err != nil { - return fmt.Errorf("failed to create file watcher: %w", err) + return nil, fmt.Errorf("failed to create file watcher: %w", err) } - defer watcher.Close() - - // addWatchPath adds a path to the watcher with platform-specific configuration. - // On Windows, uses a larger buffer (64KB) to prevent event overflow in busy directories. - addWatchPath := func(path string) error { - if runtime.GOOS == "windows" { - return watcher.AddWith(path, fsnotify.WithBufferSize(64*1024)) - } - return watcher.Add(path) + if err := watchAndCompileWorkflowsAddWatchPath(watcher, workflowsDir); err != nil { + watcher.Close() + return nil, fmt.Errorf("failed to watch directory %s: %w", workflowsDir, err) + } + if err := watchAndCompileWorkflowsWatchSubdirectories(watcher, workflowsDir); err != nil { + compileWatchLog.Printf("Failed to walk subdirectories: %v", err) } + return watcher, nil +} - // Add the workflows directory to the watcher - if err := addWatchPath(workflowsDir); err != nil { - return fmt.Errorf("failed to watch directory %s: %w", workflowsDir, err) +func watchAndCompileWorkflowsAddWatchPath(watcher *fsnotify.Watcher, path string) error { + if runtime.GOOS == "windows" { + return watcher.AddWith(path, fsnotify.WithBufferSize(64*1024)) } + return watcher.Add(path) +} - // Also watch subdirectories for include files (recursive watching) - err = filepath.Walk(workflowsDir, func(path string, info os.FileInfo, err error) error { +func watchAndCompileWorkflowsWatchSubdirectories(watcher *fsnotify.Watcher, workflowsDir string) error { + return filepath.Walk(workflowsDir, func(path string, info os.FileInfo, err error) error { if err != nil { - return nil // Skip errors but continue walking + return nil } if info.IsDir() && path != workflowsDir { - // Add subdirectories to the watcher - if err := addWatchPath(path); err != nil { + if err := watchAndCompileWorkflowsAddWatchPath(watcher, path); err != nil { compileWatchLog.Printf("Failed to watch subdirectory %s: %v", path, err) } else { compileWatchLog.Printf("Watching subdirectory: %s", path) @@ -93,28 +111,20 @@ func watchAndCompileWorkflows(ctx context.Context, markdownFile string, compiler } return nil }) - if err != nil { - compileWatchLog.Printf("Failed to walk subdirectories: %v", err) - } +} - // Always emit the begin pattern for task integration +func watchAndCompileWorkflowsPrintBegin(markdownFile string, workflowsDir string, verbose bool) { if markdownFile != "" { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Watching for file changes to %s...", markdownFile))) } else { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Watching for file changes in %s...", workflowsDir))) } - if verbose { fmt.Fprintln(os.Stderr, "Press Ctrl+C to stop watching.") } +} - // Debouncing setup - const debounceDelay = 300 * time.Millisecond - var debounceTimer *time.Timer - var debounceMu sync.Mutex - modifiedFiles := make(map[string]struct{}) - - // Compile initially if no specific file provided +func watchAndCompileWorkflowsInitial(ctx context.Context, markdownFile string, compiler *workflow.Compiler, workflowsDir string, verbose bool) { if markdownFile == "" { fmt.Fprintln(os.Stderr, "Watching for file changes") if verbose { @@ -122,115 +132,161 @@ func watchAndCompileWorkflows(ctx context.Context, markdownFile string, compiler } stats, err := compileAllWorkflowFiles(ctx, compiler, workflowsDir, verbose) if err != nil { - // Always show initial compilation errors, not just in verbose mode fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Initial compilation failed: %v", err))) } - // Print summary instead of just "Recompiled" printCompilationSummary(stats, false) - } else { - // Reset warning count before compilation - compiler.ResetWarningCount() - - // Track compilation statistics for single file - stats := &CompilationStats{} - - fmt.Fprintln(os.Stderr, "Watching for file changes") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Initial compilation of %s...", markdownFile))) - } - - // Use compileSingleFile to handle both regular workflows and campaign files - compileSingleFile(ctx, compiler, markdownFile, stats, verbose, false) - - // Get warning count from compiler - stats.Warnings = compiler.GetWarningCount() + return + } + watchAndCompileWorkflowsInitialSingle(ctx, markdownFile, compiler, verbose) +} - // Print summary instead of just "Recompiled" - printCompilationSummary(stats, false) +func watchAndCompileWorkflowsInitialSingle(ctx context.Context, markdownFile string, compiler *workflow.Compiler, verbose bool) { + compiler.ResetWarningCount() + stats := &CompilationStats{} + fmt.Fprintln(os.Stderr, "Watching for file changes") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Initial compilation of %s...", markdownFile))) } + compileSingleFile(ctx, compiler, markdownFile, stats, verbose, false) + stats.Warnings = compiler.GetWarningCount() + printCompilationSummary(stats, false) +} + +func watchAndCompileWorkflowsLoop( + ctx context.Context, + markdownFile string, + compiler *workflow.Compiler, + verbose bool, + watcher *fsnotify.Watcher, + depGraph *DependencyGraph, +) error { + const debounceDelay = 300 * time.Millisecond + var debounceTimer *time.Timer + var debounceMu sync.Mutex + modifiedFiles := make(map[string]struct{}) - // Main watch loop for { select { case event, ok := <-watcher.Events: if !ok { return errors.New("watcher channel closed") } - - // Filter out Chmod events (noisy and usually not useful for workflow changes) - if event.Has(fsnotify.Chmod) { - continue - } - - // Only process markdown files and ignore lock files - if !strings.HasSuffix(event.Name, ".md") { - continue - } - - // If watching a specific file, only process that file - if markdownFile != "" && event.Name != markdownFile { - continue - } - - compileWatchLog.Printf("Detected change: %s (%s)", event.Name, event.Op.String()) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Detected change: %s (%s)", event.Name, event.Op.String()))) - } - - // Handle file operations - switch { - case event.Has(fsnotify.Remove): - // Handle file deletion - handleFileDeleted(event.Name, verbose) - // Remove from dependency graph - depGraph.RemoveWorkflow(event.Name) - case event.Has(fsnotify.Write) || event.Has(fsnotify.Create): - // Handle file modification or creation - add to debounced compilation - func() { - debounceMu.Lock() - defer debounceMu.Unlock() - modifiedFiles[event.Name] = struct{}{} - - // Reset debounce timer - if debounceTimer != nil { - debounceTimer.Stop() - } - debounceTimer = time.AfterFunc(debounceDelay, func() { - filesToCompile := func() []string { - debounceMu.Lock() - defer debounceMu.Unlock() - files := make([]string, 0, len(modifiedFiles)) - for file := range modifiedFiles { - files = append(files, file) - } - // Clear the modifiedFiles map - modifiedFiles = make(map[string]struct{}) - return files - }() - - // Compile the modified files using dependency graph - compileModifiedFilesWithDependencies(ctx, compiler, depGraph, filesToCompile, verbose) - }) - }() - } - + watchAndCompileWorkflowsHandleEvent(watchAndCompileWorkflowsHandleEventParams{ + Ctx: ctx, + MarkdownFile: markdownFile, + Compiler: compiler, + Verbose: verbose, + DepGraph: depGraph, + Event: event, + DebounceTimer: &debounceTimer, + DebounceMu: &debounceMu, + ModifiedFiles: &modifiedFiles, + DebounceDelay: debounceDelay, + }) case err, ok := <-watcher.Errors: if !ok { return errors.New("watcher error channel closed") } - compileWatchLog.Printf("Watcher error: %v", err) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Watcher error: %v", err))) - } - + watchAndCompileWorkflowsHandleError(err, verbose) case <-ctx.Done(): - if verbose { - fmt.Fprintln(os.Stderr, "\n🛑 Stopping watch mode...") - } - if debounceTimer != nil { - debounceTimer.Stop() - } + watchAndCompileWorkflowsStop(debounceTimer, verbose) return nil } } } + +type watchAndCompileWorkflowsHandleEventParams struct { + Ctx context.Context + MarkdownFile string + Compiler *workflow.Compiler + Verbose bool + DepGraph *DependencyGraph + Event fsnotify.Event + DebounceTimer **time.Timer + DebounceMu *sync.Mutex + ModifiedFiles *map[string]struct{} + DebounceDelay time.Duration +} + +func watchAndCompileWorkflowsHandleEvent(p watchAndCompileWorkflowsHandleEventParams) { + if p.Event.Has(fsnotify.Chmod) || !strings.HasSuffix(p.Event.Name, ".md") { + return + } + if p.MarkdownFile != "" && p.Event.Name != p.MarkdownFile { + return + } + compileWatchLog.Printf("Detected change: %s (%s)", p.Event.Name, p.Event.Op.String()) + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Detected change: %s (%s)", p.Event.Name, p.Event.Op.String()))) + } + if p.Event.Has(fsnotify.Remove) { + handleFileDeleted(p.Event.Name, p.Verbose) + p.DepGraph.RemoveWorkflow(p.Event.Name) + return + } + if p.Event.Has(fsnotify.Write) || p.Event.Has(fsnotify.Create) { + watchAndCompileWorkflowsDebounce(watchAndCompileWorkflowsDebounceParams{ + Ctx: p.Ctx, + Compiler: p.Compiler, + Verbose: p.Verbose, + DepGraph: p.DepGraph, + FileName: p.Event.Name, + DebounceTimer: p.DebounceTimer, + DebounceMu: p.DebounceMu, + ModifiedFiles: p.ModifiedFiles, + DebounceDelay: p.DebounceDelay, + }) + } +} + +type watchAndCompileWorkflowsDebounceParams struct { + Ctx context.Context + Compiler *workflow.Compiler + Verbose bool + DepGraph *DependencyGraph + FileName string + DebounceTimer **time.Timer + DebounceMu *sync.Mutex + ModifiedFiles *map[string]struct{} + DebounceDelay time.Duration +} + +func watchAndCompileWorkflowsDebounce(p watchAndCompileWorkflowsDebounceParams) { + p.DebounceMu.Lock() + defer p.DebounceMu.Unlock() + (*p.ModifiedFiles)[p.FileName] = struct{}{} + if *p.DebounceTimer != nil { + (*p.DebounceTimer).Stop() + } + *p.DebounceTimer = time.AfterFunc(p.DebounceDelay, func() { + filesToCompile := watchAndCompileWorkflowsTakeModifiedFiles(p.DebounceMu, p.ModifiedFiles) + compileModifiedFilesWithDependencies(p.Ctx, p.Compiler, p.DepGraph, filesToCompile, p.Verbose) + }) +} + +func watchAndCompileWorkflowsTakeModifiedFiles(debounceMu *sync.Mutex, modifiedFiles *map[string]struct{}) []string { + debounceMu.Lock() + defer debounceMu.Unlock() + files := make([]string, 0, len(*modifiedFiles)) + for file := range *modifiedFiles { + files = append(files, file) + } + *modifiedFiles = make(map[string]struct{}) + return files +} + +func watchAndCompileWorkflowsHandleError(err error, verbose bool) { + compileWatchLog.Printf("Watcher error: %v", err) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Watcher error: %v", err))) + } +} + +func watchAndCompileWorkflowsStop(debounceTimer *time.Timer, verbose bool) { + if verbose { + fmt.Fprintln(os.Stderr, "\n🛑 Stopping watch mode...") + } + if debounceTimer != nil { + debounceTimer.Stop() + } +} diff --git a/pkg/cli/compile_workflow_processor.go b/pkg/cli/compile_workflow_processor.go index a3fd8ab1819..785806009b8 100644 --- a/pkg/cli/compile_workflow_processor.go +++ b/pkg/cli/compile_workflow_processor.go @@ -76,7 +76,38 @@ func compileWorkflowFile( ) compileWorkflowFileResult { compileWorkflowProcessorLog.Printf("Processing workflow file: %s", resolvedFile) - result := compileWorkflowFileResult{ + result := compileWorkflowFileNewResult(resolvedFile) + + // Generate lock file name + lockFile := stringutil.MarkdownToLockFile(resolvedFile) + result.lockFile = lockFile + + // Parse workflow file to get data + compileWorkflowProcessorLog.Printf("Parsing workflow file: %s", resolvedFile) + compileWorkflowFileConfigureCompiler(compiler, resolvedFile) + + workflowData, handled := compileWorkflowFileParse(compiler, resolvedFile, opts, &result) + if handled { + return result + } + result.workflowData = workflowData + + compileWorkflowProcessorLog.Printf("Starting compilation of %s", resolvedFile) + if err := compileWorkflowFileCompile(ctx, compiler, workflowData, resolvedFile, opts); err != nil { + // Don't print error here - it will be displayed in the compilation summary + // The error is stored in ValidationResult for JSON output and summary display + result.validationResult.Valid = false + result.validationResult.Errors = appendValidationErrors(result.validationResult.Errors, "compilation_error", err) + return result + } + + compileWorkflowFileMarkSuccess(&result, opts, lockFile, workflowData) + compileWorkflowProcessorLog.Printf("Successfully processed workflow file: %s", resolvedFile) + return result +} + +func compileWorkflowFileNewResult(resolvedFile string) compileWorkflowFileResult { + return compileWorkflowFileResult{ validationResult: ValidationResult{ Workflow: filepath.Base(resolvedFile), Valid: true, @@ -85,14 +116,9 @@ func compileWorkflowFile( }, success: false, } +} - // Generate lock file name - lockFile := stringutil.MarkdownToLockFile(resolvedFile) - result.lockFile = lockFile - - // Parse workflow file to get data - compileWorkflowProcessorLog.Printf("Parsing workflow file: %s", resolvedFile) - +func compileWorkflowFileConfigureCompiler(compiler *workflow.Compiler, resolvedFile string) { // Set workflow identifier for schedule scattering (use repository-relative path for stability) relPath, err := getRepositoryRelativePath(resolvedFile) if err != nil { @@ -113,7 +139,14 @@ func compileWorkflowFile( compileWorkflowProcessorLog.Printf("Repository slug for file set: %s", fileRepoSlug) } } +} +func compileWorkflowFileParse( + compiler *workflow.Compiler, + resolvedFile string, + opts compileWorkflowFileOptions, + result *compileWorkflowFileResult, +) (*workflow.WorkflowData, bool) { // Parse the workflow workflowData, err := compiler.ParseWorkflowFile(resolvedFile) if err != nil { @@ -131,7 +164,7 @@ func compileWorkflowFile( Message: "Skipped: Shared workflow component (missing 'on' field)", }) result.success = true // Consider it successful, just skipped - return result + return nil, true } // Check if this is a redirect-only workflow (not an error, just info) @@ -148,19 +181,25 @@ func compileWorkflowFile( Message: "Skipped: Redirect-only workflow (missing 'on' field, has redirect)", }) result.success = true // Consider it successful, just skipped - return result + return nil, true } // Don't print error here - it will be displayed in the compilation summary // The error is stored in ValidationResult for JSON output and summary display result.validationResult.Valid = false result.validationResult.Errors = appendValidationErrors(result.validationResult.Errors, "parse_error", err) - return result + return nil, true } - result.workflowData = workflowData - - compileWorkflowProcessorLog.Printf("Starting compilation of %s", resolvedFile) + return workflowData, false +} +func compileWorkflowFileCompile( + ctx context.Context, + compiler *workflow.Compiler, + workflowData *workflow.WorkflowData, + resolvedFile string, + opts compileWorkflowFileOptions, +) error { // Compile the workflow // Per-file actionlint is always disabled here; actionlint runs in batch after all files are compiled. if err := CompileWorkflowDataWithValidation(ctx, compiler, workflowData, resolvedFile, CompileValidationOptions{ @@ -170,13 +209,12 @@ func compileWorkflowFile( Strict: opts.strict, ValidateActionSHAs: opts.validate && !opts.noEmit, }); err != nil { - // Don't print error here - it will be displayed in the compilation summary - // The error is stored in ValidationResult for JSON output and summary display - result.validationResult.Valid = false - result.validationResult.Errors = appendValidationErrors(result.validationResult.Errors, "compilation_error", err) - return result + return err } + return nil +} +func compileWorkflowFileMarkSuccess(result *compileWorkflowFileResult, opts compileWorkflowFileOptions, lockFile string, workflowData *workflow.WorkflowData) { result.success = true if !opts.noEmit { result.validationResult.CompiledFile = lockFile @@ -184,9 +222,6 @@ func compileWorkflowFile( // Collect labels for JSON output (used by create-labels maintenance operation) result.validationResult.Labels = extractSafeOutputLabels(workflowData) - - compileWorkflowProcessorLog.Printf("Successfully processed workflow file: %s", resolvedFile) - return result } // extractSafeOutputLabels collects all unique labels referenced by workflow configuration diff --git a/pkg/cli/completion_command.go b/pkg/cli/completion_command.go index 2eaee013bbc..aa69c7a82a4 100644 --- a/pkg/cli/completion_command.go +++ b/pkg/cli/completion_command.go @@ -12,7 +12,15 @@ var completionLog = logger.New("cli:completion") // NewCompletionCommand creates the completion command with install subcommand func NewCompletionCommand() *cobra.Command { - cmd := &cobra.Command{ + cmd := newCompletionCommandRoot() + cmd.AddCommand(newCompletionCommandInstall()) + cmd.AddCommand(newCompletionCommandUninstall()) + + return cmd +} + +func newCompletionCommandRoot() *cobra.Command { + return &cobra.Command{ Use: "completion [shell]", Short: "Generate shell completion scripts for gh aw commands", Long: `Generate shell completion scripts to enable tab completion for gh aw commands. @@ -47,26 +55,32 @@ Supported shells: bash, zsh, fish, PowerShell`, ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), RunE: func(cmd *cobra.Command, args []string) error { - shell := args[0] - completionLog.Printf("Generating %s completion script", shell) - - switch shell { - case "bash": - return cmd.Root().GenBashCompletion(os.Stdout) - case "zsh": - return cmd.Root().GenZshCompletion(os.Stdout) - case "fish": - return cmd.Root().GenFishCompletion(os.Stdout, true) - case "powershell": - return cmd.Root().GenPowerShellCompletion(os.Stdout) - default: - return fmt.Errorf("unsupported shell: %s", shell) - } + return newCompletionCommandRun(cmd, args) }, } +} + +func newCompletionCommandRun(cmd *cobra.Command, args []string) error { + shell := args[0] + completionLog.Printf("Generating %s completion script", shell) + + switch shell { + case "bash": + return cmd.Root().GenBashCompletion(os.Stdout) + case "zsh": + return cmd.Root().GenZshCompletion(os.Stdout) + case "fish": + return cmd.Root().GenFishCompletion(os.Stdout, true) + case "powershell": + return cmd.Root().GenPowerShellCompletion(os.Stdout) + default: + return fmt.Errorf("unsupported shell: %s", shell) + } +} - // Add install subcommand - installCmd := &cobra.Command{ +// Add install subcommand +func newCompletionCommandInstall() *cobra.Command { + return &cobra.Command{ Use: "install", Short: "Install shell completion for the detected shell", Long: `Automatically install shell completion for your current shell. @@ -88,9 +102,11 @@ Supported shells: return InstallShellCompletion(verbose, cmd.Root()) }, } +} - // Add uninstall subcommand - uninstallCmd := &cobra.Command{ +// Add uninstall subcommand +func newCompletionCommandUninstall() *cobra.Command { + return &cobra.Command{ Use: "uninstall", Short: "Uninstall shell completion for the detected shell", Long: `Automatically uninstall shell completion for your current shell. @@ -112,9 +128,4 @@ Supported shells: return UninstallShellCompletion(verbose) }, } - - cmd.AddCommand(installCmd) - cmd.AddCommand(uninstallCmd) - - return cmd } diff --git a/pkg/cli/copilot_agent_logs.go b/pkg/cli/copilot_agent_logs.go index 2442a2b8fb0..22498ef7947 100644 --- a/pkg/cli/copilot_agent_logs.go +++ b/pkg/cli/copilot_agent_logs.go @@ -39,80 +39,108 @@ func ParseCopilotCodingAgentLogMetrics(logContent string, verbose bool) workflow copilotCodingAgentLog.Printf("Parsing GitHub Copilot coding agent log metrics: %d bytes", len(logContent)) var metrics workflow.LogMetrics - var maxTokenUsage int - lines := strings.Split(logContent, "\n") toolCallMap := make(map[string]*workflow.ToolCallInfo) var currentSequence []string turns := 0 + maxTokenUsage := 0 for _, line := range lines { - // Skip empty lines - if strings.TrimSpace(line) == "" { - continue - } + parseCopilotCodingAgentLogMetricsLine( + line, + verbose, + &metrics, + toolCallMap, + ¤tSequence, + &turns, + &maxTokenUsage, + ) + } + + parseCopilotCodingAgentLogMetricsFinalize(&metrics, toolCallMap, currentSequence, maxTokenUsage, turns) - // Count turns based on agent iteration patterns - if agentTurnPattern.MatchString(line) { - turns++ - // Start of a new turn, save previous sequence if any - if len(currentSequence) > 0 { - metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) - currentSequence = []string{} - } + copilotCodingAgentLog.Printf("Parsed metrics: tokens=%d, cost=$%.4f, turns=%d", + metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns) + + return metrics +} + +func parseCopilotCodingAgentLogMetricsLine( + line string, + verbose bool, + metrics *workflow.LogMetrics, + toolCallMap map[string]*workflow.ToolCallInfo, + currentSequence *[]string, + turns *int, + maxTokenUsage *int, +) { + if strings.TrimSpace(line) == "" { + return + } + parseCopilotCodingAgentLogMetricsTurn(line, metrics, currentSequence, turns) + parseCopilotCodingAgentLogMetricsTool(line, verbose, toolCallMap, currentSequence) + parseCopilotCodingAgentLogMetricsJSON(line, verbose, metrics, maxTokenUsage) +} + +func parseCopilotCodingAgentLogMetricsTurn(line string, metrics *workflow.LogMetrics, currentSequence *[]string, turns *int) { + if agentTurnPattern.MatchString(line) { + (*turns)++ + // Start of a new turn, save previous sequence if any + if len(*currentSequence) > 0 { + metrics.ToolSequences = append(metrics.ToolSequences, *currentSequence) + *currentSequence = []string{} } + } +} - // Extract tool calls from agent logs - if agentToolCallPattern.MatchString(line) { - toolName := extractToolName(line) - if toolName != "" { - // Track tool call - if _, exists := toolCallMap[toolName]; !exists { - toolCallMap[toolName] = &workflow.ToolCallInfo{ - Name: toolName, - CallCount: 0, - } - } - toolCallMap[toolName].CallCount++ - - // Add to current sequence - currentSequence = append(currentSequence, toolName) - - if verbose { - copilotCodingAgentLog.Printf("Found tool call: %s", toolName) - } - } +func parseCopilotCodingAgentLogMetricsTool(line string, verbose bool, toolCallMap map[string]*workflow.ToolCallInfo, currentSequence *[]string) { + if !agentToolCallPattern.MatchString(line) { + return + } + toolName := extractToolName(line) + if toolName == "" { + return + } + if _, exists := toolCallMap[toolName]; !exists { + toolCallMap[toolName] = &workflow.ToolCallInfo{ + Name: toolName, + CallCount: 0, } + } + toolCallMap[toolName].CallCount++ + *currentSequence = append(*currentSequence, toolName) + if verbose { + copilotCodingAgentLog.Printf("Found tool call: %s", toolName) + } +} - // Try to extract token usage from JSON format if available - jsonMetrics := workflow.ExtractJSONMetrics(line, verbose) - if jsonMetrics.TokenUsage > 0 || jsonMetrics.EstimatedCost > 0 { - if jsonMetrics.TokenUsage > maxTokenUsage { - maxTokenUsage = jsonMetrics.TokenUsage - } - if jsonMetrics.EstimatedCost > 0 { - metrics.EstimatedCost += jsonMetrics.EstimatedCost - } +func parseCopilotCodingAgentLogMetricsJSON(line string, verbose bool, metrics *workflow.LogMetrics, maxTokenUsage *int) { + jsonMetrics := workflow.ExtractJSONMetrics(line, verbose) + if jsonMetrics.TokenUsage > 0 || jsonMetrics.EstimatedCost > 0 { + if jsonMetrics.TokenUsage > *maxTokenUsage { + *maxTokenUsage = jsonMetrics.TokenUsage + } + if jsonMetrics.EstimatedCost > 0 { + metrics.EstimatedCost += jsonMetrics.EstimatedCost } } +} - // Add final sequence if any +func parseCopilotCodingAgentLogMetricsFinalize( + metrics *workflow.LogMetrics, + toolCallMap map[string]*workflow.ToolCallInfo, + currentSequence []string, + maxTokenUsage int, + turns int, +) { if len(currentSequence) > 0 { metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) } - - // Convert tool call map to slice for _, toolInfo := range toolCallMap { metrics.ToolCalls = append(metrics.ToolCalls, *toolInfo) } - metrics.TokenUsage = maxTokenUsage metrics.Turns = turns - - copilotCodingAgentLog.Printf("Parsed metrics: tokens=%d, cost=$%.4f, turns=%d", - metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns) - - return metrics } // extractToolName extracts a tool name from a log line diff --git a/pkg/cli/copilot_events_jsonl.go b/pkg/cli/copilot_events_jsonl.go index 04c9ef3c92c..e0b63bb12ce 100644 --- a/pkg/cli/copilot_events_jsonl.go +++ b/pkg/cli/copilot_events_jsonl.go @@ -186,32 +186,49 @@ func findFileInDir(dir, name string) string { func parseEventsJSONLMetrics(path string, verbose bool) (workflow.LogMetrics, error) { copilotEventsJSONLLog.Printf("Parsing events.jsonl from: %s", path) - var metrics workflow.LogMetrics - // Sanitize path to prevent traversal cleanPath := filepath.Clean(path) file, err := os.Open(cleanPath) if err != nil { - return metrics, fmt.Errorf("failed to open events.jsonl: %w", err) + return workflow.LogMetrics{}, fmt.Errorf("failed to open events.jsonl: %w", err) } defer file.Close() - toolCallMap := make(map[string]*workflow.ToolCallInfo) - var currentSequence []string - turns := 0 - totalTokens := 0 - fallbackTokens := 0 - sawShutdownModelMetrics := false - foundAnyEvent := false - - // Per-turn timestamps used to compute Time Between Turns (TBT) - var turnTimestamps []time.Time - + state := parseEventsJSONLMetricsState{ + toolCallMap: make(map[string]*workflow.ToolCallInfo), + } scanner := bufio.NewScanner(file) buf := make([]byte, maxScannerBufferSize) scanner.Buffer(buf, maxScannerBufferSize) + parseEventsJSONLMetricsScan(scanner, &state) + + if scanErr := scanner.Err(); scanErr != nil { + return state.metrics, fmt.Errorf("error reading events.jsonl: %w", scanErr) + } + + if !state.foundAnyEvent { + return state.metrics, errors.New("no events found in events.jsonl") + } + + parseEventsJSONLMetricsFinalize(&state, verbose) + return state.metrics, nil +} + +type parseEventsJSONLMetricsState struct { + metrics workflow.LogMetrics + toolCallMap map[string]*workflow.ToolCallInfo + currentSequence []string + turns int + totalTokens int + fallbackTokens int + sawShutdownModelMetrics bool + foundAnyEvent bool + turnTimestamps []time.Time +} + +func parseEventsJSONLMetricsScan(scanner *bufio.Scanner, state *parseEventsJSONLMetricsState) { for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || !strings.HasPrefix(line, "{") { @@ -223,127 +240,122 @@ func parseEventsJSONLMetrics(path string, verbose bool) (workflow.LogMetrics, er copilotEventsJSONLLog.Printf("Skipping malformed events.jsonl line: %v", err) continue } - fallbackTokens += extractFallbackEventTokens(entry) - - foundAnyEvent = true - - switch entry.Type { - case "session.start": - copilotEventsJSONLLog.Printf("session.start: sessionId=%s copilotVersion=%s", - entry.Data.SessionID, entry.Data.CopilotVersion) - - case "user.message": - // Each user message represents one conversation turn. - // Save the current tool sequence before starting a new turn. - turns++ - if len(currentSequence) > 0 { - metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) - currentSequence = []string{} - } - // Record the timestamp for TBT computation. - if entry.Timestamp != "" { - if ts, parseErr := time.Parse(time.RFC3339Nano, entry.Timestamp); parseErr == nil { - turnTimestamps = append(turnTimestamps, ts) - } else if ts, parseErr = time.Parse(time.RFC3339, entry.Timestamp); parseErr == nil { - turnTimestamps = append(turnTimestamps, ts) - } - } - copilotEventsJSONLLog.Printf("user.message: turn=%d", turns) - - case "tool.execution_start": - // Record the tool invocation and add to the current turn's sequence. - toolName := entry.Data.ToolName - if toolName != "" { - currentSequence = append(currentSequence, toolName) - if toolInfo, exists := toolCallMap[toolName]; exists { - toolInfo.CallCount++ - } else { - toolCallMap[toolName] = &workflow.ToolCallInfo{ - Name: toolName, - CallCount: 1, - } - } - copilotEventsJSONLLog.Printf("tool.execution_start: %s", toolName) - } + parseEventsJSONLMetricsEntry(entry, state) + } +} - case "session.shutdown": - // Aggregate token usage across all models from modelMetrics. - // Track whether the field was present (even if empty) so the fallback - // is only applied when modelMetrics is truly absent (nil), not when the - // totals happen to sum to zero. - if entry.Data.ModelMetrics != nil { - sawShutdownModelMetrics = true - for model, m := range entry.Data.ModelMetrics { - if m.Usage != nil { - modelTokens := m.Usage.InputTokens + m.Usage.OutputTokens - totalTokens += modelTokens - copilotEventsJSONLLog.Printf("session.shutdown: model=%s inputTokens=%d outputTokens=%d", - model, m.Usage.InputTokens, m.Usage.OutputTokens) - } - } - } - copilotEventsJSONLLog.Printf("session.shutdown: type=%s totalTokens=%d", - entry.Data.ShutdownType, totalTokens) +func parseEventsJSONLMetricsEntry(entry copilotEventsJSONLEntry, state *parseEventsJSONLMetricsState) { + state.fallbackTokens += extractFallbackEventTokens(entry) + state.foundAnyEvent = true + + switch entry.Type { + case "session.start": + copilotEventsJSONLLog.Printf("session.start: sessionId=%s copilotVersion=%s", + entry.Data.SessionID, entry.Data.CopilotVersion) + case "user.message": + parseEventsJSONLMetricsUserMessage(entry, state) + case "tool.execution_start": + parseEventsJSONLMetricsToolStart(entry, state) + case "session.shutdown": + parseEventsJSONLMetricsShutdown(entry, state) + } +} + +func parseEventsJSONLMetricsUserMessage(entry copilotEventsJSONLEntry, state *parseEventsJSONLMetricsState) { + // Each user message represents one conversation turn. + // Save the current tool sequence before starting a new turn. + state.turns++ + if len(state.currentSequence) > 0 { + state.metrics.ToolSequences = append(state.metrics.ToolSequences, state.currentSequence) + state.currentSequence = []string{} + } + if entry.Timestamp != "" { + if ts, parseErr := time.Parse(time.RFC3339Nano, entry.Timestamp); parseErr == nil { + state.turnTimestamps = append(state.turnTimestamps, ts) + } else if ts, parseErr = time.Parse(time.RFC3339, entry.Timestamp); parseErr == nil { + state.turnTimestamps = append(state.turnTimestamps, ts) } } + copilotEventsJSONLLog.Printf("user.message: turn=%d", state.turns) +} - if scanErr := scanner.Err(); scanErr != nil { - return metrics, fmt.Errorf("error reading events.jsonl: %w", scanErr) +func parseEventsJSONLMetricsToolStart(entry copilotEventsJSONLEntry, state *parseEventsJSONLMetricsState) { + toolName := entry.Data.ToolName + if toolName == "" { + return + } + state.currentSequence = append(state.currentSequence, toolName) + if toolInfo, exists := state.toolCallMap[toolName]; exists { + toolInfo.CallCount++ + } else { + state.toolCallMap[toolName] = &workflow.ToolCallInfo{ + Name: toolName, + CallCount: 1, + } } + copilotEventsJSONLLog.Printf("tool.execution_start: %s", toolName) +} - if !foundAnyEvent { - return metrics, errors.New("no events found in events.jsonl") +func parseEventsJSONLMetricsShutdown(entry copilotEventsJSONLEntry, state *parseEventsJSONLMetricsState) { + if entry.Data.ModelMetrics != nil { + state.sawShutdownModelMetrics = true + for model, m := range entry.Data.ModelMetrics { + if m.Usage != nil { + modelTokens := m.Usage.InputTokens + m.Usage.OutputTokens + state.totalTokens += modelTokens + copilotEventsJSONLLog.Printf("session.shutdown: model=%s inputTokens=%d outputTokens=%d", + model, m.Usage.InputTokens, m.Usage.OutputTokens) + } + } } + copilotEventsJSONLLog.Printf("session.shutdown: type=%s totalTokens=%d", + entry.Data.ShutdownType, state.totalTokens) +} - // Flush any remaining tool sequence - if len(currentSequence) > 0 { - metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) +func parseEventsJSONLMetricsFinalize(state *parseEventsJSONLMetricsState, verbose bool) { + if len(state.currentSequence) > 0 { + state.metrics.ToolSequences = append(state.metrics.ToolSequences, state.currentSequence) + } + for _, toolInfo := range state.toolCallMap { + state.metrics.ToolCalls = append(state.metrics.ToolCalls, *toolInfo) } - // Convert tool call map to slice - for _, toolInfo := range toolCallMap { - metrics.ToolCalls = append(metrics.ToolCalls, *toolInfo) + state.metrics.TokenUsage = state.totalTokens + if !state.sawShutdownModelMetrics && state.fallbackTokens > 0 { + state.metrics.TokenUsage = state.fallbackTokens } + state.metrics.Turns = state.turns + parseEventsJSONLMetricsComputeTBT(state) + + copilotEventsJSONLLog.Printf("Parsed events.jsonl: turns=%d totalTokens=%d toolCalls=%d sequences=%d", + state.turns, state.totalTokens, len(state.toolCallMap), len(state.metrics.ToolSequences)) - metrics.TokenUsage = totalTokens - if !sawShutdownModelMetrics && fallbackTokens > 0 { - metrics.TokenUsage = fallbackTokens + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage( + fmt.Sprintf("Parsed events.jsonl: %d turns, %d tokens, %d tool calls", + state.turns, state.totalTokens, len(state.toolCallMap)))) } - metrics.Turns = turns +} - // Compute Time Between Turns (TBT) from per-turn timestamps. - // TBT[i] = timestamp[i] - timestamp[i-1] for i > 0. Two or more timestamps - // are required to measure at least one interval. Only positive intervals are - // included so that identical or out-of-order timestamps don't skew the statistics. - if len(turnTimestamps) >= 2 { +func parseEventsJSONLMetricsComputeTBT(state *parseEventsJSONLMetricsState) { + if len(state.turnTimestamps) >= 2 { var tbtStats stats.StatVar - for i := 1; i < len(turnTimestamps); i++ { - tbt := turnTimestamps[i].Sub(turnTimestamps[i-1]) + for i := 1; i < len(state.turnTimestamps); i++ { + tbt := state.turnTimestamps[i].Sub(state.turnTimestamps[i-1]) if tbt > 0 { tbtStats.Add(float64(tbt)) } } if tbtStats.Count() > 0 { - metrics.AvgTimeBetweenTurns = time.Duration(tbtStats.Mean()) - metrics.MaxTimeBetweenTurns = time.Duration(tbtStats.Max()) - metrics.MedianTimeBetweenTurns = time.Duration(tbtStats.Median()) - metrics.StdDevTimeBetweenTurns = time.Duration(tbtStats.SampleStdDev()) + state.metrics.AvgTimeBetweenTurns = time.Duration(tbtStats.Mean()) + state.metrics.MaxTimeBetweenTurns = time.Duration(tbtStats.Max()) + state.metrics.MedianTimeBetweenTurns = time.Duration(tbtStats.Median()) + state.metrics.StdDevTimeBetweenTurns = time.Duration(tbtStats.SampleStdDev()) copilotEventsJSONLLog.Printf("TBT computed: avg=%s max=%s median=%s stddev=%s intervals=%d", - metrics.AvgTimeBetweenTurns, metrics.MaxTimeBetweenTurns, - metrics.MedianTimeBetweenTurns, metrics.StdDevTimeBetweenTurns, tbtStats.Count()) + state.metrics.AvgTimeBetweenTurns, state.metrics.MaxTimeBetweenTurns, + state.metrics.MedianTimeBetweenTurns, state.metrics.StdDevTimeBetweenTurns, tbtStats.Count()) } } - - copilotEventsJSONLLog.Printf("Parsed events.jsonl: turns=%d totalTokens=%d toolCalls=%d sequences=%d", - turns, totalTokens, len(toolCallMap), len(metrics.ToolSequences)) - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage( - fmt.Sprintf("Parsed events.jsonl: %d turns, %d tokens, %d tool calls", - turns, totalTokens, len(toolCallMap)))) - } - - return metrics, nil } func extractFallbackEventTokens(entry copilotEventsJSONLEntry) int { diff --git a/pkg/cli/copilot_setup.go b/pkg/cli/copilot_setup.go index 2926fd92309..bf13f9f4de3 100644 --- a/pkg/cli/copilot_setup.go +++ b/pkg/cli/copilot_setup.go @@ -101,9 +101,16 @@ func generateCopilotSetupStepsYAML(ctx context.Context, actionMode workflow.Acti actionRef := getActionRef(ctx, actionMode, version, resolver) if actionMode.IsRelease() || actionMode.IsAction() { - // Determine the action repo based on mode - actionRepo := "github/gh-aw-actions/setup-cli" - return fmt.Sprintf(`name: "Copilot Setup Steps" + return generateCopilotSetupStepsYAMLActionMode(actionRef, version) + } + + return generateCopilotSetupStepsYAMLDevMode(ctx) +} + +func generateCopilotSetupStepsYAMLActionMode(actionRef string, version string) string { + // Determine the action repo based on mode + actionRepo := "github/gh-aw-actions/setup-cli" + return fmt.Sprintf(`name: "Copilot Setup Steps" # This workflow configures the environment for GitHub Copilot Agent with gh-aw MCP server on: @@ -130,8 +137,9 @@ jobs: with: version: %s `, actionRepo, actionRef, version) - } +} +func generateCopilotSetupStepsYAMLDevMode(ctx context.Context) string { // Default (dev/script mode): try to resolve the main branch to a pinned SHA so the // downloaded script is immutable; fall back to the mutable branch ref if unavailable. installRef := "refs/heads/main" @@ -243,14 +251,7 @@ func upgradeCopilotSetupSteps(ctx context.Context, verbose bool, actionMode work func ensureCopilotSetupStepsWithUpgrade(ctx context.Context, verbose bool, actionMode workflow.ActionMode, version string, upgradeVersion bool) error { copilotSetupLog.Printf("Creating copilot-setup-steps.yml with action mode: %s, version: %s, upgradeVersion: %v", actionMode, version, upgradeVersion) - // Create a SHA resolver for release/action mode to enable SHA-pinned action references - var resolver workflow.SHAResolver - if actionMode.IsRelease() || actionMode.IsAction() { - cache := workflow.NewActionCache(".") - _ = cache.Load() // Ignore errors if cache doesn't exist yet - resolver = workflow.NewActionResolver(cache) - } - + resolver := ensureCopilotSetupStepsWithUpgradeResolver(actionMode) workflowsDir := constants.GetWorkflowDir() setupStepsPath := filepath.Join(workflowsDir, "copilot-setup-steps.yml") if err := fileutil.EnsureParentDir(setupStepsPath, constants.DirPermPublic); err != nil { @@ -260,70 +261,96 @@ func ensureCopilotSetupStepsWithUpgrade(ctx context.Context, verbose bool, actio // Check if file already exists if _, err := os.Stat(setupStepsPath); err == nil { - copilotSetupLog.Printf("File already exists: %s", setupStepsPath) + return ensureCopilotSetupStepsWithUpgradeExisting(ctx, verbose, actionMode, version, upgradeVersion, resolver, setupStepsPath) + } - // Read existing file to check if extension install step exists - content, err := os.ReadFile(setupStepsPath) - if err != nil { - return fmt.Errorf("failed to read existing copilot-setup-steps.yml: %w", err) - } + // File doesn't exist - create it + if err := os.WriteFile(setupStepsPath, []byte(generateCopilotSetupStepsYAML(ctx, actionMode, version, resolver)), constants.FilePermSensitive); err != nil { + return fmt.Errorf("failed to write copilot-setup-steps.yml: %w", err) + } + copilotSetupLog.Printf("Created file: %s", setupStepsPath) - // Check if the extension install step is already present (check for both modes) - contentStr := string(content) - hasLegacyInstall := strings.Contains(contentStr, "install-gh-aw.sh") || - (strings.Contains(contentStr, "Install gh-aw extension") && strings.Contains(contentStr, "curl -fsSL")) - hasActionInstall := strings.Contains(contentStr, "actions/setup-cli") + return nil +} - // If we have an install step and upgradeVersion is true, this is from upgrade command - // In this case, we still update the file for backward compatibility - if (hasLegacyInstall || hasActionInstall) && upgradeVersion { - copilotSetupLog.Print("Extension install step exists, attempting version upgrade (upgrade command)") +func ensureCopilotSetupStepsWithUpgradeResolver(actionMode workflow.ActionMode) workflow.SHAResolver { + // Create a SHA resolver for release/action mode to enable SHA-pinned action references + if actionMode.IsRelease() || actionMode.IsAction() { + cache := workflow.NewActionCache(".") + _ = cache.Load() // Ignore errors if cache doesn't exist yet + return workflow.NewActionResolver(cache) + } + return nil +} - upgraded, updatedContent, err := upgradeSetupCliVersionInContent(ctx, content, actionMode, version, resolver) - if err != nil { - return fmt.Errorf("failed to upgrade setup-cli version: %w", err) - } +func ensureCopilotSetupStepsWithUpgradeExisting( + ctx context.Context, + verbose bool, + actionMode workflow.ActionMode, + version string, + upgradeVersion bool, + resolver workflow.SHAResolver, + setupStepsPath string, +) error { + copilotSetupLog.Printf("File already exists: %s", setupStepsPath) + content, err := os.ReadFile(setupStepsPath) + if err != nil { + return fmt.Errorf("failed to read existing copilot-setup-steps.yml: %w", err) + } - if !upgraded { - copilotSetupLog.Print("No version upgrade needed") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessageStderr("No version upgrade needed for "+setupStepsPath)) - } - return nil - } + hasInstall := ensureCopilotSetupStepsWithUpgradeHasInstall(content) + if hasInstall && upgradeVersion { + return ensureCopilotSetupStepsWithUpgradeVersion(ctx, verbose, actionMode, version, resolver, setupStepsPath, content) + } + if hasInstall { + copilotSetupLog.Print("Extension install step already exists, file is up to date") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessageStderr(fmt.Sprintf("Skipping %s (already has gh-aw extension install step)", setupStepsPath))) + } + return nil + } - if err := os.WriteFile(setupStepsPath, updatedContent, constants.FilePermSensitive); err != nil { - return fmt.Errorf("failed to update copilot-setup-steps.yml: %w", err) - } - copilotSetupLog.Printf("Upgraded version in file: %s", setupStepsPath) + copilotSetupLog.Print("File exists without install step, rendering update instructions instead of editing") + renderCopilotSetupUpdateInstructions(ctx, setupStepsPath, actionMode, version, resolver) + return nil +} - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessageStderr(fmt.Sprintf("Updated %s with new version %s", setupStepsPath, version))) - } - return nil - } +func ensureCopilotSetupStepsWithUpgradeHasInstall(content []byte) bool { + contentStr := string(content) + hasLegacyInstall := strings.Contains(contentStr, "install-gh-aw.sh") || + (strings.Contains(contentStr, "Install gh-aw extension") && strings.Contains(contentStr, "curl -fsSL")) + hasActionInstall := strings.Contains(contentStr, "actions/setup-cli") + return hasLegacyInstall || hasActionInstall +} - // File exists - render instructions instead of editing - if hasLegacyInstall || hasActionInstall { - copilotSetupLog.Print("Extension install step already exists, file is up to date") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessageStderr(fmt.Sprintf("Skipping %s (already has gh-aw extension install step)", setupStepsPath))) - } - return nil +func ensureCopilotSetupStepsWithUpgradeVersion( + ctx context.Context, + verbose bool, + actionMode workflow.ActionMode, + version string, + resolver workflow.SHAResolver, + setupStepsPath string, + content []byte, +) error { + copilotSetupLog.Print("Extension install step exists, attempting version upgrade (upgrade command)") + upgraded, updatedContent, err := upgradeSetupCliVersionInContent(ctx, content, actionMode, version, resolver) + if err != nil { + return fmt.Errorf("failed to upgrade setup-cli version: %w", err) + } + if !upgraded { + copilotSetupLog.Print("No version upgrade needed") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessageStderr("No version upgrade needed for "+setupStepsPath)) } - - // File exists but needs update - render instructions - copilotSetupLog.Print("File exists without install step, rendering update instructions instead of editing") - renderCopilotSetupUpdateInstructions(ctx, setupStepsPath, actionMode, version, resolver) return nil } - - // File doesn't exist - create it - if err := os.WriteFile(setupStepsPath, []byte(generateCopilotSetupStepsYAML(ctx, actionMode, version, resolver)), constants.FilePermSensitive); err != nil { - return fmt.Errorf("failed to write copilot-setup-steps.yml: %w", err) + if err := os.WriteFile(setupStepsPath, updatedContent, constants.FilePermSensitive); err != nil { + return fmt.Errorf("failed to update copilot-setup-steps.yml: %w", err) + } + copilotSetupLog.Printf("Upgraded version in file: %s", setupStepsPath) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessageStderr(fmt.Sprintf("Updated %s with new version %s", setupStepsPath, version))) } - copilotSetupLog.Printf("Created file: %s", setupStepsPath) - return nil } diff --git a/pkg/cli/dependency_graph.go b/pkg/cli/dependency_graph.go index d7b1e8f6ed2..d1c35b25180 100644 --- a/pkg/cli/dependency_graph.go +++ b/pkg/cli/dependency_graph.go @@ -178,57 +178,63 @@ func (g *DependencyGraph) extractImportsFromFrontmatter(workflowPath string, fro // or an object with 'aw' subfield for agentic workflow paths. workflowDir := filepath.Dir(workflowPath) - // extractPathsFromArray resolves file paths from a []any import list. - extractPathsFromArray := func(items []any) []string { - var paths []string - for _, item := range items { - switch importItem := item.(type) { - case string: - if resolvedPath := g.resolveImportPath(importItem, workflowDir); resolvedPath != "" { - paths = append(paths, resolvedPath) - } - case map[string]any: - if pathValue, hasPath := importItem["path"]; hasPath { - if pathStr, ok := pathValue.(string); ok { - if resolvedPath := g.resolveImportPath(pathStr, workflowDir); resolvedPath != "" { - paths = append(paths, resolvedPath) - } - } - } - } - } - return paths - } - switch v := importsField.(type) { case []any: - imports = extractPathsFromArray(v) + imports = g.extractImportsFromFrontmatterPathsFromArray(v, workflowDir) case []string: - for _, importPath := range v { - if resolvedPath := g.resolveImportPath(importPath, workflowDir); resolvedPath != "" { - imports = append(imports, resolvedPath) - } - } + imports = g.extractImportsFromFrontmatterPathsFromStrings(v, workflowDir) case map[string]any: - // Object form: extract paths from the 'aw' subfield. - // The 'apm-packages' subfield contains package names, not file paths. - if awAny, hasAW := v["aw"]; hasAW { - switch aw := awAny.(type) { - case []any: - imports = extractPathsFromArray(aw) - case []string: - for _, importPath := range aw { - if resolvedPath := g.resolveImportPath(importPath, workflowDir); resolvedPath != "" { - imports = append(imports, resolvedPath) + imports = g.extractImportsFromFrontmatterPathsFromMap(v, workflowDir) + } + + return imports +} + +func (g *DependencyGraph) extractImportsFromFrontmatterPathsFromArray(items []any, workflowDir string) []string { + var paths []string + for _, item := range items { + switch importItem := item.(type) { + case string: + if resolvedPath := g.resolveImportPath(importItem, workflowDir); resolvedPath != "" { + paths = append(paths, resolvedPath) + } + case map[string]any: + if pathValue, hasPath := importItem["path"]; hasPath { + if pathStr, ok := pathValue.(string); ok { + if resolvedPath := g.resolveImportPath(pathStr, workflowDir); resolvedPath != "" { + paths = append(paths, resolvedPath) } } } } } + return paths +} +func (g *DependencyGraph) extractImportsFromFrontmatterPathsFromStrings(items []string, workflowDir string) []string { + var imports []string + for _, importPath := range items { + if resolvedPath := g.resolveImportPath(importPath, workflowDir); resolvedPath != "" { + imports = append(imports, resolvedPath) + } + } return imports } +func (g *DependencyGraph) extractImportsFromFrontmatterPathsFromMap(items map[string]any, workflowDir string) []string { + // Object form: extract paths from the 'aw' subfield. + // The 'apm-packages' subfield contains package names, not file paths. + if awAny, hasAW := items["aw"]; hasAW { + switch aw := awAny.(type) { + case []any: + return g.extractImportsFromFrontmatterPathsFromArray(aw, workflowDir) + case []string: + return g.extractImportsFromFrontmatterPathsFromStrings(aw, workflowDir) + } + } + return nil +} + // resolveImportPath resolves an import path to an absolute file path func (g *DependencyGraph) resolveImportPath(importPath string, baseDir string) string { gitRoot, err := findGitRootForPath(g.workflowsDir) diff --git a/pkg/cli/deps_report.go b/pkg/cli/deps_report.go index a1699e02fee..96858044bf1 100644 --- a/pkg/cli/deps_report.go +++ b/pkg/cli/deps_report.go @@ -43,61 +43,72 @@ func GenerateDependencyReport(ctx context.Context, verbose bool) (*DependencyRep } depsReportLog.Printf("Parsed go.mod: %d total dependencies", len(allDeps)) - // Count direct vs indirect dependencies - directCount := 0 - indirectCount := 0 - v0Count := 0 - v1Count := 0 - v2Count := 0 + counts := generateDependencyReportCounts(allDeps) + outdated := generateDependencyReportOutdated(ctx, verbose) + advisories := generateDependencyReportAdvisories(ctx, verbose) + report := &DependencyReport{ + TotalDeps: len(allDeps), + DirectDeps: counts.direct, + IndirectDeps: counts.indirect, + Outdated: outdated, + Advisories: advisories, + V0Count: counts.v0, + V1PlusCount: counts.v1, + V2PlusCount: counts.v2, + } + + depsReportLog.Printf("Report generated: %d total deps, %d outdated, %d advisories", report.TotalDeps, len(report.Outdated), len(report.Advisories)) + return report, nil +} + +type generateDependencyReportCountValues struct { + direct int + indirect int + v0 int + v1 int + v2 int +} + +func generateDependencyReportCounts(allDeps []DependencyInfoWithIndirect) generateDependencyReportCountValues { + var counts generateDependencyReportCountValues for _, dep := range allDeps { if dep.Indirect { - indirectCount++ + counts.indirect++ } else { - directCount++ + counts.direct++ } - - // Count version maturity if strings.HasPrefix(dep.Version, "v0.") { - v0Count++ + counts.v0++ } else if strings.HasPrefix(dep.Version, "v1.") { - v1Count++ + counts.v1++ } else if strings.HasPrefix(dep.Version, "v2.") || strings.HasPrefix(dep.Version, "v3.") { - v2Count++ + counts.v2++ } } + return counts +} - // Check for outdated dependencies (only direct) +func generateDependencyReportOutdated(ctx context.Context, verbose bool) []OutdatedDependency { outdated, err := CheckOutdatedDependencies(ctx, verbose) if err != nil { if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: could not check outdated dependencies: %v", err))) } - outdated = []OutdatedDependency{} + return []OutdatedDependency{} } + return outdated +} - // Check for security advisories +func generateDependencyReportAdvisories(ctx context.Context, verbose bool) []SecurityAdvisory { advisories, err := CheckSecurityAdvisories(ctx, verbose) if err != nil { if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: could not check security advisories: %v", err))) } - advisories = []SecurityAdvisory{} - } - - report := &DependencyReport{ - TotalDeps: len(allDeps), - DirectDeps: directCount, - IndirectDeps: indirectCount, - Outdated: outdated, - Advisories: advisories, - V0Count: v0Count, - V1PlusCount: v1Count, - V2PlusCount: v2Count, + return []SecurityAdvisory{} } - - depsReportLog.Printf("Report generated: %d total deps, %d outdated, %d advisories", report.TotalDeps, len(report.Outdated), len(report.Advisories)) - return report, nil + return advisories } // DisplayDependencyReport shows the comprehensive dependency report @@ -107,18 +118,32 @@ func DisplayDependencyReport(report *DependencyReport) { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("═══════════════════════════════════════")) fmt.Fprintln(os.Stderr, "") - // Summary section + displayDependencyReportSummary(report) + v0Percentage := safePercent(report.V0Count, report.TotalDeps) + + // Outdated dependencies section + if len(report.Outdated) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Outdated Dependencies")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("---------------------")) + DisplayOutdatedDependencies(report.Outdated, report.DirectDeps) + fmt.Fprintln(os.Stderr, "") + } + + displayDependencyReportSecurity(report) + displayDependencyReportMaturity(report, v0Percentage) + displayDependencyReportRecommendations(report, v0Percentage) +} + +func displayDependencyReportSummary(report *DependencyReport) { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Summary")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("-------")) fmt.Fprintf(os.Stderr, "Total dependencies: %d (%d direct, %d indirect)\n", report.TotalDeps, report.DirectDeps, report.IndirectDeps) - outdatedPercentage := 0.0 if report.DirectDeps > 0 { outdatedPercentage = float64(len(report.Outdated)) / float64(report.DirectDeps) * 100 } fmt.Fprintf(os.Stderr, "Outdated: %d (%.0f%%)\n", len(report.Outdated), outdatedPercentage) fmt.Fprintf(os.Stderr, "Security advisories: %d\n", len(report.Advisories)) - v0Percentage := safePercent(report.V0Count, report.TotalDeps) fmt.Fprintf(os.Stderr, "v0.x dependencies: %d (%.0f%%)", report.V0Count, v0Percentage) if v0Percentage > 30 { @@ -126,16 +151,9 @@ func DisplayDependencyReport(report *DependencyReport) { } fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "") +} - // Outdated dependencies section - if len(report.Outdated) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Outdated Dependencies")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("---------------------")) - DisplayOutdatedDependencies(report.Outdated, report.DirectDeps) - fmt.Fprintln(os.Stderr, "") - } - - // Security status section +func displayDependencyReportSecurity(report *DependencyReport) { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Security Status")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("---------------")) if len(report.Advisories) == 0 { @@ -144,8 +162,9 @@ func DisplayDependencyReport(report *DependencyReport) { DisplaySecurityAdvisories(report.Advisories) } fmt.Fprintln(os.Stderr, "") +} - // Dependency maturity section +func displayDependencyReportMaturity(report *DependencyReport, v0Percentage float64) { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Dependency Maturity")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("-------------------")) fmt.Fprintf(os.Stderr, "v0.x (unstable): %d (%.0f%%)", report.V0Count, v0Percentage) @@ -153,33 +172,27 @@ func DisplayDependencyReport(report *DependencyReport) { fmt.Fprintf(os.Stderr, " ⚠️") } fmt.Fprintln(os.Stderr, "") - - v1Percentage := safePercent(report.V1PlusCount, report.TotalDeps) - fmt.Fprintf(os.Stderr, "v1.x (stable): %d (%.0f%%)\n", report.V1PlusCount, v1Percentage) - - v2Percentage := safePercent(report.V2PlusCount, report.TotalDeps) - fmt.Fprintf(os.Stderr, "v2+ (mature): %d (%.0f%%)\n", report.V2PlusCount, v2Percentage) + fmt.Fprintf(os.Stderr, "v1.x (stable): %d (%.0f%%)\n", report.V1PlusCount, safePercent(report.V1PlusCount, report.TotalDeps)) + fmt.Fprintf(os.Stderr, "v2+ (mature): %d (%.0f%%)\n", report.V2PlusCount, safePercent(report.V2PlusCount, report.TotalDeps)) fmt.Fprintln(os.Stderr, "") +} - // Recommendations section - if len(report.Outdated) > 0 || len(report.Advisories) > 0 || v0Percentage > 30 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Recommendations")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("---------------")) - - if len(report.Advisories) > 0 { - fmt.Fprintf(os.Stderr, "🔴 CRITICAL: Address %d security %s immediately\n", len(report.Advisories), pluralize("advisory", len(report.Advisories))) - } - - if len(report.Outdated) > 0 { - fmt.Fprintf(os.Stderr, "📦 Update %d outdated %s\n", len(report.Outdated), pluralize("dependency", len(report.Outdated))) - } - - if v0Percentage > 30 { - fmt.Fprintf(os.Stderr, "⚠️ Reduce v0.x exposure from %.0f%% to <30%%\n", v0Percentage) - } - - fmt.Fprintln(os.Stderr, "") +func displayDependencyReportRecommendations(report *DependencyReport, v0Percentage float64) { + if len(report.Outdated) == 0 && len(report.Advisories) == 0 && v0Percentage <= 30 { + return } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Recommendations")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("---------------")) + if len(report.Advisories) > 0 { + fmt.Fprintf(os.Stderr, "🔴 CRITICAL: Address %d security %s immediately\n", len(report.Advisories), pluralize("advisory", len(report.Advisories))) + } + if len(report.Outdated) > 0 { + fmt.Fprintf(os.Stderr, "📦 Update %d outdated %s\n", len(report.Outdated), pluralize("dependency", len(report.Outdated))) + } + if v0Percentage > 30 { + fmt.Fprintf(os.Stderr, "⚠️ Reduce v0.x exposure from %.0f%% to <30%%\n", v0Percentage) + } + fmt.Fprintln(os.Stderr, "") } // DisplayDependencyReportJSON outputs the dependency report in JSON format @@ -196,41 +209,63 @@ func DisplayDependencyReportJSON(report *DependencyReport) error { v1Percentage := safePercent(report.V1PlusCount, report.TotalDeps) v2Percentage := safePercent(report.V2PlusCount, report.TotalDeps) - // Build JSON-friendly output structure - output := map[string]any{ - "summary": map[string]any{ - "total_dependencies": report.TotalDeps, - "direct_dependencies": report.DirectDeps, - "indirect_dependencies": report.IndirectDeps, - "outdated_count": len(report.Outdated), - "outdated_percentage": outdatedPercentage, - "security_advisories": len(report.Advisories), - "v0_count": report.V0Count, - "v0_percentage": v0Percentage, - "v1_count": report.V1PlusCount, - "v1_percentage": v1Percentage, - "v2_count": report.V2PlusCount, - "v2_percentage": v2Percentage, + output := displayDependencyReportJSONOutput(report, outdatedPercentage, v0Percentage, v1Percentage, v2Percentage) + + // Marshal and output to stdout + jsonData, err := json.MarshalIndent(output, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON: %w", err) + } + + fmt.Fprintln(os.Stdout, string(jsonData)) + return nil +} + +func displayDependencyReportJSONOutput(report *DependencyReport, outdatedPercentage, v0Percentage, v1Percentage, v2Percentage float64) map[string]any { + return map[string]any{ + "summary": displayDependencyReportJSONSummary(report, outdatedPercentage, v0Percentage, v1Percentage, v2Percentage), + "outdated": report.Outdated, + "security": report.Advisories, + "maturity": displayDependencyReportJSONMaturity(report, v0Percentage, v1Percentage, v2Percentage), + "recommendations": displayDependencyReportJSONRecommendations(report, v0Percentage), + } +} + +func displayDependencyReportJSONSummary(report *DependencyReport, outdatedPercentage, v0Percentage, v1Percentage, v2Percentage float64) map[string]any { + return map[string]any{ + "total_dependencies": report.TotalDeps, + "direct_dependencies": report.DirectDeps, + "indirect_dependencies": report.IndirectDeps, + "outdated_count": len(report.Outdated), + "outdated_percentage": outdatedPercentage, + "security_advisories": len(report.Advisories), + "v0_count": report.V0Count, + "v0_percentage": v0Percentage, + "v1_count": report.V1PlusCount, + "v1_percentage": v1Percentage, + "v2_count": report.V2PlusCount, + "v2_percentage": v2Percentage, + } +} + +func displayDependencyReportJSONMaturity(report *DependencyReport, v0Percentage, v1Percentage, v2Percentage float64) map[string]any { + return map[string]any{ + "v0_unstable": map[string]any{ + "count": report.V0Count, + "percentage": v0Percentage, + }, + "v1_stable": map[string]any{ + "count": report.V1PlusCount, + "percentage": v1Percentage, }, - "outdated": report.Outdated, - "security": report.Advisories, - "maturity": map[string]any{ - "v0_unstable": map[string]any{ - "count": report.V0Count, - "percentage": v0Percentage, - }, - "v1_stable": map[string]any{ - "count": report.V1PlusCount, - "percentage": v1Percentage, - }, - "v2_mature": map[string]any{ - "count": report.V2PlusCount, - "percentage": v2Percentage, - }, + "v2_mature": map[string]any{ + "count": report.V2PlusCount, + "percentage": v2Percentage, }, } +} - // Add recommendations +func displayDependencyReportJSONRecommendations(report *DependencyReport, v0Percentage float64) []string { recommendations := []string{} if len(report.Advisories) > 0 { recommendations = append(recommendations, fmt.Sprintf("Address %d security %s immediately", len(report.Advisories), pluralize("advisory", len(report.Advisories)))) @@ -241,16 +276,7 @@ func DisplayDependencyReportJSON(report *DependencyReport) error { if v0Percentage > 30 { recommendations = append(recommendations, fmt.Sprintf("Reduce v0.x exposure from %.0f%% to <30%%", v0Percentage)) } - output["recommendations"] = recommendations - - // Marshal and output to stdout - jsonData, err := json.MarshalIndent(output, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal JSON: %w", err) - } - - fmt.Fprintln(os.Stdout, string(jsonData)) - return nil + return recommendations } // DependencyInfoWithIndirect extends DependencyInfo to track indirect dependencies diff --git a/pkg/cli/deps_security.go b/pkg/cli/deps_security.go index 1f764a19fd3..70d22894faf 100644 --- a/pkg/cli/deps_security.go +++ b/pkg/cli/deps_security.go @@ -144,6 +144,15 @@ func querySecurityAdvisories(ctx context.Context, depVersions map[string]string, url := "https://api.github.com/advisories?ecosystem=go&per_page=100" depsSecurityLog.Printf("Querying GitHub Security Advisory API: url=%s, dep_count=%d", url, len(depVersions)) + apiAdvisories, err := querySecurityAdvisoriesFetch(ctx, url) + if err != nil { + return nil, err + } + depsSecurityLog.Printf("Received %d advisories from GitHub API", len(apiAdvisories)) + return querySecurityAdvisoriesFilter(apiAdvisories, depVersions, verbose), nil +} + +func querySecurityAdvisoriesFetch(ctx context.Context, url string) ([]GitHubAdvisoryResponse, error) { client := &http.Client{Timeout: constants.DefaultHTTPClientTimeout} req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -174,9 +183,10 @@ func querySecurityAdvisories(ctx context.Context, depVersions map[string]string, if err := json.Unmarshal(body, &apiAdvisories); err != nil { return nil, err } + return apiAdvisories, nil +} - depsSecurityLog.Printf("Received %d advisories from GitHub API", len(apiAdvisories)) - +func querySecurityAdvisoriesFilter(apiAdvisories []GitHubAdvisoryResponse, depVersions map[string]string, verbose bool) []SecurityAdvisory { // Filter advisories that match our dependencies var matchingAdvisories []SecurityAdvisory for _, apiAdv := range apiAdvisories { @@ -212,7 +222,7 @@ func querySecurityAdvisories(ctx context.Context, depVersions map[string]string, } } - return matchingAdvisories, nil + return matchingAdvisories } // getSeverityIcon returns an emoji icon for the severity level diff --git a/pkg/cli/devcontainer.go b/pkg/cli/devcontainer.go index 85d2f92c98a..713953b463a 100644 --- a/pkg/cli/devcontainer.go +++ b/pkg/cli/devcontainer.go @@ -77,127 +77,136 @@ func ensureDevcontainerConfig(verbose bool, additionalRepos []string) error { } devcontainerLog.Printf("Ensured directory exists: %s", devcontainerDir) - // Check if file already exists at default location - var existingConfig *DevcontainerConfig - if _, err := os.Stat(devcontainerPath); err == nil { - devcontainerLog.Printf("File already exists: %s", devcontainerPath) - - // Read existing config to update it - existingData, err := os.ReadFile(devcontainerPath) - if err != nil { - devcontainerLog.Printf("Failed to read existing config: %v", err) - return fmt.Errorf("failed to read existing devcontainer.json: %w", err) - } + existingConfig, err := ensureDevcontainerConfigLoadExisting(devcontainerPath) + if err != nil { + return err + } + ghAwRepositories := ensureDevcontainerConfigRepositoryPermissions(additionalRepos) - var config DevcontainerConfig - if err := json.Unmarshal(existingData, &config); err != nil { - devcontainerLog.Printf("Failed to parse existing config: %v", err) - return fmt.Errorf("failed to parse existing devcontainer.json: %w", err) + var config DevcontainerConfig + if existingConfig != nil { + config = ensureDevcontainerConfigUpdateExisting(existingConfig, ghAwRepositories) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessageStderr("Updated existing devcontainer at "+devcontainerPath)) + } + } else { + config = ensureDevcontainerConfigCreateNew(ghAwRepositories) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessageStderr("Created new devcontainer at "+devcontainerPath)) } - existingConfig = &config - devcontainerLog.Printf("Successfully parsed existing devcontainer.json") } - // Get current repository name from git remote + return ensureDevcontainerConfigWrite(devcontainerPath, config) +} + +func ensureDevcontainerConfigLoadExisting(devcontainerPath string) (*DevcontainerConfig, error) { + if _, err := os.Stat(devcontainerPath); err != nil { + return nil, nil + } + devcontainerLog.Printf("File already exists: %s", devcontainerPath) + existingData, err := os.ReadFile(devcontainerPath) + if err != nil { + devcontainerLog.Printf("Failed to read existing config: %v", err) + return nil, fmt.Errorf("failed to read existing devcontainer.json: %w", err) + } + var config DevcontainerConfig + if err := json.Unmarshal(existingData, &config); err != nil { + devcontainerLog.Printf("Failed to parse existing config: %v", err) + return nil, fmt.Errorf("failed to parse existing devcontainer.json: %w", err) + } + devcontainerLog.Printf("Successfully parsed existing devcontainer.json") + return &config, nil +} + +func ensureDevcontainerConfigRepositoryPermissions(additionalRepos []string) map[string]DevcontainerRepoPermissions { repoName := getCurrentRepoName() if repoName == "" { repoName = "current-repo" } + return buildRepositoryPermissions(repoName, getRepoOwner(), additionalRepos) +} - // Get the owner from the current repository - owner := getRepoOwner() - - // Prepare gh-aw specific configuration - ghAwRepositories := buildRepositoryPermissions(repoName, owner, additionalRepos) - - var config DevcontainerConfig - - if existingConfig != nil { - // Update existing configuration - devcontainerLog.Printf("Updating existing devcontainer.json") - config = *existingConfig +func ensureDevcontainerConfigUpdateExisting(existingConfig *DevcontainerConfig, ghAwRepositories map[string]DevcontainerRepoPermissions) DevcontainerConfig { + devcontainerLog.Printf("Updating existing devcontainer.json") + config := *existingConfig + ensureDevcontainerConfigMergeCustomizations(&config, ghAwRepositories) + ensureDevcontainerConfigMergeFeatures(&config) + ensureDevcontainerConfigPostCreate(&config) + return config +} - // Ensure customizations exists - if config.Customizations == nil { - config.Customizations = &DevcontainerCustomizations{} +func ensureDevcontainerConfigMergeCustomizations(config *DevcontainerConfig, ghAwRepositories map[string]DevcontainerRepoPermissions) { + if config.Customizations == nil { + config.Customizations = &DevcontainerCustomizations{} + } + if config.Customizations.VSCode == nil { + config.Customizations.VSCode = &DevcontainerVSCode{} + } + config.Customizations.VSCode.Extensions = mergeExtensions( + config.Customizations.VSCode.Extensions, + []string{"GitHub.copilot", "GitHub.copilot-chat"}, + ) + if config.Customizations.Codespaces == nil { + config.Customizations.Codespaces = &DevcontainerCodespaces{ + Repositories: make(map[string]DevcontainerRepoPermissions), } + } + if config.Customizations.Codespaces.Repositories == nil { + config.Customizations.Codespaces.Repositories = make(map[string]DevcontainerRepoPermissions) + } + for repo, perms := range ghAwRepositories { + config.Customizations.Codespaces.Repositories[repo] = perms + devcontainerLog.Printf("Updated permissions for repo: %s", repo) + } +} - // Merge VSCode extensions - if config.Customizations.VSCode == nil { - config.Customizations.VSCode = &DevcontainerVSCode{} - } - config.Customizations.VSCode.Extensions = mergeExtensions( - config.Customizations.VSCode.Extensions, - []string{"GitHub.copilot", "GitHub.copilot-chat"}, - ) - - // Merge Codespaces repositories - if config.Customizations.Codespaces == nil { - config.Customizations.Codespaces = &DevcontainerCodespaces{ - Repositories: make(map[string]DevcontainerRepoPermissions), - } - } - if config.Customizations.Codespaces.Repositories == nil { - config.Customizations.Codespaces.Repositories = make(map[string]DevcontainerRepoPermissions) - } - for repo, perms := range ghAwRepositories { - config.Customizations.Codespaces.Repositories[repo] = perms - devcontainerLog.Printf("Updated permissions for repo: %s", repo) - } +func ensureDevcontainerConfigMergeFeatures(config *DevcontainerConfig) { + if config.Features == nil { + config.Features = make(DevcontainerFeatures) + } + mergeFeatures(config.Features, map[string]any{ + "ghcr.io/devcontainers/features/github-cli:1": map[string]any{}, + "ghcr.io/devcontainers/features/copilot-cli:latest": map[string]any{}, + }) +} - // Merge features - if config.Features == nil { - config.Features = make(DevcontainerFeatures) - } - mergeFeatures(config.Features, map[string]any{ - "ghcr.io/devcontainers/features/github-cli:1": map[string]any{}, - "ghcr.io/devcontainers/features/copilot-cli:latest": map[string]any{}, - }) - - // Update postCreateCommand if not set or if it doesn't include gh-aw install - if config.PostCreateCommand == "" || !strings.Contains(config.PostCreateCommand, "install-gh-aw.sh") { - ghAwInstall := "curl -fsSL https://raw.githubusercontent.com/github/gh-aw/refs/heads/main/install-gh-aw.sh | bash" - if config.PostCreateCommand == "" { - config.PostCreateCommand = ghAwInstall - } else { - config.PostCreateCommand = config.PostCreateCommand + " && " + ghAwInstall - } - devcontainerLog.Printf("Updated postCreateCommand to include gh-aw installation") +func ensureDevcontainerConfigPostCreate(config *DevcontainerConfig) { + if config.PostCreateCommand == "" || !strings.Contains(config.PostCreateCommand, "install-gh-aw.sh") { + ghAwInstall := "curl -fsSL https://raw.githubusercontent.com/github/gh-aw/refs/heads/main/install-gh-aw.sh | bash" + if config.PostCreateCommand == "" { + config.PostCreateCommand = ghAwInstall + } else { + config.PostCreateCommand = config.PostCreateCommand + " && " + ghAwInstall } + devcontainerLog.Printf("Updated postCreateCommand to include gh-aw installation") + } +} - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessageStderr("Updated existing devcontainer at "+devcontainerPath)) - } - } else { - // Create new configuration - devcontainerLog.Printf("Creating new devcontainer.json at default location") - config = DevcontainerConfig{ - Name: "Agentic Workflows Development", - Image: "mcr.microsoft.com/devcontainers/universal:latest", - Customizations: &DevcontainerCustomizations{ - VSCode: &DevcontainerVSCode{ - Extensions: []string{ - "GitHub.copilot", - "GitHub.copilot-chat", - }, - }, - Codespaces: &DevcontainerCodespaces{ - Repositories: ghAwRepositories, +func ensureDevcontainerConfigCreateNew(ghAwRepositories map[string]DevcontainerRepoPermissions) DevcontainerConfig { + devcontainerLog.Printf("Creating new devcontainer.json at default location") + return DevcontainerConfig{ + Name: "Agentic Workflows Development", + Image: "mcr.microsoft.com/devcontainers/universal:latest", + Customizations: &DevcontainerCustomizations{ + VSCode: &DevcontainerVSCode{ + Extensions: []string{ + "GitHub.copilot", + "GitHub.copilot-chat", }, }, - Features: DevcontainerFeatures{ - "ghcr.io/devcontainers/features/github-cli:1": map[string]any{}, - "ghcr.io/devcontainers/features/copilot-cli:latest": map[string]any{}, + Codespaces: &DevcontainerCodespaces{ + Repositories: ghAwRepositories, }, - PostCreateCommand: "curl -fsSL https://raw.githubusercontent.com/github/gh-aw/refs/heads/main/install-gh-aw.sh | bash", - } - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessageStderr("Created new devcontainer at "+devcontainerPath)) - } + }, + Features: DevcontainerFeatures{ + "ghcr.io/devcontainers/features/github-cli:1": map[string]any{}, + "ghcr.io/devcontainers/features/copilot-cli:latest": map[string]any{}, + }, + PostCreateCommand: "curl -fsSL https://raw.githubusercontent.com/github/gh-aw/refs/heads/main/install-gh-aw.sh | bash", } +} - // Write config file with proper indentation +func ensureDevcontainerConfigWrite(devcontainerPath string, config DevcontainerConfig) error { data, err := json.MarshalIndent(config, "", " ") if err != nil { return fmt.Errorf("failed to marshal devcontainer.json: %w", err) @@ -243,44 +252,54 @@ func buildRepositoryPermissions(repoName, owner string, additionalRepos []string // Since permissions must be in the same organization, we automatically prepend the owner. // Reference: https://docs.github.com/en/codespaces/managing-your-codespaces/managing-repository-access-for-your-codespaces#setting-additional-repository-permissions for _, repo := range additionalRepos { - if repo == "" { - continue + fullRepoName := buildRepositoryPermissionsFullRepoName(repo, owner) + if fullRepoName != repoName { + buildRepositoryPermissionsAddReadRepository(repositories, fullRepoName) } + } - // If repo already contains '/', validate that the owner matches - // Otherwise, prepend the owner - fullRepoName := repo - if strings.Contains(repo, "/") { - // Validate that the owner matches the current repo's owner - parts := strings.Split(repo, "/") - if len(parts) >= 2 { - repoOwner := parts[0] - if owner != "" && repoOwner != owner { - // Skip repos with different owners rather than error - devcontainerLog.Printf("Skipping repository '%s' - different owner than current repo (expected: '%s')", repo, owner) - continue - } - } - } else if owner != "" { - fullRepoName = owner + "/" + repo - } + return repositories +} - if fullRepoName != repoName { - repositories[fullRepoName] = DevcontainerRepoPermissions{ - Permissions: map[string]string{ - "actions": "read", - "contents": "read", - "discussions": "read", - "issues": "read", - "pull-requests": "read", - "workflows": "read", - }, +func buildRepositoryPermissionsFullRepoName(repo, owner string) string { + if repo == "" { + return "" + } + + // If repo already contains '/', validate that the owner matches. + // Otherwise, prepend the owner. + if strings.Contains(repo, "/") { + parts := strings.Split(repo, "/") + if len(parts) >= 2 { + repoOwner := parts[0] + if owner != "" && repoOwner != owner { + devcontainerLog.Printf("Skipping repository '%s' - different owner than current repo (expected: '%s')", repo, owner) + return "" } - devcontainerLog.Printf("Added read permissions for additional repo: %s", fullRepoName) } + return repo } + if owner != "" { + return owner + "/" + repo + } + return repo +} - return repositories +func buildRepositoryPermissionsAddReadRepository(repositories map[string]DevcontainerRepoPermissions, fullRepoName string) { + if fullRepoName == "" { + return + } + repositories[fullRepoName] = DevcontainerRepoPermissions{ + Permissions: map[string]string{ + "actions": "read", + "contents": "read", + "discussions": "read", + "issues": "read", + "pull-requests": "read", + "workflows": "read", + }, + } + devcontainerLog.Printf("Added read permissions for additional repo: %s", fullRepoName) } // mergeExtensions adds new extensions to existing list, avoiding duplicates diff --git a/pkg/cli/dispatch.go b/pkg/cli/dispatch.go index 50d4341c998..de3fe38d139 100644 --- a/pkg/cli/dispatch.go +++ b/pkg/cli/dispatch.go @@ -79,19 +79,63 @@ type fileDownloadFn func(ctx context.Context, owner, repo, path, ref string) ([] // parser.DownloadFileFromGitHub implementation (used in tests to avoid real network calls). func fetchAndSaveRemoteDispatchWorkflows(ctx context.Context, content string, spec *WorkflowSpec, targetDir string, verbose bool, force bool, tracker *FileTracker, downloaders ...fileDownloadFn) error { remoteWorkflowLog.Printf("Fetching remote dispatch workflows: repo=%s, targetDir=%s, force=%v", spec.RepoSlug, targetDir, force) + downloader := fetchAndSaveRemoteDispatchWorkflowsDownloader(downloaders) + owner, repo, ref, ok := fetchAndSaveRemoteDispatchWorkflowsRepoRef(ctx, spec) + if !ok { + return nil + } + + workflowNames := extractDispatchWorkflowNames(content) + if len(workflowNames) == 0 { + return nil + } + remoteWorkflowLog.Printf("Found %d dispatch workflow(s) to fetch from %s@%s", len(workflowNames), spec.RepoSlug, ref) + + workflowBaseDir := getParentDir(spec.WorkflowPath) + absTargetDir, err := filepath.Abs(targetDir) + if err != nil { + remoteWorkflowLog.Printf("Failed to resolve absolute path for target directory %s: %v", targetDir, err) + return nil + } + + for _, workflowName := range workflowNames { + if err := fetchAndSaveRemoteDispatchWorkflowsOne(fetchAndSaveRemoteDispatchWorkflowsOneParams{ + Ctx: ctx, + WorkflowName: workflowName, + Spec: spec, + TargetDir: targetDir, + Verbose: verbose, + Force: force, + Tracker: tracker, + Downloader: downloader, + Owner: owner, + Repo: repo, + Ref: ref, + WorkflowBaseDir: workflowBaseDir, + AbsTargetDir: absTargetDir, + }); err != nil { + return err + } + } + return nil +} + +func fetchAndSaveRemoteDispatchWorkflowsDownloader(downloaders []fileDownloadFn) fileDownloadFn { downloader := fileDownloadFn(parser.DownloadFileFromGitHub) if len(downloaders) > 0 && downloaders[0] != nil { downloader = downloaders[0] } + return downloader +} + +func fetchAndSaveRemoteDispatchWorkflowsRepoRef(ctx context.Context, spec *WorkflowSpec) (string, string, string, bool) { if spec.RepoSlug == "" { - return nil + return "", "", "", false } - parts := strings.SplitN(spec.RepoSlug, "/", 2) if len(parts) != 2 { - return nil + return "", "", "", false } - owner, repo := parts[0], parts[1] ref := spec.Version if ref == "" { defaultBranch, err := getRepoDefaultBranch(ctx, spec.RepoSlug) @@ -103,160 +147,209 @@ func fetchAndSaveRemoteDispatchWorkflows(ctx context.Context, content string, sp } spec.Version = ref } + return parts[0], parts[1], ref, true +} - workflowNames := extractDispatchWorkflowNames(content) - if len(workflowNames) == 0 { +type fetchAndSaveRemoteDispatchWorkflowsOneParams struct { + Ctx context.Context + WorkflowName string + Spec *WorkflowSpec + TargetDir string + Verbose bool + Force bool + Tracker *FileTracker + Downloader fileDownloadFn + Owner string + Repo string + Ref string + WorkflowBaseDir string + AbsTargetDir string +} + +func fetchAndSaveRemoteDispatchWorkflowsOne(p fetchAndSaveRemoteDispatchWorkflowsOneParams) error { + remoteFilePath, targetPath, ok := fetchAndSaveRemoteDispatchWorkflowsPaths(p.WorkflowName, p.WorkflowBaseDir, p.TargetDir, p.AbsTargetDir, p.Verbose) + if !ok { return nil } + fileExists, skip, err := fetchAndSaveRemoteDispatchWorkflowsExisting(p.WorkflowName, targetPath, p.Spec.RepoSlug, p.Verbose, p.Force) + if err != nil || skip { + return err + } - remoteWorkflowLog.Printf("Found %d dispatch workflow(s) to fetch from %s@%s", len(workflowNames), spec.RepoSlug, ref) - - // workflowBaseDir is the directory of the source workflow in the remote repo - // (e.g. ".github/workflows"). Dispatch-workflow names are resolved relative to it. - workflowBaseDir := getParentDir(spec.WorkflowPath) - - // Pre-compute the absolute target directory for path-traversal boundary checks. - absTargetDir, err := filepath.Abs(targetDir) + workflowContent, err := p.Downloader(p.Ctx, p.Owner, p.Repo, remoteFilePath, p.Ref) if err != nil { - remoteWorkflowLog.Printf("Failed to resolve absolute path for target directory %s: %v", targetDir, err) + fetchAndSaveRemoteDispatchWorkflowsYMLFallback(fetchAndSaveRemoteDispatchWorkflowsYMLFallbackParams{ + Ctx: p.Ctx, + WorkflowName: p.WorkflowName, + RemoteFilePath: remoteFilePath, + TargetDir: p.TargetDir, + Verbose: p.Verbose, + Tracker: p.Tracker, + Downloader: p.Downloader, + Owner: p.Owner, + Repo: p.Repo, + Ref: p.Ref, + OriginalErr: err, + }) return nil } + return fetchAndSaveRemoteDispatchWorkflowsWriteMarkdown(fetchAndSaveRemoteDispatchWorkflowsWriteMarkdownParams{ + Ctx: p.Ctx, + WorkflowContent: workflowContent, + Spec: p.Spec, + RemoteFilePath: remoteFilePath, + TargetPath: targetPath, + TargetDir: p.TargetDir, + Verbose: p.Verbose, + Force: p.Force, + Tracker: p.Tracker, + FileExists: fileExists, + Ref: p.Ref, + }) +} - for _, workflowName := range workflowNames { - // Build the remote file path for this dispatch workflow - var remoteFilePath string - if workflowBaseDir != "" { - remoteFilePath = path.Join(workflowBaseDir, workflowName+".md") - } else { - remoteFilePath = workflowName + ".md" +func fetchAndSaveRemoteDispatchWorkflowsPaths(workflowName, workflowBaseDir, targetDir, absTargetDir string, verbose bool) (string, string, bool) { + var remoteFilePath string + if workflowBaseDir != "" { + remoteFilePath = path.Join(workflowBaseDir, workflowName+".md") + } else { + remoteFilePath = workflowName + ".md" + } + remoteFilePath = path.Clean(remoteFilePath) + targetPath := filepath.Join(targetDir, filepath.Clean(workflowName+".md")) + absTargetPath, absErr := filepath.Abs(targetPath) + if absErr != nil { + remoteWorkflowLog.Printf("Failed to resolve absolute path for dispatch workflow %s: %v", workflowName, absErr) + return "", "", false + } + if rel, relErr := filepath.Rel(absTargetDir, absTargetPath); relErr != nil || strings.HasPrefix(rel, "..") { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Refusing to write dispatch workflow outside target directory: %q", workflowName))) } - remoteFilePath = path.Clean(remoteFilePath) - - // The local path is just the workflow filename in targetDir - localRelPath := filepath.Clean(workflowName + ".md") - targetPath := filepath.Join(targetDir, localRelPath) + return "", "", false + } + return remoteFilePath, targetPath, true +} - // Belt-and-suspenders: verify the resolved path stays inside targetDir - absTargetPath, absErr := filepath.Abs(targetPath) - if absErr != nil { - remoteWorkflowLog.Printf("Failed to resolve absolute path for dispatch workflow %s: %v", workflowName, absErr) - continue - } - if rel, relErr := filepath.Rel(absTargetDir, absTargetPath); relErr != nil || strings.HasPrefix(rel, "..") { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Refusing to write dispatch workflow outside target directory: %q", workflowName))) - } - continue +func fetchAndSaveRemoteDispatchWorkflowsExisting(workflowName, targetPath, repoSlug string, verbose bool, force bool) (bool, bool, error) { + if _, statErr := os.Stat(targetPath); statErr != nil { + return false, false, nil + } + if force { + return true, false, nil + } + existingSourceRepo := readSourceRepoFromFile(targetPath) + if existingSourceRepo == repoSlug { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Dispatch workflow from same source already exists, skipping: "+targetPath)) } + return true, true, nil + } + return true, false, fmt.Errorf( + "dispatch workflow %q already exists at %s (existing source: %q, installing from: %q); remove the file or use --force to overwrite", + workflowName, targetPath, sourceRepoLabel(existingSourceRepo), repoSlug, + ) +} - // Check whether the target file already exists. - fileExists := false - if _, statErr := os.Stat(targetPath); statErr == nil { - fileExists = true - if !force { - // Allow if the existing file comes from the same source repository. - existingSourceRepo := readSourceRepoFromFile(targetPath) - if existingSourceRepo == spec.RepoSlug { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Dispatch workflow from same source already exists, skipping: "+targetPath)) - } - continue - } - // Different or missing source — this is a conflict. - return fmt.Errorf( - "dispatch workflow %q already exists at %s (existing source: %q, installing from: %q); remove the file or use --force to overwrite", - workflowName, targetPath, sourceRepoLabel(existingSourceRepo), spec.RepoSlug, - ) - } - } +type fetchAndSaveRemoteDispatchWorkflowsYMLFallbackParams struct { + Ctx context.Context + WorkflowName string + RemoteFilePath string + TargetDir string + Verbose bool + Tracker *FileTracker + Downloader fileDownloadFn + Owner string + Repo string + Ref string + OriginalErr error +} - // Download from the source repository — try .md first, then .yml as fallback - // (the dispatch-workflow validator accepts either .md or .yml files locally). - workflowContent, err := downloader(ctx, owner, repo, remoteFilePath, ref) - if err != nil { - remoteWorkflowLog.Printf(".md fetch failed for dispatch workflow %s, trying .yml fallback", workflowName) - // .md not found — try .yml fallback (e.g. plain GitHub Actions workflow) - ymlRemotePath := path.Clean(strings.TrimSuffix(remoteFilePath, ".md") + ".yml") - ymlLocalPath := filepath.Join(targetDir, filepath.Clean(workflowName+".yml")) - - ymlContent, ymlErr := downloader(ctx, owner, repo, ymlRemotePath, ref) - if ymlErr != nil { - // Neither .md nor .yml found — best-effort, continue - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch dispatch workflow %s: %v", remoteFilePath, err))) - } - continue - } - // .yml fallback succeeded — write it (no source field for yml) - if mkErr := os.MkdirAll(filepath.Dir(ymlLocalPath), constants.DirPermPublic); mkErr != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for dispatch workflow %s: %v", ymlRemotePath, mkErr))) - } - continue - } - // Capture whether file exists before writing (for correct tracker classification). - _, ymlFileExistsErr := os.Stat(ymlLocalPath) - ymlFileExists := ymlFileExistsErr == nil - if writeErr := os.WriteFile(ymlLocalPath, ymlContent, constants.FilePermSensitive); writeErr != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write dispatch workflow %s: %v", ymlRemotePath, writeErr))) - } - continue - } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched dispatch workflow (.yml): "+ymlLocalPath)) - } - if tracker != nil { - if ymlFileExists { - tracker.TrackModified(ymlLocalPath) - } else { - tracker.TrackCreated(ymlLocalPath) - } - } - continue +func fetchAndSaveRemoteDispatchWorkflowsYMLFallback(p fetchAndSaveRemoteDispatchWorkflowsYMLFallbackParams) { + remoteWorkflowLog.Printf(".md fetch failed for dispatch workflow %s, trying .yml fallback", p.WorkflowName) + ymlRemotePath := path.Clean(strings.TrimSuffix(p.RemoteFilePath, ".md") + ".yml") + ymlLocalPath := filepath.Join(p.TargetDir, filepath.Clean(p.WorkflowName+".yml")) + ymlContent, ymlErr := p.Downloader(p.Ctx, p.Owner, p.Repo, ymlRemotePath, p.Ref) + if ymlErr != nil { + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch dispatch workflow %s: %v", p.RemoteFilePath, p.OriginalErr))) } + return + } + fetchAndSaveRemoteDispatchWorkflowsWriteYML(ymlRemotePath, ymlLocalPath, ymlContent, p.Verbose, p.Tracker) +} - // Embed the source field so future adds can detect same-source conflicts. - depSourceString := spec.RepoSlug + "/" + remoteFilePath + "@" + ref - if updated, srcErr := addSourceToWorkflow(string(workflowContent), depSourceString); srcErr == nil { - workflowContent = []byte(updated) +func fetchAndSaveRemoteDispatchWorkflowsWriteYML(ymlRemotePath, ymlLocalPath string, ymlContent []byte, verbose bool, tracker *FileTracker) { + if mkErr := os.MkdirAll(filepath.Dir(ymlLocalPath), constants.DirPermPublic); mkErr != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for dispatch workflow %s: %v", ymlRemotePath, mkErr))) } - - // Create parent directory if needed - if err := os.MkdirAll(filepath.Dir(targetPath), constants.DirPermPublic); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for dispatch workflow %s: %v", remoteFilePath, err))) - } - continue + return + } + _, ymlFileExistsErr := os.Stat(ymlLocalPath) + ymlFileExists := ymlFileExistsErr == nil + if writeErr := os.WriteFile(ymlLocalPath, ymlContent, constants.FilePermSensitive); writeErr != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write dispatch workflow %s: %v", ymlRemotePath, writeErr))) } + return + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched dispatch workflow (.yml): "+ymlLocalPath)) + } + fetchAndSaveRemoteDispatchWorkflowsTrack(tracker, ymlLocalPath, ymlFileExists) +} - // Write the file - if err := os.WriteFile(targetPath, workflowContent, constants.FilePermSensitive); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write dispatch workflow %s: %v", remoteFilePath, err))) - } - continue - } +type fetchAndSaveRemoteDispatchWorkflowsWriteMarkdownParams struct { + Ctx context.Context + WorkflowContent []byte + Spec *WorkflowSpec + RemoteFilePath string + TargetPath string + TargetDir string + Verbose bool + Force bool + Tracker *FileTracker + FileExists bool + Ref string +} - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched dispatch workflow: "+targetPath)) +func fetchAndSaveRemoteDispatchWorkflowsWriteMarkdown(p fetchAndSaveRemoteDispatchWorkflowsWriteMarkdownParams) error { + depSourceString := p.Spec.RepoSlug + "/" + p.RemoteFilePath + "@" + p.Ref + if updated, srcErr := addSourceToWorkflow(string(p.WorkflowContent), depSourceString); srcErr == nil { + p.WorkflowContent = []byte(updated) + } + if err := os.MkdirAll(filepath.Dir(p.TargetPath), constants.DirPermPublic); err != nil { + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for dispatch workflow %s: %v", p.RemoteFilePath, err))) } - - // Track the file - if tracker != nil { - if fileExists { - tracker.TrackModified(targetPath) - } else { - tracker.TrackCreated(targetPath) - } + return nil + } + if err := os.WriteFile(p.TargetPath, p.WorkflowContent, constants.FilePermSensitive); err != nil { + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write dispatch workflow %s: %v", p.RemoteFilePath, err))) } - - fetchDownloadedWorkflowFrontmatterImports(ctx, workflowContent, spec, remoteFilePath, targetDir, verbose, force, tracker) + return nil } - + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched dispatch workflow: "+p.TargetPath)) + } + fetchAndSaveRemoteDispatchWorkflowsTrack(p.Tracker, p.TargetPath, p.FileExists) + fetchDownloadedWorkflowFrontmatterImports(p.Ctx, p.WorkflowContent, p.Spec, p.RemoteFilePath, p.TargetDir, p.Verbose, p.Force, p.Tracker) return nil } +func fetchAndSaveRemoteDispatchWorkflowsTrack(tracker *FileTracker, path string, fileExists bool) { + if tracker == nil { + return + } + if fileExists { + tracker.TrackModified(path) + } else { + tracker.TrackCreated(path) + } +} + func fetchDownloadedWorkflowFrontmatterImports(ctx context.Context, workflowContent []byte, parentSpec *WorkflowSpec, remoteFilePath, targetDir string, verbose bool, force bool, tracker *FileTracker) { depSpec := &WorkflowSpec{ RepoSpec: RepoSpec{ @@ -285,184 +378,273 @@ func fetchDownloadedWorkflowFrontmatterImports(ctx context.Context, workflowCont // Source conflicts are reported as warnings (not errors) because the main file is already written. func fetchAndSaveDispatchWorkflowsFromParsedFile(ctx context.Context, destFile string, spec *WorkflowSpec, targetDir string, verbose bool, force bool, tracker *FileTracker) { remoteWorkflowLog.Printf("Fetching import-derived dispatch workflows from parsed file: %s, repo=%s", destFile, spec.RepoSlug) - if spec.RepoSlug == "" { + owner, repo, ref, ok := fetchAndSaveDispatchWorkflowsFromParsedFileRepo(spec) + if !ok { return } + workflowNames, originalCount := fetchAndSaveDispatchWorkflowsFromParsedFileNames(destFile) + if len(workflowNames) == 0 { + return + } + remoteWorkflowLog.Printf("Processing %d import-derived dispatch workflow(s) (filtered from %d)", len(workflowNames), originalCount) + + workflowBaseDir := getParentDir(spec.WorkflowPath) + absTargetDir, absErr := filepath.Abs(targetDir) + if absErr != nil { + remoteWorkflowLog.Printf("Failed to resolve absolute path for target directory %s: %v", targetDir, absErr) + return + } + for _, workflowName := range workflowNames { + fetchAndSaveDispatchWorkflowsFromParsedFileOne(fetchAndSaveDispatchWorkflowsFromParsedFileOneParams{ + Ctx: ctx, + WorkflowName: workflowName, + Spec: spec, + TargetDir: targetDir, + Verbose: verbose, + Force: force, + Tracker: tracker, + Owner: owner, + Repo: repo, + Ref: ref, + WorkflowBaseDir: workflowBaseDir, + AbsTargetDir: absTargetDir, + }) + } +} + +func fetchAndSaveDispatchWorkflowsFromParsedFileRepo(spec *WorkflowSpec) (string, string, string, bool) { + if spec.RepoSlug == "" { + return "", "", "", false + } parts := strings.SplitN(spec.RepoSlug, "/", 2) if len(parts) != 2 { - return + return "", "", "", false } - owner, repo := parts[0], parts[1] ref := spec.Version if ref == "" { ref = "main" } + return parts[0], parts[1], ref, true +} - // Parse the locally-saved workflow to get the full merged safe-outputs config. +func fetchAndSaveDispatchWorkflowsFromParsedFileNames(destFile string) ([]string, int) { compiler := workflow.NewCompiler() data, err := compiler.ParseWorkflowFile(destFile) if err != nil { remoteWorkflowLog.Printf("Failed to parse workflow file %s for import-derived dispatch workflows: %v", destFile, err) - return + return nil, 0 } if data == nil || data.SafeOutputs == nil || data.SafeOutputs.DispatchWorkflow == nil { - return + return nil, 0 } - workflowNames := data.SafeOutputs.DispatchWorkflow.Workflows - if len(workflowNames) == 0 { - return - } - - // Filter out GitHub Actions expression syntax filtered := make([]string, 0, len(workflowNames)) for _, name := range workflowNames { if !strings.Contains(name, "${{") { filtered = append(filtered, name) } } - if len(filtered) == 0 { - return - } - - remoteWorkflowLog.Printf("Processing %d import-derived dispatch workflow(s) (filtered from %d)", len(filtered), len(workflowNames)) + return filtered, len(workflowNames) +} - workflowBaseDir := getParentDir(spec.WorkflowPath) +type fetchAndSaveDispatchWorkflowsFromParsedFileOneParams struct { + Ctx context.Context + WorkflowName string + Spec *WorkflowSpec + TargetDir string + Verbose bool + Force bool + Tracker *FileTracker + Owner string + Repo string + Ref string + WorkflowBaseDir string + AbsTargetDir string +} - absTargetDir, absErr := filepath.Abs(targetDir) - if absErr != nil { - remoteWorkflowLog.Printf("Failed to resolve absolute path for target directory %s: %v", targetDir, absErr) +func fetchAndSaveDispatchWorkflowsFromParsedFileOne(p fetchAndSaveDispatchWorkflowsFromParsedFileOneParams) { + remoteFilePath, targetPath, ok := fetchAndSaveDispatchWorkflowsFromParsedFilePaths(p.WorkflowName, p.WorkflowBaseDir, p.TargetDir, p.AbsTargetDir, p.Verbose) + if !ok { + return + } + fileExists, skip := fetchAndSaveDispatchWorkflowsFromParsedFileExisting(p.WorkflowName, targetPath, p.Spec.RepoSlug, p.Verbose, p.Force) + if skip { + return + } + workflowContent, err := parser.DownloadFileFromGitHub(p.Ctx, p.Owner, p.Repo, remoteFilePath, p.Ref) + if err != nil { + fetchAndSaveDispatchWorkflowsFromParsedFileYMLFallback(fetchAndSaveDispatchWorkflowsFromParsedFileYMLFallbackParams{ + Ctx: p.Ctx, + WorkflowName: p.WorkflowName, + RemoteFilePath: remoteFilePath, + TargetDir: p.TargetDir, + Verbose: p.Verbose, + Tracker: p.Tracker, + Owner: p.Owner, + Repo: p.Repo, + Ref: p.Ref, + OriginalErr: err, + }) return } + fetchAndSaveDispatchWorkflowsFromParsedFileWriteMarkdown(fetchAndSaveDispatchWorkflowsFromParsedFileWriteMarkdownParams{ + Ctx: p.Ctx, + WorkflowContent: workflowContent, + Spec: p.Spec, + RemoteFilePath: remoteFilePath, + TargetPath: targetPath, + TargetDir: p.TargetDir, + Verbose: p.Verbose, + Force: p.Force, + Tracker: p.Tracker, + FileExists: fileExists, + Ref: p.Ref, + }) +} - for _, workflowName := range filtered { - // Early rejection of path traversal patterns (authoritative check is filepath.Rel below). - if strings.Contains(workflowName, "..") { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping dispatch workflow with unsafe name: %q", workflowName))) - } - continue +func fetchAndSaveDispatchWorkflowsFromParsedFilePaths(workflowName, workflowBaseDir, targetDir, absTargetDir string, verbose bool) (string, string, bool) { + if strings.Contains(workflowName, "..") { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping dispatch workflow with unsafe name: %q", workflowName))) } - - var remoteFilePath string - if workflowBaseDir != "" { - remoteFilePath = path.Join(workflowBaseDir, workflowName+".md") - } else { - remoteFilePath = workflowName + ".md" + return "", "", false + } + var remoteFilePath string + if workflowBaseDir != "" { + remoteFilePath = path.Join(workflowBaseDir, workflowName+".md") + } else { + remoteFilePath = workflowName + ".md" + } + remoteFilePath = path.Clean(remoteFilePath) + targetPath := filepath.Join(targetDir, filepath.Clean(workflowName+".md")) + absTargetPath, absErr := filepath.Abs(targetPath) + if absErr != nil { + remoteWorkflowLog.Printf("Failed to resolve absolute path for dispatch workflow %s: %v", workflowName, absErr) + return "", "", false + } + if rel, relErr := filepath.Rel(absTargetDir, absTargetPath); relErr != nil || strings.HasPrefix(rel, "..") { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Refusing to write dispatch workflow outside target directory: %q", workflowName))) } - remoteFilePath = path.Clean(remoteFilePath) - - localRelPath := filepath.Clean(workflowName + ".md") - targetPath := filepath.Join(targetDir, localRelPath) + return "", "", false + } + return remoteFilePath, targetPath, true +} - absTargetPath, absErr2 := filepath.Abs(targetPath) - if absErr2 != nil { - remoteWorkflowLog.Printf("Failed to resolve absolute path for dispatch workflow %s: %v", workflowName, absErr2) - continue - } - if rel, relErr := filepath.Rel(absTargetDir, absTargetPath); relErr != nil || strings.HasPrefix(rel, "..") { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Refusing to write dispatch workflow outside target directory: %q", workflowName))) - } - continue +func fetchAndSaveDispatchWorkflowsFromParsedFileExisting(workflowName, targetPath, repoSlug string, verbose bool, force bool) (bool, bool) { + if _, statErr := os.Stat(targetPath); statErr != nil { + return false, false + } + if force { + return true, false + } + existingSourceRepo := readSourceRepoFromFile(targetPath) + if existingSourceRepo == repoSlug { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Dispatch workflow (from import) from same source already exists, skipping: "+targetPath)) } + return true, true + } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf( + "Dispatch workflow %q already exists at %s from a different source (existing: %q, needed: %q); use --force to overwrite", + workflowName, targetPath, sourceRepoLabel(existingSourceRepo), repoSlug, + ))) + return true, true +} - // Check whether the target file already exists. - fileExists := false - if _, statErr := os.Stat(targetPath); statErr == nil { - fileExists = true - if !force { - existingSourceRepo := readSourceRepoFromFile(targetPath) - if existingSourceRepo == spec.RepoSlug { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Dispatch workflow (from import) from same source already exists, skipping: "+targetPath)) - } - continue - } - // Different or missing source — warn and skip (post-write best-effort). - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf( - "Dispatch workflow %q already exists at %s from a different source (existing: %q, needed: %q); use --force to overwrite", - workflowName, targetPath, sourceRepoLabel(existingSourceRepo), spec.RepoSlug, - ))) - continue - } - } +type fetchAndSaveDispatchWorkflowsFromParsedFileYMLFallbackParams struct { + Ctx context.Context + WorkflowName string + RemoteFilePath string + TargetDir string + Verbose bool + Tracker *FileTracker + Owner string + Repo string + Ref string + OriginalErr error +} - // Download from source repository — try .md first, then .yml as fallback - workflowContent, err := parser.DownloadFileFromGitHub(ctx, owner, repo, remoteFilePath, ref) - if err != nil { - // .md not found — try .yml fallback - ymlRemotePath := path.Clean(strings.TrimSuffix(remoteFilePath, ".md") + ".yml") - ymlLocalPath := filepath.Join(targetDir, filepath.Clean(workflowName+".yml")) - - ymlContent, ymlErr := parser.DownloadFileFromGitHub(ctx, owner, repo, ymlRemotePath, ref) - if ymlErr != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch dispatch workflow %s: %v", remoteFilePath, err))) - } - continue - } - if mkErr := os.MkdirAll(filepath.Dir(ymlLocalPath), constants.DirPermPublic); mkErr != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for dispatch workflow %s: %v", ymlRemotePath, mkErr))) - } - continue - } - // Capture whether file exists before writing (for correct tracker classification). - _, ymlFileExistsErr := os.Stat(ymlLocalPath) - ymlFileExists := ymlFileExistsErr == nil - if writeErr := os.WriteFile(ymlLocalPath, ymlContent, constants.FilePermSensitive); writeErr != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write dispatch workflow %s: %v", ymlRemotePath, writeErr))) - } - continue - } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched dispatch workflow (.yml, from import): "+ymlLocalPath)) - } - if tracker != nil { - if ymlFileExists { - tracker.TrackModified(ymlLocalPath) - } else { - tracker.TrackCreated(ymlLocalPath) - } - } - continue +func fetchAndSaveDispatchWorkflowsFromParsedFileYMLFallback(p fetchAndSaveDispatchWorkflowsFromParsedFileYMLFallbackParams) { + ymlRemotePath := path.Clean(strings.TrimSuffix(p.RemoteFilePath, ".md") + ".yml") + ymlLocalPath := filepath.Join(p.TargetDir, filepath.Clean(p.WorkflowName+".yml")) + ymlContent, ymlErr := parser.DownloadFileFromGitHub(p.Ctx, p.Owner, p.Repo, ymlRemotePath, p.Ref) + if ymlErr != nil { + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch dispatch workflow %s: %v", p.RemoteFilePath, p.OriginalErr))) } + return + } + fetchAndSaveDispatchWorkflowsFromParsedFileWriteYML(ymlRemotePath, ymlLocalPath, ymlContent, p.Verbose, p.Tracker) +} - // Embed the source field for future conflict detection. - depSourceString := spec.RepoSlug + "/" + remoteFilePath + "@" + ref - if updated, srcErr := addSourceToWorkflow(string(workflowContent), depSourceString); srcErr == nil { - workflowContent = []byte(updated) +func fetchAndSaveDispatchWorkflowsFromParsedFileWriteYML(ymlRemotePath, ymlLocalPath string, ymlContent []byte, verbose bool, tracker *FileTracker) { + if mkErr := os.MkdirAll(filepath.Dir(ymlLocalPath), constants.DirPermPublic); mkErr != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for dispatch workflow %s: %v", ymlRemotePath, mkErr))) } - - if err := os.MkdirAll(filepath.Dir(targetPath), constants.DirPermPublic); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for dispatch workflow %s: %v", remoteFilePath, err))) - } - continue + return + } + _, ymlFileExistsErr := os.Stat(ymlLocalPath) + ymlFileExists := ymlFileExistsErr == nil + if writeErr := os.WriteFile(ymlLocalPath, ymlContent, constants.FilePermSensitive); writeErr != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write dispatch workflow %s: %v", ymlRemotePath, writeErr))) } + return + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched dispatch workflow (.yml, from import): "+ymlLocalPath)) + } + fetchAndSaveDispatchWorkflowsFromParsedFileTrack(tracker, ymlLocalPath, ymlFileExists) +} - if err := os.WriteFile(targetPath, workflowContent, constants.FilePermSensitive); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write dispatch workflow %s: %v", remoteFilePath, err))) - } - continue - } +type fetchAndSaveDispatchWorkflowsFromParsedFileWriteMarkdownParams struct { + Ctx context.Context + WorkflowContent []byte + Spec *WorkflowSpec + RemoteFilePath string + TargetPath string + TargetDir string + Verbose bool + Force bool + Tracker *FileTracker + FileExists bool + Ref string +} - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched dispatch workflow (from import): "+targetPath)) +func fetchAndSaveDispatchWorkflowsFromParsedFileWriteMarkdown(p fetchAndSaveDispatchWorkflowsFromParsedFileWriteMarkdownParams) { + depSourceString := p.Spec.RepoSlug + "/" + p.RemoteFilePath + "@" + p.Ref + if updated, srcErr := addSourceToWorkflow(string(p.WorkflowContent), depSourceString); srcErr == nil { + p.WorkflowContent = []byte(updated) + } + if err := os.MkdirAll(filepath.Dir(p.TargetPath), constants.DirPermPublic); err != nil { + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for dispatch workflow %s: %v", p.RemoteFilePath, err))) } - - if tracker != nil { - if fileExists { - tracker.TrackModified(targetPath) - } else { - tracker.TrackCreated(targetPath) - } + return + } + if err := os.WriteFile(p.TargetPath, p.WorkflowContent, constants.FilePermSensitive); err != nil { + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write dispatch workflow %s: %v", p.RemoteFilePath, err))) } + return + } + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched dispatch workflow (from import): "+p.TargetPath)) + } + fetchAndSaveDispatchWorkflowsFromParsedFileTrack(p.Tracker, p.TargetPath, p.FileExists) + fetchDownloadedWorkflowFrontmatterImports(p.Ctx, p.WorkflowContent, p.Spec, p.RemoteFilePath, p.TargetDir, p.Verbose, p.Force, p.Tracker) +} - fetchDownloadedWorkflowFrontmatterImports(ctx, workflowContent, spec, remoteFilePath, targetDir, verbose, force, tracker) +func fetchAndSaveDispatchWorkflowsFromParsedFileTrack(tracker *FileTracker, path string, fileExists bool) { + if tracker == nil { + return + } + if fileExists { + tracker.TrackModified(path) + } else { + tracker.TrackCreated(path) } } diff --git a/pkg/cli/docker_images.go b/pkg/cli/docker_images.go index 45a37af4285..fdcc4a73706 100644 --- a/pkg/cli/docker_images.go +++ b/pkg/cli/docker_images.go @@ -148,73 +148,67 @@ func StartDockerImageDownload(ctx context.Context, image string) bool { pullState.downloading[image] = true // Start the download in a goroutine with retry logic - go func() { - defer func() { - func() { - pullState.mu.Lock() - defer pullState.mu.Unlock() - delete(pullState.downloading, image) - }() - if r := recover(); r != nil { - dockerImagesLog.Printf("Panic in docker image download for %s (recovered): %v", image, r) - } - }() - - dockerImagesLog.Printf("Starting download of image %s", image) - - // Retry configuration - maxAttempts := 3 - waitTime := 5 // seconds - - var lastErr error - var lastOutput []byte - - for attempt := 1; attempt <= maxAttempts; attempt++ { - // Check if context was cancelled - if ctx.Err() != nil { - dockerImagesLog.Printf("Download of image %s cancelled: %v", image, ctx.Err()) - return - } - - dockerImagesLog.Printf("Attempt %d of %d: Pulling image %s", attempt, maxAttempts, image) - - cmd := exec.CommandContext(ctx, "docker", "pull", image) - output, err := cmd.CombinedOutput() - - if err == nil { - // Success - dockerImagesLog.Printf("Successfully downloaded image %s", image) - return - } - - lastErr = err - lastOutput = output - - // If not the last attempt, wait and retry - if attempt < maxAttempts { - dockerImagesLog.Printf("Failed to download image %s (attempt %d/%d). Retrying in %ds...", image, attempt, maxAttempts, waitTime) - - // Use context-aware sleep - timer := time.NewTimer(time.Duration(waitTime) * time.Second) - select { - case <-timer.C: - // Continue to next retry - case <-ctx.Done(): - timer.Stop() - // Context cancelled during sleep - dockerImagesLog.Printf("Download of image %s cancelled during retry wait: %v", image, ctx.Err()) - return - } - - waitTime *= 2 // Exponential backoff - } + go startDockerImageDownloadPull(ctx, image) + + return true +} + +func startDockerImageDownloadPull(ctx context.Context, image string) { + defer startDockerImageDownloadCleanup(image) + dockerImagesLog.Printf("Starting download of image %s", image) + + maxAttempts := 3 + waitTime := 5 // seconds + var lastErr error + var lastOutput []byte + + for attempt := 1; attempt <= maxAttempts; attempt++ { + if ctx.Err() != nil { + dockerImagesLog.Printf("Download of image %s cancelled: %v", image, ctx.Err()) + return + } + + dockerImagesLog.Printf("Attempt %d of %d: Pulling image %s", attempt, maxAttempts, image) + cmd := exec.CommandContext(ctx, "docker", "pull", image) + output, err := cmd.CombinedOutput() + if err == nil { + dockerImagesLog.Printf("Successfully downloaded image %s", image) + return } - // All attempts failed - dockerImagesLog.Printf("Failed to download image %s after %d attempts: %v\nOutput: %s", image, maxAttempts, lastErr, string(lastOutput)) + lastErr = err + lastOutput = output + if attempt < maxAttempts && !startDockerImageDownloadWaitForRetry(ctx, image, attempt, maxAttempts, &waitTime) { + return + } + } + + dockerImagesLog.Printf("Failed to download image %s after %d attempts: %v\nOutput: %s", image, maxAttempts, lastErr, string(lastOutput)) +} + +func startDockerImageDownloadCleanup(image string) { + func() { + pullState.mu.Lock() + defer pullState.mu.Unlock() + delete(pullState.downloading, image) }() + if r := recover(); r != nil { + dockerImagesLog.Printf("Panic in docker image download for %s (recovered): %v", image, r) + } +} - return true +func startDockerImageDownloadWaitForRetry(ctx context.Context, image string, attempt int, maxAttempts int, waitTime *int) bool { + dockerImagesLog.Printf("Failed to download image %s (attempt %d/%d). Retrying in %ds...", image, attempt, maxAttempts, *waitTime) + timer := time.NewTimer(time.Duration(*waitTime) * time.Second) + select { + case <-timer.C: + *waitTime *= 2 // Exponential backoff + return true + case <-ctx.Done(): + timer.Stop() + dockerImagesLog.Printf("Download of image %s cancelled during retry wait: %v", image, ctx.Err()) + return false + } } // CheckAndPrepareDockerImages checks if required Docker images are available @@ -232,35 +226,7 @@ func CheckAndPrepareDockerImages(ctx context.Context, useZizmor, usePoutine, use // Check if Docker daemon is available before attempting any image operations if !IsDockerAvailable(ctx) { - var requestedTools []string - var paramsList []string - if useZizmor { - tool := "zizmor" - requestedTools = append(requestedTools, tool) - paramsList = append(paramsList, tool+": false") - } - if usePoutine { - tool := "poutine" - requestedTools = append(requestedTools, tool) - paramsList = append(paramsList, tool+": false") - } - if useActionlint { - tool := "actionlint" - requestedTools = append(requestedTools, tool) - paramsList = append(paramsList, tool+": false") - } - if useRunnerGuard { - tool := "runner-guard" - requestedTools = append(requestedTools, tool) - paramsList = append(paramsList, tool+": false") - } - verb := "requires" - if len(requestedTools) > 1 { - verb = "require" - } - return &DockerUnavailableError{ - Message: fmt.Sprintf("docker is not available (cannot connect to Docker daemon). %s %s Docker. Please install and start Docker, or set %s to skip static analysis", strings.Join(requestedTools, " and "), verb, strings.Join(paramsList, " and ")), - } + return checkAndPrepareDockerImagesUnavailable(useZizmor, usePoutine, useActionlint, useRunnerGuard) } var missingImages []string @@ -298,25 +264,48 @@ func CheckAndPrepareDockerImages(ctx context.Context, useZizmor, usePoutine, use // If any images are downloading or were just started if len(downloadingImages) > 0 || len(missingImages) > 0 { - var msg strings.Builder - msg.WriteString("Docker images are being downloaded. Please wait and retry the compile command.\n\n") - - if len(missingImages) > 0 { - msg.WriteString("Started downloading: ") - msg.WriteString(strings.Join(missingImages, ", ")) - msg.WriteString("\n") - } + return errors.New(checkAndPrepareDockerImagesDownloadMessage(missingImages, downloadingImages)) + } - if len(downloadingImages) > 0 { - msg.WriteString("Currently downloading: ") - msg.WriteString(strings.Join(downloadingImages, ", ")) - msg.WriteString("\n") - } + return nil +} - msg.WriteString("\nRetry in 15-30 seconds.") +func checkAndPrepareDockerImagesUnavailable(useZizmor, usePoutine, useActionlint, useRunnerGuard bool) error { + var requestedTools []string + var paramsList []string + checkAndPrepareDockerImagesAppendTool(useZizmor, "zizmor", &requestedTools, ¶msList) + checkAndPrepareDockerImagesAppendTool(usePoutine, "poutine", &requestedTools, ¶msList) + checkAndPrepareDockerImagesAppendTool(useActionlint, "actionlint", &requestedTools, ¶msList) + checkAndPrepareDockerImagesAppendTool(useRunnerGuard, "runner-guard", &requestedTools, ¶msList) + verb := "requires" + if len(requestedTools) > 1 { + verb = "require" + } + return &DockerUnavailableError{ + Message: fmt.Sprintf("docker is not available (cannot connect to Docker daemon). %s %s Docker. Please install and start Docker, or set %s to skip static analysis", strings.Join(requestedTools, " and "), verb, strings.Join(paramsList, " and ")), + } +} - return errors.New(msg.String()) +func checkAndPrepareDockerImagesAppendTool(use bool, tool string, requestedTools *[]string, paramsList *[]string) { + if use { + *requestedTools = append(*requestedTools, tool) + *paramsList = append(*paramsList, tool+": false") } +} - return nil +func checkAndPrepareDockerImagesDownloadMessage(missingImages []string, downloadingImages []string) string { + var msg strings.Builder + msg.WriteString("Docker images are being downloaded. Please wait and retry the compile command.\n\n") + if len(missingImages) > 0 { + msg.WriteString("Started downloading: ") + msg.WriteString(strings.Join(missingImages, ", ")) + msg.WriteString("\n") + } + if len(downloadingImages) > 0 { + msg.WriteString("Currently downloading: ") + msg.WriteString(strings.Join(downloadingImages, ", ")) + msg.WriteString("\n") + } + msg.WriteString("\nRetry in 15-30 seconds.") + return msg.String() } diff --git a/pkg/cli/download_workflow.go b/pkg/cli/download_workflow.go index 19d5b74da34..00b7f8d3079 100644 --- a/pkg/cli/download_workflow.go +++ b/pkg/cli/download_workflow.go @@ -76,74 +76,91 @@ func downloadWorkflowContentViaGitClone(ctx context.Context, repo, path, ref str githubHost := getGitHubHostForRepo(repo) repoURL := fmt.Sprintf("%s/%s.git", githubHost, repo) + if err := downloadWorkflowContentViaGitClonePrepare(ctx, tmpDir, repoURL, path); err != nil { + return nil, err + } + + if err := downloadWorkflowContentViaGitCloneFetch(ctx, tmpDir, ref); err != nil { + return nil, err + } + + content, err := downloadWorkflowContentViaGitCloneRead(tmpDir, path) + if err != nil { + return nil, err + } + downloadLog.Printf("Read cloned file: path=%s, size=%d bytes", path, len(content)) + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Successfully fetched via git sparse checkout")) + } + + return content, nil +} + +func downloadWorkflowContentViaGitClonePrepare(ctx context.Context, tmpDir, repoURL, path string) error { // Initialize git repository initCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "init") if output, err := initCmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to initialize git repository: %w\nOutput: %s", err, string(output)) + return fmt.Errorf("failed to initialize git repository: %w\nOutput: %s", err, string(output)) } - - // Add remote remoteCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "remote", "add", "origin", repoURL) if output, err := remoteCmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to add remote: %w\nOutput: %s", err, string(output)) + return fmt.Errorf("failed to add remote: %w\nOutput: %s", err, string(output)) } - - // Enable sparse-checkout sparseCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "config", "core.sparseCheckout", "true") if output, err := sparseCmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to enable sparse checkout: %w\nOutput: %s", err, string(output)) + return fmt.Errorf("failed to enable sparse checkout: %w\nOutput: %s", err, string(output)) } + return downloadWorkflowContentViaGitCloneWriteSparseCheckout(tmpDir, path) +} - // Set sparse-checkout pattern to only include the file we need +func downloadWorkflowContentViaGitCloneWriteSparseCheckout(tmpDir, path string) error { sparseInfoDir := filepath.Join(tmpDir, ".git", "info") if err := os.MkdirAll(sparseInfoDir, constants.DirPermPublic); err != nil { - return nil, fmt.Errorf("failed to create sparse-checkout directory: %w", err) + return fmt.Errorf("failed to create sparse-checkout directory: %w", err) } - sparseCheckoutFile := filepath.Join(sparseInfoDir, "sparse-checkout") // Use owner-only read/write permissions (0600) for security best practices if err := os.WriteFile(sparseCheckoutFile, []byte(path+"\n"), constants.FilePermSensitive); err != nil { - return nil, fmt.Errorf("failed to write sparse-checkout file: %w", err) + return fmt.Errorf("failed to write sparse-checkout file: %w", err) } + return nil +} - // Check if ref is a SHA (40 hex characters) +func downloadWorkflowContentViaGitCloneFetch(ctx context.Context, tmpDir, ref string) error { isSHA := len(ref) == 40 && gitutil.IsHexString(ref) downloadLog.Printf("Fetching ref via sparse checkout: is_sha=%t", isSHA) - if isSHA { - // For SHA refs, fetch without specifying a ref (fetch all) then checkout the specific commit - // Note: sparse-checkout with SHA refs may not reduce bandwidth as much as with branch refs, - // because the server needs to send enough history to reach the specific commit. - // However, it still limits the working directory to only the requested file. - fetchCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "fetch", "--depth", "1", "origin", ref) - if _, err := fetchCmd.CombinedOutput(); err != nil { - // If fetching specific SHA fails, try fetching all branches with depth 1 - fetchCmd = exec.CommandContext(ctx, "git", "-C", tmpDir, "fetch", "--depth", "1", "origin") - if output, err := fetchCmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to fetch repository: %w\nOutput: %s", err, string(output)) - } - } + return downloadWorkflowContentViaGitCloneFetchSHA(ctx, tmpDir, ref) + } + fetchCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "fetch", "--depth", "1", "origin", ref) + if output, err := fetchCmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to fetch ref %s: %w\nOutput: %s", ref, err, string(output)) + } + checkoutCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "checkout", "FETCH_HEAD") + if output, err := checkoutCmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to checkout FETCH_HEAD: %w\nOutput: %s", err, string(output)) + } + return nil +} - // Checkout the specific commit - checkoutCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "checkout", ref) - if output, err := checkoutCmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to checkout commit %s: %w\nOutput: %s", ref, err, string(output)) - } - } else { - // For branch/tag refs, fetch the specific ref - fetchCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "fetch", "--depth", "1", "origin", ref) +func downloadWorkflowContentViaGitCloneFetchSHA(ctx context.Context, tmpDir, ref string) error { + // For SHA refs, fetch without specifying a ref (fetch all) then checkout the specific commit. + fetchCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "fetch", "--depth", "1", "origin", ref) + if _, err := fetchCmd.CombinedOutput(); err != nil { + fetchCmd = exec.CommandContext(ctx, "git", "-C", tmpDir, "fetch", "--depth", "1", "origin") if output, err := fetchCmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to fetch ref %s: %w\nOutput: %s", ref, err, string(output)) - } - - // Checkout FETCH_HEAD - checkoutCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "checkout", "FETCH_HEAD") - if output, err := checkoutCmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("failed to checkout FETCH_HEAD: %w\nOutput: %s", err, string(output)) + return fmt.Errorf("failed to fetch repository: %w\nOutput: %s", err, string(output)) } } + checkoutCmd := exec.CommandContext(ctx, "git", "-C", tmpDir, "checkout", ref) + if output, err := checkoutCmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to checkout commit %s: %w\nOutput: %s", ref, err, string(output)) + } + return nil +} - // Read the file +func downloadWorkflowContentViaGitCloneRead(tmpDir, path string) ([]byte, error) { filePath := filepath.Join(tmpDir, path) if err := fileutil.ValidatePathWithinBase(tmpDir, filePath); err != nil { downloadLog.Printf("Refusing to read file outside clone directory: %s", filePath) @@ -154,12 +171,6 @@ func downloadWorkflowContentViaGitClone(ctx context.Context, repo, path, ref str downloadLog.Printf("Failed to read cloned file: path=%s, err=%v", filePath, err) return nil, fmt.Errorf("failed to read file from cloned repository: %w", err) } - downloadLog.Printf("Read cloned file: path=%s, size=%d bytes", path, len(content)) - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Successfully fetched via git sparse checkout")) - } - return content, nil } diff --git a/pkg/cli/drain3_integration.go b/pkg/cli/drain3_integration.go index 40af6b519f2..d5853d8ecd9 100644 --- a/pkg/cli/drain3_integration.go +++ b/pkg/cli/drain3_integration.go @@ -126,9 +126,19 @@ func buildDrain3InsightsMultiRun(processedRuns []ProcessedRun) []ObservabilityIn func buildAgentEventsFromProcessedRun(processedRun ProcessedRun, metrics MetricsData, toolUsage []ToolUsageInfo) []agentdrain.AgentEvent { var events []agentdrain.AgentEvent + buildAgentEventsFromProcessedRunPlan(&events, metrics) + buildAgentEventsFromProcessedRunToolUsage(&events, toolUsage) + buildAgentEventsFromProcessedRunErrors(&events, processedRun) + buildAgentEventsFromProcessedRunNoops(&events, processedRun) + buildAgentEventsFromProcessedRunFinish(&events, processedRun, metrics) + + return events +} + +func buildAgentEventsFromProcessedRunPlan(events *[]agentdrain.AgentEvent, metrics MetricsData) { // Synthesise a planning event from overall metrics. if metrics.Turns > 0 { - events = append(events, agentdrain.AgentEvent{ + *events = append(*events, agentdrain.AgentEvent{ Stage: "plan", Fields: map[string]string{ "turns": strconv.Itoa(metrics.Turns), @@ -136,10 +146,11 @@ func buildAgentEventsFromProcessedRun(processedRun ProcessedRun, metrics Metrics }, }) } +} - // Tool-call events from the per-tool usage summary. +func buildAgentEventsFromProcessedRunToolUsage(events *[]agentdrain.AgentEvent, toolUsage []ToolUsageInfo) { for _, tu := range toolUsage { - events = append(events, agentdrain.AgentEvent{ + *events = append(*events, agentdrain.AgentEvent{ Stage: "tool_call", Fields: map[string]string{ "tool": tu.Name, @@ -147,10 +158,17 @@ func buildAgentEventsFromProcessedRun(processedRun ProcessedRun, metrics Metrics }, }) } +} + +func buildAgentEventsFromProcessedRunErrors(events *[]agentdrain.AgentEvent, processedRun ProcessedRun) { + buildAgentEventsFromProcessedRunMCPFailures(events, processedRun) + buildAgentEventsFromProcessedRunMissingTools(events, processedRun) + buildAgentEventsFromProcessedRunMissingData(events, processedRun) +} - // MCP failures become error-stage events. +func buildAgentEventsFromProcessedRunMCPFailures(events *[]agentdrain.AgentEvent, processedRun ProcessedRun) { for _, f := range processedRun.MCPFailures { - events = append(events, agentdrain.AgentEvent{ + *events = append(*events, agentdrain.AgentEvent{ Stage: "error", Fields: map[string]string{ "type": "mcp_failure", @@ -159,10 +177,11 @@ func buildAgentEventsFromProcessedRun(processedRun ProcessedRun, metrics Metrics }, }) } +} - // Missing tools are capability-friction errors. +func buildAgentEventsFromProcessedRunMissingTools(events *[]agentdrain.AgentEvent, processedRun ProcessedRun) { for _, mt := range processedRun.MissingTools { - events = append(events, agentdrain.AgentEvent{ + *events = append(*events, agentdrain.AgentEvent{ Stage: "error", Fields: map[string]string{ "type": "missing_tool", @@ -171,10 +190,11 @@ func buildAgentEventsFromProcessedRun(processedRun ProcessedRun, metrics Metrics }, }) } +} - // Missing data is a different error class. +func buildAgentEventsFromProcessedRunMissingData(events *[]agentdrain.AgentEvent, processedRun ProcessedRun) { for _, md := range processedRun.MissingData { - events = append(events, agentdrain.AgentEvent{ + *events = append(*events, agentdrain.AgentEvent{ Stage: "error", Fields: map[string]string{ "type": "missing_data", @@ -183,10 +203,11 @@ func buildAgentEventsFromProcessedRun(processedRun ProcessedRun, metrics Metrics }, }) } +} - // No-ops map to tool_result stage. +func buildAgentEventsFromProcessedRunNoops(events *[]agentdrain.AgentEvent, processedRun ProcessedRun) { for _, n := range processedRun.Noops { - events = append(events, agentdrain.AgentEvent{ + *events = append(*events, agentdrain.AgentEvent{ Stage: "tool_result", Fields: map[string]string{ "status": "noop", @@ -194,14 +215,16 @@ func buildAgentEventsFromProcessedRun(processedRun ProcessedRun, metrics Metrics }, }) } +} +func buildAgentEventsFromProcessedRunFinish(events *[]agentdrain.AgentEvent, processedRun ProcessedRun, metrics MetricsData) { // Synthesise a finish event only when the run has a meaningful conclusion. conclusion := processedRun.Run.Conclusion if conclusion == "" { conclusion = processedRun.Run.Status } if conclusion != "" || metrics.TokenUsage > 0 { - events = append(events, agentdrain.AgentEvent{ + *events = append(*events, agentdrain.AgentEvent{ Stage: "finish", Fields: map[string]string{ "status": conclusion, @@ -209,8 +232,6 @@ func buildAgentEventsFromProcessedRun(processedRun ProcessedRun, metrics Metrics }, }) } - - return events } // buildInsightsFromDrain3Analysis converts drain3 coordinator analysis into diff --git a/pkg/cli/drain3_train.go b/pkg/cli/drain3_train.go index 56eef360124..708c6ee7867 100644 --- a/pkg/cli/drain3_train.go +++ b/pkg/cli/drain3_train.go @@ -30,10 +30,31 @@ func TrainDrain3Weights(processedRuns []ProcessedRun, outputDir string, verbose fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Training log pattern weights from %d run(s)...", len(processedRuns)))) + weightsData, err := trainDrain3WeightsBuildWeights(processedRuns, verbose) + if err != nil { + return err + } + + outputPath := filepath.Join(outputDir, drain3WeightsFilename) + if err := os.WriteFile(outputPath, weightsData, constants.FilePermPublic); err != nil { + return fmt.Errorf("log pattern training: write weights file: %w", err) + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Log pattern weights written to: "+outputPath)) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage( + "To embed these weights as default, copy the file and rebuild:\n"+ + " cp "+outputPath+" pkg/agentdrain/data/default_weights.json\n"+ + " make build", + )) + + return nil +} + +func trainDrain3WeightsBuildWeights(processedRuns []ProcessedRun, verbose bool) ([]byte, error) { cfg := agentdrain.DefaultConfig() coordinator, err := agentdrain.NewCoordinator(cfg, defaultAgentDrainStages) if err != nil { - return fmt.Errorf("log pattern training: create coordinator: %w", err) + return nil, fmt.Errorf("log pattern training: create coordinator: %w", err) } totalEvents := 0 @@ -66,9 +87,13 @@ func TrainDrain3Weights(processedRuns []ProcessedRun, outputDir string, verbose weightsData, err := coordinator.SaveWeightsJSON() if err != nil { - return fmt.Errorf("log pattern training: serialize weights: %w", err) + return nil, fmt.Errorf("log pattern training: serialize weights: %w", err) } + return trainDrain3WeightsPrettyPrintWeights(weightsData), nil +} + +func trainDrain3WeightsPrettyPrintWeights(weightsData []byte) []byte { // Pretty-print the weights for readability. var raw map[string]any if unmarshalErr := json.Unmarshal(weightsData, &raw); unmarshalErr != nil { @@ -79,17 +104,5 @@ func TrainDrain3Weights(processedRuns []ProcessedRun, outputDir string, verbose weightsData = pretty } - outputPath := filepath.Join(outputDir, drain3WeightsFilename) - if err := os.WriteFile(outputPath, weightsData, constants.FilePermPublic); err != nil { - return fmt.Errorf("log pattern training: write weights file: %w", err) - } - - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Log pattern weights written to: "+outputPath)) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage( - "To embed these weights as default, copy the file and rebuild:\n"+ - " cp "+outputPath+" pkg/agentdrain/data/default_weights.json\n"+ - " make build", - )) - - return nil + return weightsData } diff --git a/pkg/cli/enable.go b/pkg/cli/enable.go index 540426b6aa8..ccec94a3d57 100644 --- a/pkg/cli/enable.go +++ b/pkg/cli/enable.go @@ -21,6 +21,14 @@ import ( var enableLog = logger.New("cli:enable") +type toggleWorkflowsByNamesTarget struct { + Name string + ID int64 // 0 if unknown + LockFileBase string // e.g., dev.lock.yml + CurrentState string // known state or "unknown" + HasGitHubEntry bool +} + // EnableWorkflowsByNames enables workflows by specific names, or all if no names provided func EnableWorkflowsByNames(ctx context.Context, workflowNames []string, repoOverride string) error { enableLog.Printf("EnableWorkflowsByNames called: workflow_count=%d, repo=%s", len(workflowNames), repoOverride) @@ -35,37 +43,12 @@ func DisableWorkflowsByNames(ctx context.Context, workflowNames []string, repoOv // toggleWorkflowsByNames toggles workflows by specific names, or all if no names provided func toggleWorkflowsByNames(ctx context.Context, workflowNames []string, enable bool, repoOverride string) error { - action := "enable" - if !enable { - action = "disable" - } - + action := toggleWorkflowsByNamesAction(enable) enableLog.Printf("Toggle workflows: action=%s, count=%d, repo=%s", action, len(workflowNames), repoOverride) // If no specific workflow names provided, enable/disable all workflows if len(workflowNames) == 0 { - enableLog.Print("No specific workflows provided, processing all workflows") - fmt.Fprintf(os.Stderr, "No specific workflows provided. %sing all workflows...\n", strings.ToUpper(action[:1])+action[1:]) - // Get all workflow names and process them - mdFiles, err := getMarkdownWorkflowFiles("") - if err != nil { - return fmt.Errorf("no workflow files found to %s: %w", action, err) - } - - if len(mdFiles) == 0 { - return fmt.Errorf("no markdown workflow files found to %s", action) - } - - // Extract all workflow names - var allWorkflowNames []string - for _, file := range mdFiles { - base := filepath.Base(file) - name := normalizeWorkflowID(base) - allWorkflowNames = append(allWorkflowNames, name) - } - - // Recursively call with all workflow names - return toggleWorkflowsByNames(ctx, allWorkflowNames, enable, repoOverride) + return toggleWorkflowsByNamesAll(ctx, action, enable, repoOverride) } // Check if gh CLI is available @@ -79,6 +62,74 @@ func toggleWorkflowsByNames(ctx context.Context, workflowNames []string, enable return fmt.Errorf("no workflow files found to %s: %w", action, err) } + githubWorkflows := toggleWorkflowsByNamesFetchGitHub(repoOverride) + targets, notFoundNames := toggleWorkflowsByNamesResolveTargets(ctx, workflowNames, mdFiles, githubWorkflows, enable) + + // Report any workflows that weren't found + if len(notFoundNames) > 0 { + return toggleWorkflowsByNamesNotFoundError(notFoundNames) + } + + // If no targets after filtering, everything was already in the desired state + if len(targets) == 0 { + enableLog.Printf("No workflows need to be %sd - all already in desired state", action) + fmt.Fprintf(os.Stderr, "All specified workflows are already %sd\n", action) + return nil + } + + enableLog.Printf("Proceeding to %s %d workflows", action, len(targets)) + // Show what will be changed + toggleWorkflowsByNamesPrintTargets(action, targets) + + // Perform the action + var failures []string + for _, t := range targets { + if ok := toggleWorkflowsByNamesRunTarget(t, action, enable, repoOverride); !ok { + failures = append(failures, t.Name) + } + } + + // Return error if any workflows failed to be processed + if len(failures) > 0 { + if enable { + return fmt.Errorf("failed to enable %d workflow(s): %s", len(failures), strings.Join(failures, ", ")) + } else { + return fmt.Errorf("failed to disable %d workflow(s): %s", len(failures), strings.Join(failures, ", ")) + } + } + + return nil +} + +func toggleWorkflowsByNamesAction(enable bool) string { + if enable { + return "enable" + } + return "disable" +} + +func toggleWorkflowsByNamesAll(ctx context.Context, action string, enable bool, repoOverride string) error { + enableLog.Print("No specific workflows provided, processing all workflows") + fmt.Fprintf(os.Stderr, "No specific workflows provided. %sing all workflows...\n", strings.ToUpper(action[:1])+action[1:]) + // Get all workflow names and process them + mdFiles, err := getMarkdownWorkflowFiles("") + if err != nil { + return fmt.Errorf("no workflow files found to %s: %w", action, err) + } + if len(mdFiles) == 0 { + return fmt.Errorf("no markdown workflow files found to %s", action) + } + + // Extract all workflow names and recursively call with all workflow names + allWorkflowNames := make([]string, 0, len(mdFiles)) + for _, file := range mdFiles { + base := filepath.Base(file) + allWorkflowNames = append(allWorkflowNames, normalizeWorkflowID(base)) + } + return toggleWorkflowsByNames(ctx, allWorkflowNames, enable, repoOverride) +} + +func toggleWorkflowsByNamesFetchGitHub(repoOverride string) map[string]*GitHubWorkflow { // Get GitHub workflows status for comparison; warn but continue if unavailable enableLog.Print("Fetching GitHub workflows status for comparison") githubWorkflows, err := fetchGitHubWorkflows(repoOverride, false) @@ -88,191 +139,154 @@ func toggleWorkflowsByNames(ctx context.Context, workflowNames []string, enable githubWorkflows = make(map[string]*GitHubWorkflow) } enableLog.Printf("Retrieved %d GitHub workflows from remote", len(githubWorkflows)) + return githubWorkflows +} - // Internal target model to support enabling by ID or lock filename - type workflowTarget struct { - Name string - ID int64 // 0 if unknown - LockFileBase string // e.g., dev.lock.yml - CurrentState string // known state or "unknown" - HasGitHubEntry bool - } - - var targets []workflowTarget +func toggleWorkflowsByNamesResolveTargets(ctx context.Context, workflowNames []string, mdFiles []string, githubWorkflows map[string]*GitHubWorkflow, enable bool) ([]toggleWorkflowsByNamesTarget, []string) { + var targets []toggleWorkflowsByNamesTarget var notFoundNames []string - - // Find matching workflows by name for _, workflowName := range workflowNames { - found := false - for _, file := range mdFiles { - base := filepath.Base(file) - name := normalizeWorkflowID(base) - - // Check if this workflow matches the requested name - if name == workflowName { - found = true - - // Determine lock file and GitHub status (if available) - lockFile := stringutil.MarkdownToLockFile(file) - lockFileBase := filepath.Base(lockFile) - - githubWorkflow, exists := githubWorkflows[name] - - // If enabling and lock file doesn't exist locally, try to compile it - if enable { - if _, err := os.Stat(lockFile); os.IsNotExist(err) { - if err := compileWorkflow(ctx, file, false, false, ""); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to compile workflow %s to create lock file: %v", name, err))) - // If we can't compile and there's no GitHub entry, skip because we can't address it - if !exists { - continue - } - } - } - } - - // Skip if no work is needed based on known GitHub state - if exists { - if enable && githubWorkflow.State == "active" { - // Already enabled - fmt.Fprintf(os.Stderr, "Workflow %s is already enabled\n", name) - continue - } - if !enable && githubWorkflow.State == "disabled_manually" { - // Already disabled - fmt.Fprintf(os.Stderr, "Workflow %s is already disabled\n", name) - continue - } - } - - t := workflowTarget{ - Name: name, - ID: 0, - LockFileBase: lockFileBase, - CurrentState: "unknown", - HasGitHubEntry: exists, - } - if exists { - t.ID = githubWorkflow.ID - t.CurrentState = githubWorkflow.State - } - targets = append(targets, t) - break - } - } + t, found, include := toggleWorkflowsByNamesFindTarget(ctx, workflowName, mdFiles, githubWorkflows, enable) if !found { notFoundNames = append(notFoundNames, workflowName) + continue + } + if include { + targets = append(targets, t) } } + return targets, notFoundNames +} - // Report any workflows that weren't found - if len(notFoundNames) > 0 { - enableLog.Printf("Workflows not found: %v", notFoundNames) - suggestions := []string{ - fmt.Sprintf("Run '%s status' to see all available workflows", string(constants.CLIExtensionPrefix)), - "Check for typos in the workflow names", - "Ensure the workflows have been compiled and pushed to GitHub", +func toggleWorkflowsByNamesFindTarget(ctx context.Context, workflowName string, mdFiles []string, githubWorkflows map[string]*GitHubWorkflow, enable bool) (toggleWorkflowsByNamesTarget, bool, bool) { + for _, file := range mdFiles { + base := filepath.Base(file) + name := normalizeWorkflowID(base) + if name != workflowName { + continue } + lockFile := stringutil.MarkdownToLockFile(file) + lockFileBase := filepath.Base(lockFile) + githubWorkflow, exists := githubWorkflows[name] - // Add fuzzy match suggestions for each not found workflow - if len(notFoundNames) == 1 { - similarNames := suggestWorkflowNames(notFoundNames[0]) - if len(similarNames) > 0 { - suggestions = append([]string{fmt.Sprintf("Did you mean: %s?", strings.Join(similarNames, ", "))}, suggestions...) - } + if enable && toggleWorkflowsByNamesNeedsCompile(ctx, file, lockFile, name, exists) { + return toggleWorkflowsByNamesTarget{}, true, false + } + if exists && toggleWorkflowsByNamesAlreadyDesired(name, enable, githubWorkflow.State) { + return toggleWorkflowsByNamesTarget{}, true, false + } + t := toggleWorkflowsByNamesTarget{Name: name, LockFileBase: lockFileBase, CurrentState: "unknown", HasGitHubEntry: exists} + if exists { + t.ID = githubWorkflow.ID + t.CurrentState = githubWorkflow.State } + return t, true, true + } + return toggleWorkflowsByNamesTarget{}, false, false +} - return errors.New(console.FormatErrorWithSuggestions( - "workflows not found: "+strings.Join(notFoundNames, ", "), - suggestions, - )) +func toggleWorkflowsByNamesNeedsCompile(ctx context.Context, file, lockFile, name string, exists bool) bool { + if _, err := os.Stat(lockFile); !os.IsNotExist(err) { + return false + } + if err := compileWorkflow(ctx, file, false, false, ""); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to compile workflow %s to create lock file: %v", name, err))) + // If we can't compile and there's no GitHub entry, skip because we can't address it + return !exists } + return false +} - // If no targets after filtering, everything was already in the desired state - if len(targets) == 0 { - enableLog.Printf("No workflows need to be %sd - all already in desired state", action) - fmt.Fprintf(os.Stderr, "All specified workflows are already %sd\n", action) - return nil +func toggleWorkflowsByNamesAlreadyDesired(name string, enable bool, state string) bool { + if enable && state == "active" { + // Already enabled + fmt.Fprintf(os.Stderr, "Workflow %s is already enabled\n", name) + return true + } + if !enable && state == "disabled_manually" { + // Already disabled + fmt.Fprintf(os.Stderr, "Workflow %s is already disabled\n", name) + return true } + return false +} - enableLog.Printf("Proceeding to %s %d workflows", action, len(targets)) - // Show what will be changed +func toggleWorkflowsByNamesNotFoundError(notFoundNames []string) error { + enableLog.Printf("Workflows not found: %v", notFoundNames) + suggestions := []string{ + fmt.Sprintf("Run '%s status' to see all available workflows", string(constants.CLIExtensionPrefix)), + "Check for typos in the workflow names", + "Ensure the workflows have been compiled and pushed to GitHub", + } + // Add fuzzy match suggestions for each not found workflow + if len(notFoundNames) == 1 { + similarNames := suggestWorkflowNames(notFoundNames[0]) + if len(similarNames) > 0 { + suggestions = append([]string{fmt.Sprintf("Did you mean: %s?", strings.Join(similarNames, ", "))}, suggestions...) + } + } + return errors.New(console.FormatErrorWithSuggestions( + "workflows not found: "+strings.Join(notFoundNames, ", "), + suggestions, + )) +} + +func toggleWorkflowsByNamesPrintTargets(action string, targets []toggleWorkflowsByNamesTarget) { fmt.Fprintf(os.Stderr, "The following workflows will be %sd:\n", action) for _, t := range targets { fmt.Fprintf(os.Stderr, " %s (current state: %s)\n", t.Name, t.CurrentState) } +} - // Perform the action - var failures []string +func toggleWorkflowsByNamesRunTarget(t toggleWorkflowsByNamesTarget, action string, enable bool, repoOverride string) bool { + cmd := toggleWorkflowsByNamesCommandForTarget(t, enable, repoOverride) + if output, err := cmd.CombinedOutput(); err != nil { + toggleWorkflowsByNamesPrintFailure(action, t.Name, output, err) + return false + } + fmt.Fprintf(os.Stderr, "%sd workflow: %s\n", strings.ToUpper(action[:1])+action[1:], t.Name) + return true +} - for _, t := range targets { - var cmd *exec.Cmd - if enable { - // Prefer enabling by ID, otherwise fall back to lock file name - if t.ID != 0 { - args := []string{"workflow", "enable", strconv.FormatInt(t.ID, 10)} - if repoOverride != "" { - args = append(args, "--repo", repoOverride) - } - cmd = workflow.ExecGH(args...) - } else { - args := []string{"workflow", "enable", t.LockFileBase} - if repoOverride != "" { - args = append(args, "--repo", repoOverride) - } - cmd = workflow.ExecGH(args...) - } - } else { - // First cancel any running workflows (by ID when available, else by lock file name) - if t.ID != 0 { - if err := cancelWorkflowRuns(t.ID); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to cancel runs for workflow %s: %v", t.Name, err))) - } - // Prefer disabling by lock file name for reliability - args := []string{"workflow", "disable", t.LockFileBase} - if repoOverride != "" { - args = append(args, "--repo", repoOverride) - } - cmd = workflow.ExecGH(args...) - } else { - if err := cancelWorkflowRunsByLockFile(t.LockFileBase); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to cancel runs for workflow %s: %v", t.Name, err))) - } - args := []string{"workflow", "disable", t.LockFileBase} - if repoOverride != "" { - args = append(args, "--repo", repoOverride) - } - cmd = workflow.ExecGH(args...) - } +func toggleWorkflowsByNamesCommandForTarget(t toggleWorkflowsByNamesTarget, enable bool, repoOverride string) *exec.Cmd { + if enable { + // Prefer enabling by ID, otherwise fall back to lock file name + if t.ID != 0 { + return toggleWorkflowsByNamesGHCommand(repoOverride, "workflow", "enable", strconv.FormatInt(t.ID, 10)) } + return toggleWorkflowsByNamesGHCommand(repoOverride, "workflow", "enable", t.LockFileBase) + } - if output, err := cmd.CombinedOutput(); err != nil { - if len(output) > 0 { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Failed to %s workflow %s: %v\n%s", action, t.Name, err, string(output)))) - // Provide clearer hint on common permission issues - outStr := strings.ToLower(string(output)) - if strings.Contains(outStr, "http 403") || strings.Contains(outStr, "resource not accessible by integration") { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Hint: Disabling/enabling workflows requires repository admin or maintainer permissions. Ensure your gh auth has write/admin access to this repo.")) - } - } else { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Failed to %s workflow %s: %v", action, t.Name, err))) - } - failures = append(failures, t.Name) - } else { - fmt.Fprintf(os.Stderr, "%sd workflow: %s\n", strings.ToUpper(action[:1])+action[1:], t.Name) + // First cancel any running workflows (by ID when available, else by lock file name) + if t.ID != 0 { + if err := cancelWorkflowRuns(t.ID); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to cancel runs for workflow %s: %v", t.Name, err))) } + } else if err := cancelWorkflowRunsByLockFile(t.LockFileBase); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to cancel runs for workflow %s: %v", t.Name, err))) } + // Prefer disabling by lock file name for reliability + return toggleWorkflowsByNamesGHCommand(repoOverride, "workflow", "disable", t.LockFileBase) +} - // Return error if any workflows failed to be processed - if len(failures) > 0 { - if enable { - return fmt.Errorf("failed to enable %d workflow(s): %s", len(failures), strings.Join(failures, ", ")) - } else { - return fmt.Errorf("failed to disable %d workflow(s): %s", len(failures), strings.Join(failures, ", ")) - } +func toggleWorkflowsByNamesGHCommand(repoOverride string, args ...string) *exec.Cmd { + if repoOverride != "" { + args = append(args, "--repo", repoOverride) } + return workflow.ExecGH(args...) +} - return nil +func toggleWorkflowsByNamesPrintFailure(action, workflowName string, output []byte, err error) { + if len(output) == 0 { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Failed to %s workflow %s: %v", action, workflowName, err))) + return + } + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Failed to %s workflow %s: %v\n%s", action, workflowName, err, string(output)))) + // Provide clearer hint on common permission issues + outStr := strings.ToLower(string(output)) + if strings.Contains(outStr, "http 403") || strings.Contains(outStr, "resource not accessible by integration") { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Hint: Disabling/enabling workflows requires repository admin or maintainer permissions. Ensure your gh auth has write/admin access to this repo.")) + } } // DisableAllWorkflowsExcept disables all workflows except the specified ones @@ -281,12 +295,47 @@ func DisableAllWorkflowsExcept(repoSlug string, exceptWorkflows []string, verbos enableLog.Printf("Disabling all workflows except: count=%d, repo=%s", len(exceptWorkflows), repoSlug) workflowsDir := constants.GetWorkflowDir() + allYAMLFiles, ok := disableAllWorkflowsExceptYAMLFiles(workflowsDir, verbose) + if !ok { + return nil + } + + // Create a set of workflows to keep enabled + keepEnabled := disableAllWorkflowsExceptKeepSet(exceptWorkflows) + + // Filter to find workflows to disable + workflowsToDisable := disableAllWorkflowsExceptFilter(allYAMLFiles, keepEnabled, verbose) + if len(workflowsToDisable) == 0 { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No workflows to disable")) + } + return nil + } + + // Show what will be disabled + fmt.Fprintf(os.Stderr, "Disabling %d workflow(s) in cloned repository:\n", len(workflowsToDisable)) + for _, wf := range workflowsToDisable { + fmt.Fprintf(os.Stderr, " %s\n", wf) + } + + // Disable each workflow + failures := disableAllWorkflowsExceptDisable(workflowsToDisable, repoSlug, verbose) + + if len(failures) > 0 { + return fmt.Errorf("failed to disable %d workflow(s): %s", len(failures), strings.Join(failures, ", ")) + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Disabled %d workflow(s)", len(workflowsToDisable)))) + return nil +} + +func disableAllWorkflowsExceptYAMLFiles(workflowsDir string, verbose bool) ([]string, bool) { // Check if workflows directory exists if _, err := os.Stat(workflowsDir); os.IsNotExist(err) { if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No .github/workflows directory found, nothing to disable")) } - return nil + return nil, false } // Get all .yml and .yaml files @@ -299,29 +348,26 @@ func DisableAllWorkflowsExcept(repoSlug string, exceptWorkflows []string, verbos if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No YAML workflow files found")) } - return nil + return nil, false } + return allYAMLFiles, true +} - // Create a set of workflows to keep enabled - keepEnabled := make(map[string]struct { - }) +func disableAllWorkflowsExceptKeepSet(exceptWorkflows []string) map[string]struct{} { + keepEnabled := make(map[string]struct{}) for _, workflowName := range exceptWorkflows { // Add both .md and .lock.yml variants - keepEnabled[workflowName+".md"] = struct { - }{} - keepEnabled[workflowName+".lock.yml"] = struct { - }{} - keepEnabled[workflowName] = struct { // In case the full filename is provided - }{} + keepEnabled[workflowName+".md"] = struct{}{} + keepEnabled[workflowName+".lock.yml"] = struct{}{} + keepEnabled[workflowName] = struct{}{} // In case the full filename is provided } + return keepEnabled +} - // Filter to find workflows to disable +func disableAllWorkflowsExceptFilter(allYAMLFiles []string, keepEnabled map[string]struct{}, verbose bool) []string { var workflowsToDisable []string - for _, yamlFile := range allYAMLFiles { base := filepath.Base(yamlFile) - - // Skip if it's in the keep-enabled set if setutil.Contains(keepEnabled, base) { if verbose { fmt.Fprintf(os.Stderr, "Keeping enabled: %s\n", base) @@ -329,7 +375,6 @@ func DisableAllWorkflowsExcept(repoSlug string, exceptWorkflows []string, verbos continue } - // Check if the base name without extension matches nameWithoutExt := strings.TrimSuffix(base, filepath.Ext(base)) if setutil.Contains(keepEnabled, nameWithoutExt) { if verbose { @@ -337,24 +382,12 @@ func DisableAllWorkflowsExcept(repoSlug string, exceptWorkflows []string, verbos } continue } - workflowsToDisable = append(workflowsToDisable, base) } + return workflowsToDisable +} - if len(workflowsToDisable) == 0 { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No workflows to disable")) - } - return nil - } - - // Show what will be disabled - fmt.Fprintf(os.Stderr, "Disabling %d workflow(s) in cloned repository:\n", len(workflowsToDisable)) - for _, wf := range workflowsToDisable { - fmt.Fprintf(os.Stderr, " %s\n", wf) - } - - // Disable each workflow +func disableAllWorkflowsExceptDisable(workflowsToDisable []string, repoSlug string, verbose bool) []string { var failures []string for _, wf := range workflowsToDisable { args := []string{"workflow", "disable", wf} @@ -368,17 +401,9 @@ func DisableAllWorkflowsExcept(repoSlug string, exceptWorkflows []string, verbos fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to disable workflow %s: %v\n%s", wf, err, string(output)))) } failures = append(failures, wf) - } else { - if verbose { - fmt.Fprintf(os.Stderr, "Disabled workflow: %s\n", wf) - } + } else if verbose { + fmt.Fprintf(os.Stderr, "Disabled workflow: %s\n", wf) } } - - if len(failures) > 0 { - return fmt.Errorf("failed to disable %d workflow(s): %s", len(failures), strings.Join(failures, ", ")) - } - - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Disabled %d workflow(s)", len(workflowsToDisable)))) - return nil + return failures } diff --git a/pkg/cli/engine_secrets.go b/pkg/cli/engine_secrets.go index 95d540673db..3887820e9f9 100644 --- a/pkg/cli/engine_secrets.go +++ b/pkg/cli/engine_secrets.go @@ -236,6 +236,21 @@ func checkAndEnsureEngineSecretsForEngine(config EngineSecretConfig) error { func ensureSecretAvailable(req SecretRequirement, config EngineSecretConfig) error { engineSecretsLog.Printf("Ensuring secret available: %s", req.Name) + if handled, err := ensureSecretAvailableRepository(req, config); handled || err != nil { + return err + } + + if envValue := ensureSecretAvailableEnvValue(req); envValue != "" { + if handled, err := ensureSecretAvailableUseEnv(req, config, envValue); handled || err != nil { + return err + } + } + + // Secret not found, prompt user for it + return engineSecretsPromptFn(req, config) +} + +func ensureSecretAvailableRepository(req SecretRequirement, config EngineSecretConfig) (bool, error) { // Check if secret already exists in the repository if setutil.Contains(config.ExistingSecrets, req.Name) { if mustValidateExistingSecretValue(req) { @@ -243,67 +258,69 @@ func ensureSecretAvailable(req SecretRequirement, config EngineSecretConfig) err fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Paste the current or replacement fine-grained PAT so gh aw can validate it and update the repository secret.")) revalidateConfig := config revalidateConfig.OverwriteExistingSecret = true - return engineSecretsPromptFn(req, revalidateConfig) + return true, engineSecretsPromptFn(req, revalidateConfig) } fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Using existing %s secret in repository", req.Name))) - return nil + return true, nil } // Check alternative secret names in repository for _, alt := range req.AlternativeEnvVars { if setutil.Contains(config.ExistingSecrets, alt) { - if mustValidateExistingSecretValue(req) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(alt+" already exists in the repository, but GitHub does not expose stored secret values for validation.")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Paste the current or replacement fine-grained PAT so gh aw can validate it and store it as %s.", req.Name))) - revalidateConfig := config - revalidateConfig.OverwriteExistingSecret = true - return engineSecretsPromptFn(req, revalidateConfig) - } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Using existing %s secret in repository (alternative for %s)", alt, req.Name))) - return nil + return true, ensureSecretAvailableRepositoryAlternative(req, config, alt) } } + return false, nil +} + +func ensureSecretAvailableRepositoryAlternative(req SecretRequirement, config EngineSecretConfig, alt string) error { + if mustValidateExistingSecretValue(req) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(alt+" already exists in the repository, but GitHub does not expose stored secret values for validation.")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Paste the current or replacement fine-grained PAT so gh aw can validate it and store it as %s.", req.Name))) + revalidateConfig := config + revalidateConfig.OverwriteExistingSecret = true + return engineSecretsPromptFn(req, revalidateConfig) + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Using existing %s secret in repository (alternative for %s)", alt, req.Name))) + return nil +} +func ensureSecretAvailableEnvValue(req SecretRequirement) string { // Check environment variable envValue := os.Getenv(req.Name) //nolint:osgetenvlibrary - if envValue == "" { - // Check alternative environment variables - for _, alt := range req.AlternativeEnvVars { - envValue = os.Getenv(alt) //nolint:osgetenvlibrary - if envValue != "" { - engineSecretsLog.Printf("Found secret in alternative env var: %s", alt) - break - } + if envValue != "" { + return envValue + } + // Check alternative environment variables + for _, alt := range req.AlternativeEnvVars { + envValue = os.Getenv(alt) //nolint:osgetenvlibrary + if envValue != "" { + engineSecretsLog.Printf("Found secret in alternative env var: %s", alt) + return envValue } } + return "" +} - if envValue != "" { - // Validate if it's a Copilot token - if req.IsEngineSecret && req.EngineName == string(constants.CopilotEngine) { - if err := stringutil.ValidateCopilotPAT(envValue); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("%s in environment is not a valid fine-grained PAT: %s", req.Name, stringutil.GetPATTypeDescription(envValue)))) - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(err.Error())) - // Continue to prompt for a new token - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Found valid %s in environment", req.Name))) - // Upload to repository if we have a repo slug - if config.RepoSlug != "" { - return engineSecretsUploadFn(req.Name, envValue, config.RepoSlug, config.Verbose, config.OverwriteExistingSecret) - } - return nil - } - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Found %s in environment", req.Name))) - // Upload to repository if we have a repo slug - if config.RepoSlug != "" { - return engineSecretsUploadFn(req.Name, envValue, config.RepoSlug, config.Verbose, config.OverwriteExistingSecret) - } - return nil +func ensureSecretAvailableUseEnv(req SecretRequirement, config EngineSecretConfig, envValue string) (bool, error) { + // Validate if it's a Copilot token + if req.IsEngineSecret && req.EngineName == string(constants.CopilotEngine) { + if err := stringutil.ValidateCopilotPAT(envValue); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("%s in environment is not a valid fine-grained PAT: %s", req.Name, stringutil.GetPATTypeDescription(envValue)))) + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(err.Error())) + // Continue to prompt for a new token + return false, nil } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Found valid %s in environment", req.Name))) + } else { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Found %s in environment", req.Name))) } - // Secret not found, prompt user for it - return engineSecretsPromptFn(req, config) + // Upload to repository if we have a repo slug + if config.RepoSlug != "" { + return true, engineSecretsUploadFn(req.Name, envValue, config.RepoSlug, config.Verbose, config.OverwriteExistingSecret) + } + return true, nil } // promptForSecret prompts the user to provide a secret value @@ -331,13 +348,35 @@ func mustValidateExistingSecretValue(req SecretRequirement) bool { func promptForCopilotPATUnified(req SecretRequirement, config EngineSecretConfig) error { preconfiguredPATURL := buildCopilotPATCreationURL() + promptForCopilotPATUnifiedInstructions(preconfiguredPATURL) + if err := promptForCopilotPATUnifiedOfferBrowser(preconfiguredPATURL, config); err != nil { + return err + } + token, err := promptForCopilotPATUnifiedReadToken(config) + if err != nil { + return err + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Valid fine-grained Copilot token received")) + + // Upload to repository if we have a repo slug + if config.RepoSlug != "" { + return uploadSecretToRepo(req.Name, token, config.RepoSlug, config.Verbose, config.OverwriteExistingSecret) + } + + return nil +} + +func promptForCopilotPATUnifiedInstructions(preconfiguredPATURL string) { fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "Create a fine-grained Personal Access Token (PAT) from the preconfigured page below, then paste it back here.") fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "Preconfigured token creation page:") fmt.Fprintln(os.Stderr, console.FormatCommandMessage(" "+preconfiguredPATURL)) fmt.Fprintln(os.Stderr, "") +} +func promptForCopilotPATUnifiedOfferBrowser(preconfiguredPATURL string, config EngineSecretConfig) error { openBrowser := true confirmForm := console.NewConfirmForm( huh.NewConfirm(). @@ -351,14 +390,20 @@ func promptForCopilotPATUnified(req SecretRequirement, config EngineSecretConfig return promptCancelled() } // Non-interactive: skip the consent gate and fall through to token input - } else if openBrowser { - if openBootstrapBrowser(preconfiguredPATURL) { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Opened the preconfigured Copilot PAT page in your browser.")) - } else { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Couldn't open your browser automatically (no supported opener found) — open the URL above manually.")) - } + return nil } + if !openBrowser { + return nil + } + if openBootstrapBrowser(preconfiguredPATURL) { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Opened the preconfigured Copilot PAT page in your browser.")) + } else { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Couldn't open your browser automatically (no supported opener found) — open the URL above manually.")) + } + return nil +} +func promptForCopilotPATUnifiedReadToken(config EngineSecretConfig) (string, error) { var token string form := console.NewInputForm( huh.NewInput(). @@ -380,21 +425,11 @@ func promptForCopilotPATUnified(req SecretRequirement, config EngineSecretConfig if err := form.RunWithContext(config.ctx()); err != nil { if console.IsCancelled(err) { - return promptCancelled() + return "", promptCancelled() } - return fmt.Errorf("failed to get Copilot token: %w", err) + return "", fmt.Errorf("failed to get Copilot token: %w", err) } - - token = strings.TrimSpace(token) - - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Valid fine-grained Copilot token received")) - - // Upload to repository if we have a repo slug - if config.RepoSlug != "" { - return uploadSecretToRepo(req.Name, token, config.RepoSlug, config.Verbose, config.OverwriteExistingSecret) - } - - return nil + return strings.TrimSpace(token), nil } func buildCopilotPATCreationURL() string { @@ -739,12 +774,7 @@ func displayMissingSecrets(requirements []SecretRequirement, repoSlug string, ex func displaySecretsSummaryTable(requirements []SecretRequirement, existingSecrets map[string]struct { }) { // Filter to only required secrets (not optional) - var requiredOnly []SecretRequirement - for _, req := range requirements { - if !req.Optional { - requiredOnly = append(requiredOnly, req) - } - } + requiredOnly := displaySecretsSummaryTableRequired(requirements) // If no required secrets, don't show the table if len(requiredOnly) == 0 { @@ -756,51 +786,62 @@ func displaySecretsSummaryTable(requirements []SecretRequirement, existingSecret fmt.Fprintln(os.Stderr, "") // Calculate max width for alignment - maxNameWidth := 0 - for _, req := range requiredOnly { - if len(req.Name) > maxNameWidth { - maxNameWidth = len(req.Name) - } - } + maxNameWidth := displaySecretsSummaryTableMaxNameWidth(requiredOnly) // Display each required secret with status for _, req := range requiredOnly { - // Check if secret exists - exists := setutil.Contains(existingSecrets, req.Name) - var altUsed string - if !exists { - // Check alternatives - for _, alt := range req.AlternativeEnvVars { - if setutil.Contains(existingSecrets, alt) { - exists = true - altUsed = alt - break - } - } - } - - // Format status indicator - var statusLine string - if exists { - if altUsed != "" { - statusLine = console.FormatSuccessMessage(fmt.Sprintf("(via %s)", altUsed)) - } else { - statusLine = console.FormatSuccessMessage("") - } - } else { - statusLine = console.FormatErrorMessage("") - } - // Format secret name with padding nameWithPadding := fmt.Sprintf("%-*s", maxNameWidth, req.Name) // Display the line - fmt.Fprintf(os.Stderr, " %s %s - %s\n", statusLine, nameWithPadding, req.WhenNeeded) + fmt.Fprintf(os.Stderr, " %s %s - %s\n", displaySecretsSummaryTableStatusLine(req, existingSecrets), nameWithPadding, req.WhenNeeded) } fmt.Fprintln(os.Stderr, "") } +func displaySecretsSummaryTableRequired(requirements []SecretRequirement) []SecretRequirement { + var requiredOnly []SecretRequirement + for _, req := range requirements { + if !req.Optional { + requiredOnly = append(requiredOnly, req) + } + } + return requiredOnly +} + +func displaySecretsSummaryTableMaxNameWidth(requirements []SecretRequirement) int { + maxNameWidth := 0 + for _, req := range requirements { + if len(req.Name) > maxNameWidth { + maxNameWidth = len(req.Name) + } + } + return maxNameWidth +} + +func displaySecretsSummaryTableStatusLine(req SecretRequirement, existingSecrets map[string]struct{}) string { + exists := setutil.Contains(existingSecrets, req.Name) + var altUsed string + if !exists { + for _, alt := range req.AlternativeEnvVars { + if setutil.Contains(existingSecrets, alt) { + exists = true + altUsed = alt + break + } + } + } + + if !exists { + return console.FormatErrorMessage("") + } + if altUsed != "" { + return console.FormatSuccessMessage(fmt.Sprintf("(via %s)", altUsed)) + } + return console.FormatSuccessMessage("") +} + // splitRepoSlug splits "owner/repo" into [owner, repo] // Uses repoutil.SplitRepoSlug internally but provides backward-compatible array return func splitRepoSlug(slug string) [2]string { diff --git a/pkg/cli/experiments_analyze_statistics.go b/pkg/cli/experiments_analyze_statistics.go index 6b9ca49d149..b134cc4071b 100644 --- a/pkg/cli/experiments_analyze_statistics.go +++ b/pkg/cli/experiments_analyze_statistics.go @@ -115,33 +115,7 @@ func computeExperimentAnalysis(exp ExperimentVariantStats, cfg *workflow.Experim MinSamples: defaultMinSamples, } - // Extract metadata from config when available. - if cfg != nil { - a.Hypothesis = cfg.Hypothesis - a.AnalysisType = cfg.AnalysisType - if cfg.MinSamples > 0 { - a.MinSamples = cfg.MinSamples - } - for _, g := range cfg.GuardrailMetrics { - a.Guardrails = append(a.Guardrails, GuardrailStatus{ - Name: g.Name, - Threshold: g.Threshold, - }) - } - // Populate metric and resolve any eval reference to its question text. - if cfg.Metric != "" { - a.Metric = cfg.Metric - evalID, isEval := workflow.ParseExperimentMetricEvalReference(cfg.Metric) - if isEval && evalID != "" && evals != nil { - for _, q := range evals.Questions { - if q.ID == evalID { - a.MetricQuestion = q.Question - break - } - } - } - } - } + computeExperimentAnalysisConfig(&a, cfg, evals) // Degenerate: fewer than 2 variants cannot be meaningfully analysed. if len(exp.Variants) < 2 { @@ -159,48 +133,94 @@ func computeExperimentAnalysis(exp ExperimentVariantStats, cfg *workflow.Experim expectedPcts := expectedProportions(variantNames, cfg) // Populate per-variant entries. + computeExperimentAnalysisVariants(&a, exp, variantNames, expectedPcts) + + k := len(variantNames) + + // Chi-square goodness-of-fit balance test. + computeExperimentAnalysisBalance(&a, exp, variantNames, expectedPcts) + + // Bonferroni correction for K ≥ 3 variants (§11.3). + if k >= 3 { + a.BonferroniAlpha = 0.05 / float64(k-1) + } + + // Recommendation (R-STAT-007). + computeExperimentAnalysisRecommendation(&a, exp.Name, k) + return a +} + +func computeExperimentAnalysisConfig(a *ExperimentAnalysis, cfg *workflow.ExperimentConfig, evals *workflow.EvalsConfig) { + // Extract metadata from config when available. + if cfg == nil { + return + } + a.Hypothesis = cfg.Hypothesis + a.AnalysisType = cfg.AnalysisType + if cfg.MinSamples > 0 { + a.MinSamples = cfg.MinSamples + } + for _, g := range cfg.GuardrailMetrics { + a.Guardrails = append(a.Guardrails, GuardrailStatus{Name: g.Name, Threshold: g.Threshold}) + } + // Populate metric and resolve any eval reference to its question text. + if cfg.Metric != "" { + a.Metric = cfg.Metric + computeExperimentAnalysisMetricQuestion(a, cfg.Metric, evals) + } +} + +func computeExperimentAnalysisMetricQuestion(a *ExperimentAnalysis, metric string, evals *workflow.EvalsConfig) { + evalID, isEval := workflow.ParseExperimentMetricEvalReference(metric) + if !isEval || evalID == "" || evals == nil { + return + } + for _, q := range evals.Questions { + if q.ID == evalID { + a.MetricQuestion = q.Question + return + } + } +} + +func computeExperimentAnalysisVariants(a *ExperimentAnalysis, exp ExperimentVariantStats, variantNames []string, expectedPcts []float64) { for i, name := range variantNames { count := exp.Variants[name] - obsPct := safePercent(count, exp.Total) a.Variants = append(a.Variants, VariantAnalysis{ Name: name, Count: count, - ObservedPct: obsPct, + ObservedPct: safePercent(count, exp.Total), ExpectedPct: expectedPcts[i] * 100, MinSamples: a.MinSamples, BelowMinSamples: count < a.MinSamples, }) } +} +func computeExperimentAnalysisBalance(a *ExperimentAnalysis, exp ExperimentVariantStats, variantNames []string, expectedPcts []float64) { k := len(variantNames) - - // Chi-square goodness-of-fit balance test. - if exp.Total > 0 && k >= 2 { - chi2 := 0.0 - for i, name := range variantNames { - expected := float64(exp.Total) * expectedPcts[i] - if expected > 0 { - diff := float64(exp.Variants[name]) - expected - chi2 += (diff * diff) / expected - } - } - df := k - 1 - pval := chiSquarePValue(chi2, df) - a.ChiSquare = chi2 - a.DegreesOfFreedom = df - a.PValue = pval - a.IsBalanced = pval >= balanceSignificanceThreshold - experimentsStatsLog.Printf("Chi-square balance test for %q: χ²=%.3f df=%d p=%.3f balanced=%v", exp.Name, chi2, df, pval, a.IsBalanced) - } else { + if exp.Total <= 0 || k < 2 { a.IsBalanced = true // insufficient data to assess balance + return } - - // Bonferroni correction for K ≥ 3 variants (§11.3). - if k >= 3 { - a.BonferroniAlpha = 0.05 / float64(k-1) + chi2 := 0.0 + for i, name := range variantNames { + expected := float64(exp.Total) * expectedPcts[i] + if expected > 0 { + diff := float64(exp.Variants[name]) - expected + chi2 += (diff * diff) / expected + } } + df := k - 1 + pval := chiSquarePValue(chi2, df) + a.ChiSquare = chi2 + a.DegreesOfFreedom = df + a.PValue = pval + a.IsBalanced = pval >= balanceSignificanceThreshold + experimentsStatsLog.Printf("Chi-square balance test for %q: χ²=%.3f df=%d p=%.3f balanced=%v", exp.Name, chi2, df, pval, a.IsBalanced) +} - // Recommendation (R-STAT-007). +func computeExperimentAnalysisRecommendation(a *ExperimentAnalysis, experimentName string, k int) { belowCount := 0 minObserved := math.MaxInt for _, v := range a.Variants { @@ -215,15 +235,13 @@ func computeExperimentAnalysis(exp ExperimentVariantStats, cfg *workflow.Experim a.Recommendation = "EXTEND" a.Rationale = fmt.Sprintf("%d of %d variant(s) below min_samples threshold (min observed: %d / %d)", belowCount, k, minObserved, a.MinSamples) - experimentsStatsLog.Printf("Experiment %q recommendation: EXTEND (%d/%d variants below min_samples=%d)", exp.Name, belowCount, k, a.MinSamples) + experimentsStatsLog.Printf("Experiment %q recommendation: EXTEND (%d/%d variants below min_samples=%d)", experimentName, belowCount, k, a.MinSamples) } else { a.Recommendation = "READY_FOR_ANALYSIS" a.Rationale = fmt.Sprintf("all %d variants have reached min_samples (%d); proceed with outcome metric analysis", k, a.MinSamples) - experimentsStatsLog.Printf("Experiment %q recommendation: READY_FOR_ANALYSIS (all %d variants above min_samples=%d)", exp.Name, k, a.MinSamples) + experimentsStatsLog.Printf("Experiment %q recommendation: READY_FOR_ANALYSIS (all %d variants above min_samples=%d)", experimentName, k, a.MinSamples) } - - return a } // expectedProportions returns the expected fraction (0.0–1.0) for each variant, in the @@ -307,22 +325,46 @@ func printExperimentAnalyses(analyses []ExperimentAnalysis) { func printOneExperimentAnalysis(a ExperimentAnalysis) { fmt.Fprintf(os.Stderr, "\n [%s]\n", a.ExperimentName) + printOneExperimentAnalysisHeader(a) + fmt.Fprintf(os.Stderr, " Min samples: %d per variant\n", a.MinSamples) + + // Per-variant progress table. + printOneExperimentAnalysisVariants(a) + + // Balance test. + printOneExperimentAnalysisBalance(a) + + // Bonferroni correction. + if a.BonferroniAlpha > 0 { + fmt.Fprintf(os.Stderr, " Bonferroni : α_adjusted = %.4f (for %d variants, K-1 comparisons)\n", + a.BonferroniAlpha, len(a.Variants)) + } + + // Guardrails. + printOneExperimentAnalysisGuardrails(a) + + // Recommendation. + printOneExperimentAnalysisRecommendation(a) +} + +func printOneExperimentAnalysisHeader(a ExperimentAnalysis) { if a.Hypothesis != "" { fmt.Fprintf(os.Stderr, " Hypothesis : %s\n", a.Hypothesis) } if a.AnalysisType != "" { fmt.Fprintf(os.Stderr, " Test type : %s\n", a.AnalysisType) } - if a.Metric != "" { - if a.MetricQuestion != "" { - fmt.Fprintf(os.Stderr, " Metric : %s — %s\n", a.Metric, a.MetricQuestion) - } else { - fmt.Fprintf(os.Stderr, " Metric : %s\n", a.Metric) - } + if a.Metric == "" { + return } - fmt.Fprintf(os.Stderr, " Min samples: %d per variant\n", a.MinSamples) + if a.MetricQuestion != "" { + fmt.Fprintf(os.Stderr, " Metric : %s — %s\n", a.Metric, a.MetricQuestion) + } else { + fmt.Fprintf(os.Stderr, " Metric : %s\n", a.Metric) + } +} - // Per-variant progress table. +func printOneExperimentAnalysisVariants(a ExperimentAnalysis) { fmt.Fprintf(os.Stderr, "\n %-20s %6s %7s %7s %s\n", "Variant", "Count", "Obs%", "Exp%", "Progress") fmt.Fprintf(os.Stderr, " %s\n", strings.Repeat("─", 62)) for _, v := range a.Variants { @@ -333,27 +375,23 @@ func printOneExperimentAnalysis(a ExperimentAnalysis) { fmt.Fprintf(os.Stderr, " %-20s %6d %6.1f%% %6.1f%% %s\n", v.Name, v.Count, v.ObservedPct, v.ExpectedPct, progressStr) } +} - // Balance test. +func printOneExperimentAnalysisBalance(a ExperimentAnalysis) { fmt.Fprintln(os.Stderr) - if a.TotalRuns > 0 { - balancedStr := "balanced ✓" - if !a.IsBalanced { - balancedStr = "unbalanced ✗" - } - fmt.Fprintf(os.Stderr, " Balance : χ² = %.3f df = %d p = %.3f (%s)\n", - a.ChiSquare, a.DegreesOfFreedom, a.PValue, balancedStr) - } else { + if a.TotalRuns == 0 { fmt.Fprintln(os.Stderr, " Balance : no data") + return } - - // Bonferroni correction. - if a.BonferroniAlpha > 0 { - fmt.Fprintf(os.Stderr, " Bonferroni : α_adjusted = %.4f (for %d variants, K-1 comparisons)\n", - a.BonferroniAlpha, len(a.Variants)) + balancedStr := "balanced ✓" + if !a.IsBalanced { + balancedStr = "unbalanced ✗" } + fmt.Fprintf(os.Stderr, " Balance : χ² = %.3f df = %d p = %.3f (%s)\n", + a.ChiSquare, a.DegreesOfFreedom, a.PValue, balancedStr) +} - // Guardrails. +func printOneExperimentAnalysisGuardrails(a ExperimentAnalysis) { if len(a.Guardrails) > 0 { parts := make([]string, 0, len(a.Guardrails)) for _, g := range a.Guardrails { @@ -362,8 +400,9 @@ func printOneExperimentAnalysis(a ExperimentAnalysis) { fmt.Fprintf(os.Stderr, " Guardrails : %s\n", strings.Join(parts, " • ")) fmt.Fprintln(os.Stderr, " (pass/fail evaluation requires per-run outcome metric data)") } +} - // Recommendation. +func printOneExperimentAnalysisRecommendation(a ExperimentAnalysis) { fmt.Fprintln(os.Stderr) switch a.Recommendation { case "EXTEND": diff --git a/pkg/cli/experiments_command.go b/pkg/cli/experiments_command.go index c3474f02243..dec9969bc42 100644 --- a/pkg/cli/experiments_command.go +++ b/pkg/cli/experiments_command.go @@ -780,35 +780,15 @@ func printExperimentDetails(d *ExperimentDetails) { fmt.Fprintf(os.Stderr, " Branch: %s\n", d.Branch) fmt.Fprintf(os.Stderr, " Total runs: %d\n", d.TotalRuns) + printExperimentDetailsExperiments(d) + printExperimentAnalyses(d.Analyses) + printExperimentDetailsRecentRuns(d) +} + +func printExperimentDetailsExperiments(d *ExperimentDetails) { if len(d.Experiments) > 0 { for _, exp := range d.Experiments { - // Sort variants for deterministic display. - type kv struct { - k string - v int - } - pairs := make([]kv, 0, len(exp.Variants)) - for k, v := range exp.Variants { - pairs = append(pairs, kv{k, v}) - } - slices.SortFunc(pairs, func(a, b kv) int { - switch { - case a.k < b.k: - return -1 - case a.k > b.k: - return 1 - default: - return 0 - } - }) - rows := make([][]string, 0, len(pairs)) - for _, p := range pairs { - pct := 0 - if exp.Total > 0 { - pct = p.v * 100 / exp.Total - } - rows = append(rows, []string{p.k, strconv.Itoa(p.v), strconv.Itoa(pct) + "%"}) - } + rows := printExperimentDetailsVariantRows(exp) if len(rows) > 0 { fmt.Fprintf(os.Stderr, "\n%s", console.RenderTable(console.TableConfig{ Title: fmt.Sprintf("%s (total: %d)", exp.Name, exp.Total), @@ -821,9 +801,40 @@ func printExperimentDetails(d *ExperimentDetails) { } else { fmt.Fprintln(os.Stderr, "\nNo experiment data found (state.json not present or empty).") } +} - printExperimentAnalyses(d.Analyses) +func printExperimentDetailsVariantRows(exp ExperimentVariantStats) [][]string { + // Sort variants for deterministic display. + type kv struct { + k string + v int + } + pairs := make([]kv, 0, len(exp.Variants)) + for k, v := range exp.Variants { + pairs = append(pairs, kv{k, v}) + } + slices.SortFunc(pairs, func(a, b kv) int { + switch { + case a.k < b.k: + return -1 + case a.k > b.k: + return 1 + default: + return 0 + } + }) + rows := make([][]string, 0, len(pairs)) + for _, p := range pairs { + pct := 0 + if exp.Total > 0 { + pct = p.v * 100 / exp.Total + } + rows = append(rows, []string{p.k, strconv.Itoa(p.v), strconv.Itoa(pct) + "%"}) + } + return rows +} +func printExperimentDetailsRecentRuns(d *ExperimentDetails) { if len(d.RecentRuns) > 0 { rows := make([][]string, 0, len(d.RecentRuns)) for _, run := range d.RecentRuns { diff --git a/pkg/cli/fetch.go b/pkg/cli/fetch.go index 45ea841a9ab..3f8683fc25f 100644 --- a/pkg/cli/fetch.go +++ b/pkg/cli/fetch.go @@ -90,19 +90,13 @@ func fetchRemoteWorkflow(ctx context.Context, spec *WorkflowSpec, verbose bool) spec.RepoSlug, spec.WorkflowPath, spec.Version) // Parse owner and repo from the slug - parts := strings.SplitN(spec.RepoSlug, "/", 2) - if len(parts) != 2 { + owner, repo, ok := fetchRemoteWorkflowRepoParts(spec.RepoSlug) + if !ok { return nil, fmt.Errorf("invalid repository slug: %s", spec.RepoSlug) } - owner := parts[0] - repo := parts[1] // Determine the ref to use - ref := spec.Version - if ref == "" { - ref = "main" // Default to main branch - remoteWorkflowLog.Print("No version specified, defaulting to 'main'") - } + ref := fetchRemoteWorkflowRef(spec.Version) if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Fetching %s/%s/%s@%s...", owner, repo, spec.WorkflowPath, ref))) @@ -118,33 +112,11 @@ func fetchRemoteWorkflow(ctx context.Context, spec *WorkflowSpec, verbose bool) } // Download the workflow file from GitHub - content, err := downloadFileFromGitHubForHost(ctx, owner, repo, spec.WorkflowPath, ref, spec.Host) + content, sourcePath, err := fetchRemoteWorkflowDownload(ctx, spec, owner, repo, ref) if err != nil { - // Try with common workflow directory prefixes if the direct path fails. - // This handles short workflow names without path separators (e.g. "my-workflow.md"). - if !strings.HasPrefix(spec.WorkflowPath, "workflows/") && !strings.Contains(spec.WorkflowPath, "/") { - for _, prefix := range []string{"workflows/", constants.WorkflowsDirSlash} { - altPath := prefix + spec.WorkflowPath - if !strings.HasSuffix(altPath, ".md") { - altPath += ".md" - } - remoteWorkflowLog.Printf("Direct path failed, trying: %s", altPath) - if altContent, altErr := downloadFileFromGitHubForHost(ctx, owner, repo, altPath, ref, spec.Host); altErr == nil { - remoteWorkflowLog.Printf("Downloaded workflow via alt path: %s (%d bytes)", altPath, len(altContent)) - return &FetchedWorkflow{ - Content: altContent, - CommitSHA: commitSHA, - IsLocal: false, - SourcePath: altPath, - }, nil - } - } - } - return nil, fmt.Errorf("failed to download workflow from %s/%s/%s@%s: %w", owner, repo, spec.WorkflowPath, ref, err) + return nil, err } - remoteWorkflowLog.Printf("Downloaded workflow: path=%s bytes=%d", spec.WorkflowPath, len(content)) - if verbose { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Downloaded workflow (%d bytes)", len(content)))) } @@ -153,10 +125,51 @@ func fetchRemoteWorkflow(ctx context.Context, spec *WorkflowSpec, verbose bool) Content: content, CommitSHA: commitSHA, IsLocal: false, - SourcePath: spec.WorkflowPath, + SourcePath: sourcePath, }, nil } +func fetchRemoteWorkflowRepoParts(repoSlug string) (string, string, bool) { + parts := strings.SplitN(repoSlug, "/", 2) + if len(parts) != 2 { + return "", "", false + } + return parts[0], parts[1], true +} + +func fetchRemoteWorkflowRef(version string) string { + if version != "" { + return version + } + remoteWorkflowLog.Print("No version specified, defaulting to 'main'") + return "main" // Default to main branch +} + +func fetchRemoteWorkflowDownload(ctx context.Context, spec *WorkflowSpec, owner, repo, ref string) ([]byte, string, error) { + content, err := downloadFileFromGitHubForHost(ctx, owner, repo, spec.WorkflowPath, ref, spec.Host) + if err == nil { + remoteWorkflowLog.Printf("Downloaded workflow: path=%s bytes=%d", spec.WorkflowPath, len(content)) + return content, spec.WorkflowPath, nil + } + + // Try with common workflow directory prefixes if the direct path fails. + // This handles short workflow names without path separators (e.g. "my-workflow.md"). + if !strings.HasPrefix(spec.WorkflowPath, "workflows/") && !strings.Contains(spec.WorkflowPath, "/") { + for _, prefix := range []string{"workflows/", constants.WorkflowsDirSlash} { + altPath := prefix + spec.WorkflowPath + if !strings.HasSuffix(altPath, ".md") { + altPath += ".md" + } + remoteWorkflowLog.Printf("Direct path failed, trying: %s", altPath) + if altContent, altErr := downloadFileFromGitHubForHost(ctx, owner, repo, altPath, ref, spec.Host); altErr == nil { + remoteWorkflowLog.Printf("Downloaded workflow via alt path: %s (%d bytes)", altPath, len(altContent)) + return altContent, altPath, nil + } + } + } + return nil, "", fmt.Errorf("failed to download workflow from %s/%s/%s@%s: %w", owner, repo, spec.WorkflowPath, ref, err) +} + func resolveCommitSHAWithRetries(ctx context.Context, owner, repo, ref, workflowPath, host string, verbose bool) (string, error) { attempts := len(shaResolutionRetryDelays) + 1 var lastErr error @@ -314,69 +327,76 @@ func fetchGenericURLWorkflow(ctx context.Context, spec *WorkflowSpec, verbose bo switch { case ct == "text/markdown" || ct == "text/x-markdown": - remoteWorkflowLog.Printf("URL returned markdown content (%d bytes)", len(resource.Body)) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Downloaded workflow markdown (%d bytes)", len(resource.Body)))) - } - return &FetchedWorkflow{ - Content: resource.Body, - CommitSHA: "", - IsLocal: false, - SourcePath: spec.RawURL, - }, nil + return fetchGenericURLWorkflowMarkdown(spec.RawURL, resource.Body, verbose), nil case ct == "application/json" || strings.HasSuffix(ct, "+json"): - remoteWorkflowLog.Printf("URL returned JSON content (%d bytes); converting", len(resource.Body)) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Downloaded JSON workflow (%d bytes); converting to markdown...", len(resource.Body)))) - } + return fetchGenericURLWorkflowJSON(spec, resource.Body, verbose) - remoteWorkflowLog.Printf("JSON payload:\n%s", string(resource.Body)) - - var wf JSONWorkflow - if err := json.Unmarshal(resource.Body, &wf); err != nil { - return nil, fmt.Errorf("failed to parse JSON workflow from URL: %w", err) + default: + if ct == "" { + return nil, errors.New(console.FormatErrorMessage( + "URL did not return a Content-Type header. Expected text/markdown or application/json.")) } + return nil, errors.New(console.FormatErrorMessage( + fmt.Sprintf("unsupported Content-Type %q from URL. Expected text/markdown or application/json.", ct))) + } +} - nameOverride := selectJSONImportNameOverride(spec.WorkflowName, &wf) - generated, err := ConvertJSONWorkflowToMarkdown(&wf, ConvertOptions{NameOverride: nameOverride}) - if err != nil { - return nil, fmt.Errorf("failed to convert JSON workflow: %w", err) - } +func fetchGenericURLWorkflowMarkdown(rawURL string, body []byte, verbose bool) *FetchedWorkflow { + remoteWorkflowLog.Printf("URL returned markdown content (%d bytes)", len(body)) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Downloaded workflow markdown (%d bytes)", len(body)))) + } + return &FetchedWorkflow{ + Content: body, + CommitSHA: "", + IsLocal: false, + SourcePath: rawURL, + } +} - remoteWorkflowLog.Printf("Converted JSON to markdown: filename=%s bytes=%d warnings=%d", - generated.Filename, len(generated.Markdown), len(generated.Warnings)) +func fetchGenericURLWorkflowJSON(spec *WorkflowSpec, body []byte, verbose bool) (*FetchedWorkflow, error) { + remoteWorkflowLog.Printf("URL returned JSON content (%d bytes); converting", len(body)) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Downloaded JSON workflow (%d bytes); converting to markdown...", len(body)))) + } - if verbose { - for _, w := range generated.Warnings { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("JSON workflow import: "+w)) - } - } + remoteWorkflowLog.Printf("JSON payload:\n%s", string(body)) - // Use the generated filename as the WorkflowName on the spec so that - // downstream code (e.g. add_command.go) uses the correct file name. - spec.WorkflowName = generated.Filename + var wf JSONWorkflow + if err := json.Unmarshal(body, &wf); err != nil { + return nil, fmt.Errorf("failed to parse JSON workflow from URL: %w", err) + } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Converted JSON workflow to markdown (%d bytes)", len(generated.Markdown)))) - } + generated, err := ConvertJSONWorkflowToMarkdown(&wf, ConvertOptions{NameOverride: selectJSONImportNameOverride(spec.WorkflowName, &wf)}) + if err != nil { + return nil, fmt.Errorf("failed to convert JSON workflow: %w", err) + } - return &FetchedWorkflow{ - Content: []byte(generated.Markdown), - CommitSHA: "", - IsLocal: false, - SourcePath: spec.RawURL, - ConvertedFromJSON: true, - JSONConversionWarnings: generated.Warnings, - }, nil + remoteWorkflowLog.Printf("Converted JSON to markdown: filename=%s bytes=%d warnings=%d", + generated.Filename, len(generated.Markdown), len(generated.Warnings)) + fetchGenericURLWorkflowPrintWarnings(generated.Warnings, verbose) + spec.WorkflowName = generated.Filename + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Converted JSON workflow to markdown (%d bytes)", len(generated.Markdown)))) + } - default: - if ct == "" { - return nil, errors.New(console.FormatErrorMessage( - "URL did not return a Content-Type header. Expected text/markdown or application/json.")) - } - return nil, errors.New(console.FormatErrorMessage( - fmt.Sprintf("unsupported Content-Type %q from URL. Expected text/markdown or application/json.", ct))) + return &FetchedWorkflow{ + Content: []byte(generated.Markdown), + CommitSHA: "", + IsLocal: false, + SourcePath: spec.RawURL, + ConvertedFromJSON: true, + JSONConversionWarnings: generated.Warnings, + }, nil +} + +func fetchGenericURLWorkflowPrintWarnings(warnings []string, verbose bool) { + if !verbose { + return + } + for _, w := range warnings { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("JSON workflow import: "+w)) } } diff --git a/pkg/cli/firewall_log.go b/pkg/cli/firewall_log.go index c2cadb62922..49bb1a17491 100644 --- a/pkg/cli/firewall_log.go +++ b/pkg/cli/firewall_log.go @@ -250,76 +250,80 @@ func parseFirewallLog(logPath string, verbose bool) (*FirewallAnalysis, error) { analysis := &FirewallAnalysis{ RequestsByDomain: make(map[string]DomainRequestStats), } - allowedDomainsSet := make(map[string]struct{}) blockedDomainsSet := make(map[string]struct{}) scanner := bufio.NewScanner(file) for scanner.Scan() { - line := scanner.Text() - - entry := parseFirewallLogLine(line) + entry := parseFirewallLogLine(scanner.Text()) if entry == nil { continue } + parseFirewallLogEntry(entry, analysis, allowedDomainsSet, blockedDomainsSet) + } - // Skip internal Squid error entries (client IP ::1, no domain, no destination) - // These are internal Squid connection errors (e.g., error:transaction-end-before-headers) - // and are not actual external network requests. - // Example: 1773003472.027 ::1:52010 - -:- 0.0 - 0 NONE_NONE:HIER_NONE error:transaction-end-before-headers "-" - if strings.HasPrefix(entry.ClientIPPort, "::1:") && entry.Domain == "-" && (entry.DestIPPort == "-:-" || entry.DestIPPort == "-") { - continue - } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading firewall log: %w", err) + } - analysis.TotalRequests++ - - // Determine if request was allowed or blocked - isAllowed := isRequestAllowed(entry.Decision, entry.Status) - - // Extract domain - when domain is "-" (iptables-dropped traffic not visible to Squid), - // fall back to dest IP:port so blocked requests show their actual destination instead of "-" - // Only fall back if destIPPort is a valid host:port (not "-" or "-:-" which are placeholder values) - domain := entry.Domain - if domain == "-" && entry.DestIPPort != "-" && entry.DestIPPort != "-:-" { - domain = entry.DestIPPort - } else if domain == "-" { - // Both domain and destIPPort are placeholders: iptables dropped the traffic before - // Squid could identify the destination. Use a sentinel so the entry appears in - // RequestsByDomain for informational purposes, but do NOT add it to the domain sets - // since "-" is not an actionable domain name. - domain = unknownDomain - } + parseFirewallLogFinalize(analysis, allowedDomainsSet, blockedDomainsSet) + return analysis, nil +} - if isAllowed { - analysis.AllowedRequests++ - if domain != unknownDomain { - allowedDomainsSet[domain] = struct{}{} - } - } else { - analysis.BlockedRequests++ - if domain != unknownDomain { - blockedDomainsSet[domain] = struct{}{} - } - } +func parseFirewallLogEntry(entry *FirewallLogEntry, analysis *FirewallAnalysis, allowedDomainsSet, blockedDomainsSet map[string]struct{}) { + // Skip internal Squid error entries (client IP ::1, no domain, no destination) + // These are internal Squid connection errors (e.g., error:transaction-end-before-headers) + // and are not actual external network requests. + // Example: 1773003472.027 ::1:52010 - -:- 0.0 - 0 NONE_NONE:HIER_NONE error:transaction-end-before-headers "-" + if strings.HasPrefix(entry.ClientIPPort, "::1:") && entry.Domain == "-" && (entry.DestIPPort == "-:-" || entry.DestIPPort == "-") { + return + } - // Track request count per domain - stats := analysis.RequestsByDomain[domain] - if isAllowed { - stats.Allowed++ - } else { - stats.Blocked++ + analysis.TotalRequests++ + isAllowed := isRequestAllowed(entry.Decision, entry.Status) + domain := parseFirewallLogDomain(entry) + if isAllowed { + analysis.AllowedRequests++ + if domain != unknownDomain { + allowedDomainsSet[domain] = struct{}{} + } + } else { + analysis.BlockedRequests++ + if domain != unknownDomain { + blockedDomainsSet[domain] = struct{}{} } - analysis.RequestsByDomain[domain] = stats } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading firewall log: %w", err) + stats := analysis.RequestsByDomain[domain] + if isAllowed { + stats.Allowed++ + } else { + stats.Blocked++ } + analysis.RequestsByDomain[domain] = stats +} +func parseFirewallLogDomain(entry *FirewallLogEntry) string { + // Extract domain - when domain is "-" (iptables-dropped traffic not visible to Squid), + // fall back to dest IP:port so blocked requests show their actual destination instead of "-" + // Only fall back if destIPPort is a valid host:port (not "-" or "-:-" which are placeholder values) + if entry.Domain == "-" && entry.DestIPPort != "-" && entry.DestIPPort != "-:-" { + return entry.DestIPPort + } + if entry.Domain == "-" { + // Both domain and destIPPort are placeholders: iptables dropped the traffic before + // Squid could identify the destination. Use a sentinel so the entry appears in + // RequestsByDomain for informational purposes, but do NOT add it to the domain sets + // since "-" is not an actionable domain name. + return unknownDomain + } + return entry.Domain +} + +func parseFirewallLogFinalize(analysis *FirewallAnalysis, allowedDomainsSet, blockedDomainsSet map[string]struct{}) { // Convert sets to sorted slices analysis.AllowedDomains = sliceutil.MapKeys(allowedDomainsSet) analysis.BlockedDomains = sliceutil.MapKeys(blockedDomainsSet) - sort.Strings(analysis.AllowedDomains) sort.Strings(analysis.BlockedDomains) @@ -328,7 +332,6 @@ func parseFirewallLog(logPath string, verbose bool) (*FirewallAnalysis, error) { analysis.TotalRequests, analysis.AllowedRequests, analysis.BlockedRequests, len(analysis.AllowedDomains), len(analysis.BlockedDomains)) } - return analysis, nil } // analyzeFirewallLogs analyzes firewall logs in a run directory @@ -339,47 +342,52 @@ func analyzeFirewallLogs(runDir string, verbose bool) (*FirewallAnalysis, error) // Look for firewall logs in the run directory // The logs could be in several locations depending on how they were uploaded + if analysis, handled, err := analyzeFirewallLogsSandbox(runDir, verbose); handled || err != nil { + return analysis, err + } + + if analysis, handled, err := analyzeFirewallLogsLegacyDirs(runDir, verbose); handled || err != nil { + return analysis, err + } + + return analyzeFirewallLogsIndividual(runDir, verbose) +} + +func analyzeFirewallLogsSandbox(runDir string, verbose bool) (*FirewallAnalysis, bool, error) { // First, check for sandbox/firewall/logs/ directory (new path after artifact download) // Firewall logs are uploaded from /tmp/gh-aw/sandbox/firewall/logs/ and the common parent // /tmp/gh-aw/ is stripped during artifact upload, resulting in sandbox/firewall/logs/ after download. // Within that directory, AWF writes Squid access logs to a squid-logs/ subdirectory. sandboxFirewallLogsDir := filepath.Join(runDir, "sandbox", "firewall", "logs") - if fileutil.DirExists(sandboxFirewallLogsDir) { - // AWF writes the Squid access.log to a squid-logs/ subdirectory within --proxy-logs-dir. - // Check for that subdirectory first before falling back to the parent directory. - squidSubDir := filepath.Join(sandboxFirewallLogsDir, "squid-logs") - if fileutil.DirExists(squidSubDir) { - firewallLogLog.Printf("Found firewall logs directory: sandbox/firewall/logs/squid-logs") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found firewall logs directory: sandbox/firewall/logs/squid-logs")) - } - analysis, err := analyzeMultipleFirewallLogs(squidSubDir, verbose) - if err != nil { - return nil, err - } - if analysis != nil { - return analysis, nil - } - } - // Fall back to direct *.log files at the sandbox/firewall/logs/ level (older AWF layout). - firewallLogLog.Printf("Found firewall logs directory: sandbox/firewall/logs") + if !fileutil.DirExists(sandboxFirewallLogsDir) { + return nil, false, nil + } + squidSubDir := filepath.Join(sandboxFirewallLogsDir, "squid-logs") + if fileutil.DirExists(squidSubDir) { + firewallLogLog.Printf("Found firewall logs directory: sandbox/firewall/logs/squid-logs") if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found firewall logs directory: sandbox/firewall/logs")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found firewall logs directory: sandbox/firewall/logs/squid-logs")) + } + analysis, err := analyzeMultipleFirewallLogs(squidSubDir, verbose) + if err != nil || analysis != nil { + return analysis, true, err } - return analyzeMultipleFirewallLogs(sandboxFirewallLogsDir, verbose) } + // Fall back to direct *.log files at the sandbox/firewall/logs/ level (older AWF layout). + firewallLogLog.Printf("Found firewall logs directory: sandbox/firewall/logs") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found firewall logs directory: sandbox/firewall/logs")) + } + analysis, err := analyzeMultipleFirewallLogs(sandboxFirewallLogsDir, verbose) + return analysis, true, err +} - // Second, check for directories starting with squid-logs or firewall-logs (legacy paths) - // The actual directories may have workflow-specific suffixes like: - // - squid-logs-smoke-copilot-firewall - // - squid-logs-changeset-generator - // - firewall-logs-{workflow-name} +func analyzeFirewallLogsLegacyDirs(runDir string, verbose bool) (*FirewallAnalysis, bool, error) { entries, err := os.ReadDir(runDir) if err != nil { firewallLogLog.Printf("Failed to read run directory: %v", err) - return nil, fmt.Errorf("failed to read run directory: %w", err) + return nil, false, fmt.Errorf("failed to read run directory: %w", err) } - for _, entry := range entries { if !entry.IsDir() { continue @@ -391,11 +399,14 @@ func analyzeFirewallLogs(runDir string, verbose bool) (*FirewallAnalysis, error) if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found firewall logs directory: "+name)) } - return analyzeMultipleFirewallLogs(logsDir, verbose) + analysis, err := analyzeMultipleFirewallLogs(logsDir, verbose) + return analysis, true, err } } + return nil, false, nil +} - // Check for individual log files in the run directory +func analyzeFirewallLogsIndividual(runDir string, verbose bool) (*FirewallAnalysis, error) { files, err := filepath.Glob(filepath.Join(runDir, "*.log")) if err != nil { return nil, fmt.Errorf("failed to find firewall log files: %w", err) diff --git a/pkg/cli/firewall_policy.go b/pkg/cli/firewall_policy.go index 9b8eaa2f525..07a3de163c4 100644 --- a/pkg/cli/firewall_policy.go +++ b/pkg/cli/firewall_policy.go @@ -317,6 +317,26 @@ func findMatchingRule(entry AuditLogEntry, rules []PolicyRule) *PolicyRule { func enrichWithPolicyRules(entries []AuditLogEntry, manifest *PolicyManifest) *PolicyAnalysis { firewallPolicyLog.Printf("Enriching %d entries with %d policy rules", len(entries), len(manifest.Rules)) + ruleHitMap, deniedRequests, uniqueDomains, allowedCount, deniedCount := enrichWithPolicyRulesEntries(entries, manifest) + ruleHits := enrichWithPolicyRulesRuleHits(manifest, ruleHitMap) + policySummary := enrichWithPolicyRulesSummary(manifest) + totalProcessed := allowedCount + deniedCount + + firewallPolicyLog.Printf("Enrichment complete: total=%d, allowed=%d, denied=%d, unique_domains=%d", + totalProcessed, allowedCount, deniedCount, len(uniqueDomains)) + + return &PolicyAnalysis{ + PolicySummary: policySummary, + RuleHits: ruleHits, + DeniedRequests: deniedRequests, + TotalRequests: totalProcessed, + AllowedCount: allowedCount, + DeniedCount: deniedCount, + UniqueDomains: len(uniqueDomains), + } +} + +func enrichWithPolicyRulesEntries(entries []AuditLogEntry, manifest *PolicyManifest) (map[string]int, []EnrichedRequest, map[string]struct{}, int, int) { ruleHitMap := make(map[string]int) var deniedRequests []EnrichedRequest uniqueDomains := make(map[string]struct { @@ -335,50 +355,55 @@ func enrichWithPolicyRules(entries []AuditLogEntry, manifest *PolicyManifest) *P continue } - // Strip port for domain tracking and normalize case - domain := host - if idx := strings.LastIndex(host, ":"); idx != -1 { - domain = host[:idx] - } - uniqueDomains[strings.ToLower(domain)] = struct { - }{} - - rule := findMatchingRule(entry, manifest.Rules) - - var enriched EnrichedRequest - enriched.Timestamp = entry.Timestamp - enriched.Host = host - enriched.Status = entry.Status - - if rule != nil { - enriched.RuleID = rule.ID - enriched.Action = rule.Action - ruleHitMap[rule.ID]++ - - if rule.Action == "deny" { - enriched.Reason = rule.Description - deniedRequests = append(deniedRequests, enriched) - deniedCount++ - } else { - allowedCount++ - } + uniqueDomains[enrichWithPolicyRulesDomain(host)] = struct{}{} + enriched, allowed := enrichWithPolicyRulesEntry(entry, manifest, ruleHitMap) + if allowed { + allowedCount++ } else { - // No matching rule — derive outcome from observed entry decision - if isEntryAllowed(entry) { - enriched.RuleID = "(unattributed-allow)" - enriched.Action = "allow" - enriched.Reason = "Allowed (rule not identified)" - allowedCount++ - } else { - enriched.RuleID = "(implicit-deny)" - enriched.Action = "deny" - enriched.Reason = "No matching policy rule" - deniedRequests = append(deniedRequests, enriched) - deniedCount++ - } + deniedRequests = append(deniedRequests, enriched) + deniedCount++ } } + return ruleHitMap, deniedRequests, uniqueDomains, allowedCount, deniedCount +} + +func enrichWithPolicyRulesDomain(host string) string { + // Strip port for domain tracking and normalize case + domain := host + if idx := strings.LastIndex(host, ":"); idx != -1 { + domain = host[:idx] + } + return strings.ToLower(domain) +} + +func enrichWithPolicyRulesEntry(entry AuditLogEntry, manifest *PolicyManifest, ruleHitMap map[string]int) (EnrichedRequest, bool) { + rule := findMatchingRule(entry, manifest.Rules) + enriched := EnrichedRequest{Timestamp: entry.Timestamp, Host: entry.Host, Status: entry.Status} + if rule != nil { + enriched.RuleID = rule.ID + enriched.Action = rule.Action + ruleHitMap[rule.ID]++ + if rule.Action == "deny" { + enriched.Reason = rule.Description + } + return enriched, rule.Action == "allow" + } + + // No matching rule — derive outcome from observed entry decision + if isEntryAllowed(entry) { + enriched.RuleID = "(unattributed-allow)" + enriched.Action = "allow" + enriched.Reason = "Allowed (rule not identified)" + return enriched, true + } + enriched.RuleID = "(implicit-deny)" + enriched.Action = "deny" + enriched.Reason = "No matching policy rule" + return enriched, false +} + +func enrichWithPolicyRulesRuleHits(manifest *PolicyManifest, ruleHitMap map[string]int) []RuleHitStats { // Build rule hits table, preserving rule order var ruleHits []RuleHitStats for _, rule := range manifest.Rules { @@ -388,7 +413,10 @@ func enrichWithPolicyRules(entries []AuditLogEntry, manifest *PolicyManifest) *P Hits: hits, }) } + return ruleHits +} +func enrichWithPolicyRulesSummary(manifest *PolicyManifest) string { // Build policy summary string sslBump := "disabled" if manifest.SSLBumpEnabled { @@ -398,22 +426,7 @@ func enrichWithPolicyRules(entries []AuditLogEntry, manifest *PolicyManifest) *P if manifest.DLPEnabled { dlp = "enabled" } - policySummary := fmt.Sprintf("%d rules, SSL Bump %s, DLP %s", len(manifest.Rules), sslBump, dlp) - - totalProcessed := allowedCount + deniedCount - - firewallPolicyLog.Printf("Enrichment complete: total=%d, allowed=%d, denied=%d, unique_domains=%d", - totalProcessed, allowedCount, deniedCount, len(uniqueDomains)) - - return &PolicyAnalysis{ - PolicySummary: policySummary, - RuleHits: ruleHits, - DeniedRequests: deniedRequests, - TotalRequests: totalProcessed, - AllowedCount: allowedCount, - DeniedCount: deniedCount, - UniqueDomains: len(uniqueDomains), - } + return fmt.Sprintf("%d rules, SSL Bump %s, DLP %s", len(manifest.Rules), sslBump, dlp) } // detectFirewallAuditArtifacts looks for policy-manifest.json and audit.jsonl in the run directory. @@ -427,59 +440,66 @@ func enrichWithPolicyRules(entries []AuditLogEntry, manifest *PolicyManifest) *P func detectFirewallAuditArtifacts(runDir string) (manifestPath, auditJSONLPath string, err error) { firewallPolicyLog.Printf("Detecting firewall audit artifacts in: %s", runDir) - // checkDir probes dir for policy-manifest.json and audit.jsonl, populating the - // return variables for any files not yet found. Returns true when both are found. - checkDir := func(dir, label string) bool { - if manifestPath == "" { - candidate := filepath.Join(dir, "policy-manifest.json") - if fileutil.FileExists(candidate) { - manifestPath = candidate - firewallPolicyLog.Printf("Found policy manifest in %s: %s", label, manifestPath) - } - } - if auditJSONLPath == "" { - candidate := filepath.Join(dir, "audit.jsonl") - if fileutil.FileExists(candidate) { - auditJSONLPath = candidate - firewallPolicyLog.Printf("Found audit JSONL in %s: %s", label, auditJSONLPath) - } - } - return manifestPath != "" && auditJSONLPath != "" - } - // 1. Primary path: sandbox/firewall/audit/ after flattenUnifiedArtifact - if checkDir(filepath.Join(runDir, "sandbox", "firewall", "audit"), "sandbox/firewall/audit") { + if detectFirewallAuditArtifactsCheckDir(filepath.Join(runDir, "sandbox", "firewall", "audit"), "sandbox/firewall/audit", &manifestPath, &auditJSONLPath) { return } // 2 & 3. Non-flattened unified agent artifact (before flattenUnifiedArtifact is called, // e.g., when the audit command is run on a directory populated via `gh run download`). // Handles "agent", "agent-artifacts", and workflow_call prefixed names (e.g. "hash-agent"). - if agentDir := findArtifactDir(runDir, "agent", "agent-artifacts"); agentDir != "" { - // Guard: findArtifactDir checks existence but not type; skip if it resolved to a file. - if info, err := os.Stat(agentDir); err != nil || !info.IsDir() { - firewallPolicyLog.Printf("Skipping agent artifact path (not a directory): %s", agentDir) - } else { - agentBase := filepath.Base(agentDir) - // New artifact structure (actions/upload-artifact v4+, /tmp/gh-aw/ prefix stripped): - // /sandbox/firewall/audit/ - if !checkDir(filepath.Join(agentDir, "sandbox", "firewall", "audit"), agentBase+"/sandbox/firewall/audit") { - // Old artifact structure (/tmp/gh-aw/ prefix preserved inside the artifact): - // /tmp/gh-aw/sandbox/firewall/audit/ - checkDir(filepath.Join(agentDir, "tmp", "gh-aw", "sandbox", "firewall", "audit"), agentBase+constants.AWFAuditDir) - } - if manifestPath != "" && auditJSONLPath != "" { - return manifestPath, auditJSONLPath, nil - } - } + if detectFirewallAuditArtifactsAgentDir(runDir, &manifestPath, &auditJSONLPath) { + return manifestPath, auditJSONLPath, nil } // 4. Legacy separate firewall-audit-logs artifact (backward compat for older runs that // uploaded the audit directory as a standalone artifact named firewall-audit-logs). + if err = detectFirewallAuditArtifactsLegacy(runDir, &manifestPath, &auditJSONLPath); err != nil { + return + } + + return manifestPath, auditJSONLPath, nil +} + +func detectFirewallAuditArtifactsCheckDir(dir, label string, manifestPath, auditJSONLPath *string) bool { + if *manifestPath == "" { + candidate := filepath.Join(dir, "policy-manifest.json") + if fileutil.FileExists(candidate) { + *manifestPath = candidate + firewallPolicyLog.Printf("Found policy manifest in %s: %s", label, *manifestPath) + } + } + if *auditJSONLPath == "" { + candidate := filepath.Join(dir, "audit.jsonl") + if fileutil.FileExists(candidate) { + *auditJSONLPath = candidate + firewallPolicyLog.Printf("Found audit JSONL in %s: %s", label, *auditJSONLPath) + } + } + return *manifestPath != "" && *auditJSONLPath != "" +} + +func detectFirewallAuditArtifactsAgentDir(runDir string, manifestPath, auditJSONLPath *string) bool { + agentDir := findArtifactDir(runDir, "agent", "agent-artifacts") + if agentDir == "" { + return false + } + // Guard: findArtifactDir checks existence but not type; skip if it resolved to a file. + if info, err := os.Stat(agentDir); err != nil || !info.IsDir() { + firewallPolicyLog.Printf("Skipping agent artifact path (not a directory): %s", agentDir) + return false + } + agentBase := filepath.Base(agentDir) + if !detectFirewallAuditArtifactsCheckDir(filepath.Join(agentDir, "sandbox", "firewall", "audit"), agentBase+"/sandbox/firewall/audit", manifestPath, auditJSONLPath) { + detectFirewallAuditArtifactsCheckDir(filepath.Join(agentDir, "tmp", "gh-aw", "sandbox", "firewall", "audit"), agentBase+constants.AWFAuditDir, manifestPath, auditJSONLPath) + } + return *manifestPath != "" && *auditJSONLPath != "" +} + +func detectFirewallAuditArtifactsLegacy(runDir string, manifestPath, auditJSONLPath *string) error { entries, readErr := os.ReadDir(runDir) if readErr != nil { - err = fmt.Errorf("detectFirewallAuditArtifacts: reading run dir %s: %w", runDir, readErr) - return + return fmt.Errorf("detectFirewallAuditArtifacts: reading run dir %s: %w", runDir, readErr) } for _, entry := range entries { if !entry.IsDir() { @@ -487,11 +507,10 @@ func detectFirewallAuditArtifacts(runDir string) (manifestPath, auditJSONLPath s } name := entry.Name() if strings.HasPrefix(name, "firewall-audit") { - checkDir(filepath.Join(runDir, name), name) + detectFirewallAuditArtifactsCheckDir(filepath.Join(runDir, name), name, manifestPath, auditJSONLPath) } } - - return manifestPath, auditJSONLPath, nil + return nil } // analyzeFirewallPolicy loads policy manifest and audit JSONL, then enriches with rule attribution. diff --git a/pkg/cli/fix_codemods.go b/pkg/cli/fix_codemods.go index 15914c14d06..624c10a7d5f 100644 --- a/pkg/cli/fix_codemods.go +++ b/pkg/cli/fix_codemods.go @@ -44,7 +44,15 @@ type CodemodResult struct { // GetAllCodemods returns all available codemods in the registry func GetAllCodemods() []Codemod { - codemods := []Codemod{ + codemods := getAllCodemodsCore() + codemods = append(codemods, getAllCodemodsMigration()...) + codemods = append(codemods, getAllCodemodsRecent()...) + fixCodemodsLog.Printf("Loaded codemod registry: %d codemods available", len(codemods)) + return codemods +} + +func getAllCodemodsCore() []Codemod { + return []Codemod{ getTimeoutMinutesCodemod(), getNetworkFirewallCodemod(), getCommandToSlashCommandCodemod(), @@ -63,18 +71,23 @@ func GetAllCodemods() []Codemod { getDiscussionTriggerCategoriesLowercaseCodemod(), getMCPModeToTypeCodemod(), getInstallScriptURLCodemod(), - getBashAnonymousRemovalCodemod(), // Replace bash: with bash: false - getBashSingleQuotedArgsCodemod(), // Rewrite single-quoted bash args to double-quoted form - getActivationOutputsCodemod(), // Transform needs.activation.outputs.* to steps.sanitized.outputs.* - getRolesToOnRolesCodemod(), // Move top-level roles to on.roles - getBotsToOnBotsCodemod(), // Move top-level bots to on.bots - getEngineStepsToTopLevelCodemod(), // Move engine.steps to top-level steps - getEngineMaxRunsToTopLevelCodemod(), // Move engine.max-runs to top-level max-turns - getMaxRunsToMaxTurnsCodemod(), // Rename top-level max-runs to max-turns - getEngineMaxTurnsToTopLevelCodemod(), // Move engine.max-turns to top-level max-turns - getStepsRunSecretsToEnvCodemod(), // Move all ${{ ... }} expressions in step run fields to step env bindings - getEngineEnvSecretsCodemod(), // Remove unsafe secret-bearing engine.env entries - getTopLevelEnvSecretsGuidedErrorCodemod(), // Detect secrets in top-level env: and emit guided error + getBashAnonymousRemovalCodemod(), // Replace bash: with bash: false + getBashSingleQuotedArgsCodemod(), // Rewrite single-quoted bash args to double-quoted form + getActivationOutputsCodemod(), // Transform needs.activation.outputs.* to steps.sanitized.outputs.* + getRolesToOnRolesCodemod(), // Move top-level roles to on.roles + getBotsToOnBotsCodemod(), // Move top-level bots to on.bots + getEngineStepsToTopLevelCodemod(), // Move engine.steps to top-level steps + getEngineMaxRunsToTopLevelCodemod(), // Move engine.max-runs to top-level max-turns + getMaxRunsToMaxTurnsCodemod(), // Rename top-level max-runs to max-turns + getEngineMaxTurnsToTopLevelCodemod(), // Move engine.max-turns to top-level max-turns + getStepsRunSecretsToEnvCodemod(), // Move all ${{ ... }} expressions in step run fields to step env bindings + getEngineEnvSecretsCodemod(), // Remove unsafe secret-bearing engine.env entries + getTopLevelEnvSecretsGuidedErrorCodemod(), // Detect secrets in top-level env: and emit guided error + } +} + +func getAllCodemodsMigration() []Codemod { + return []Codemod{ getAssignToAgentDefaultAgentCodemod(), // Rename deprecated default-agent to name in assign-to-agent getPlaywrightDomainsToNetworkAllowedCodemod(), // Migrate tools.playwright.allowed_domains to network.allowed getExpiresIntegerToDayStringCodemod(), // Convert expires integer (days) to string with 'd' suffix @@ -95,21 +108,24 @@ func GetAllCodemods() []Codemod { getDependabotPermissionsCodemod(), // Add vulnerability-alerts: read when dependabot toolset is used getGitHubReposToAllowedReposCodemod(), // Rename deprecated tools.github.repos to tools.github.allowed-repos getCopilotRequestsFeatureToPermissionsCodemod(), // Migrate features.copilot-requests to permissions.copilot-requests - getByokCopilotFeatureRemovalCodemod(), // Remove deprecated features.byok-copilot (Copilot BYOK is default) - getInlineAgentsFeatureRemovalCodemod(), // Remove deprecated features.inline-agents (inline sub-agents now default) - getCliProxyFeatureToGitHubModeCodemod(), // Migrate features.cli-proxy: true to tools.github.mode: gh-proxy - getDIFCProxyToIntegrityProxyCodemod(), // Migrate deprecated features.difc-proxy to tools.github.integrity-proxy - getMountAsCLIsToCLIProxyCodemod(), // Rename tools.mount-as-clis to tools.cli-proxy and remove features.mcp-cli - getSandboxMCPContainerRemovalCodemod(), // Remove deprecated sandbox.mcp.container (now managed internally) - getSandboxMCPVersionRemovalCodemod(), // Remove deprecated sandbox.mcp.version (now managed internally) - getSandboxAgentFalseRemovalCodemod(), // Remove deprecated sandbox.agent: false (rejected in strict mode) - getInferToDisableModelInvocationCodemod(), // Migrate deprecated 'infer' to 'disable-model-invocation' - getRunInstallScriptsToRuntimesNodeCodemod(), // Move top-level run-install-scripts under runtimes.node - getMentionsAllowTeamMembersCodemod(), // Rename allow-team-members to allowed-collaborators in safe-outputs.mentions - getEngineCopilotSDKDriverToDriverCodemod(), // Rename deprecated engine.copilot-sdk-driver to engine.driver } - fixCodemodsLog.Printf("Loaded codemod registry: %d codemods available", len(codemods)) - return codemods +} + +func getAllCodemodsRecent() []Codemod { + return []Codemod{ + getByokCopilotFeatureRemovalCodemod(), // Remove deprecated features.byok-copilot (Copilot BYOK is default) + getInlineAgentsFeatureRemovalCodemod(), // Remove deprecated features.inline-agents (inline sub-agents now default) + getCliProxyFeatureToGitHubModeCodemod(), // Migrate features.cli-proxy: true to tools.github.mode: gh-proxy + getDIFCProxyToIntegrityProxyCodemod(), // Migrate deprecated features.difc-proxy to tools.github.integrity-proxy + getMountAsCLIsToCLIProxyCodemod(), // Rename tools.mount-as-clis to tools.cli-proxy and remove features.mcp-cli + getSandboxMCPContainerRemovalCodemod(), // Remove deprecated sandbox.mcp.container (now managed internally) + getSandboxMCPVersionRemovalCodemod(), // Remove deprecated sandbox.mcp.version (now managed internally) + getSandboxAgentFalseRemovalCodemod(), // Remove deprecated sandbox.agent: false (rejected in strict mode) + getInferToDisableModelInvocationCodemod(), // Migrate deprecated 'infer' to 'disable-model-invocation' + getRunInstallScriptsToRuntimesNodeCodemod(), // Move top-level run-install-scripts under runtimes.node + getMentionsAllowTeamMembersCodemod(), // Rename allow-team-members to allowed-collaborators in safe-outputs.mentions + getEngineCopilotSDKDriverToDriverCodemod(), // Rename deprecated engine.copilot-sdk-driver to engine.driver + } } // GetCodemods returns all codemods except any explicitly disabled by ID. diff --git a/pkg/cli/fix_command.go b/pkg/cli/fix_command.go index 8d1c948ef1d..9c1619778b3 100644 --- a/pkg/cli/fix_command.go +++ b/pkg/cli/fix_command.go @@ -116,36 +116,11 @@ func listAvailableCodemods() error { func runFixCommand(workflowIDs []string, write bool, verbose bool, workflowDir string, disabledCodemodIDs []string) error { fixLog.Printf("Running fix command: workflowIDs=%v, write=%v, verbose=%v, workflowDir=%s, disabledCodemodIDs=%v", workflowIDs, write, verbose, workflowDir, disabledCodemodIDs) - // Set up workflow directory (using default if not specified) - if workflowDir == "" { - workflowDir = constants.GetWorkflowDir() - fixLog.Printf("Using default workflow directory: %s", workflowDir) - } else { - workflowDir = filepath.Clean(workflowDir) - fixLog.Printf("Using custom workflow directory: %s", workflowDir) - } - // Get workflow files to process - var files []string - var err error - - if len(workflowIDs) > 0 { - // Process specific workflows - for _, workflowID := range workflowIDs { - file, err := resolveWorkflowFileInDir(workflowID, verbose, workflowDir) - if err != nil { - return err - } - files = append(files, file) - } - } else { - // Process all workflows in the workflow directory - files, err = getMarkdownWorkflowFiles(workflowDir) - if err != nil { - return err - } + files, err := runFixCommandWorkflowFiles(workflowIDs, verbose, workflowDir) + if err != nil { + return err } - if len(files) == 0 { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No workflow files found.")) return nil @@ -159,45 +134,93 @@ func runFixCommand(workflowIDs []string, write bool, verbose bool, workflowDir s fixLog.Printf("Loaded %d codemods", len(codemods)) // Process each file - var totalFixed int - var totalFiles int - var totalGuidedErrors int - var totalProcessingErrors int - var workflowsNeedingFixes []workflowFixInfo + stats := runFixCommandProcessFiles(files, codemods, write, verbose) + + runFixCommandUpdateSupportFiles(write, verbose) + runFixCommandDeleteDeprecatedSchema(write, verbose) + runFixCommandPrintSummary(stats, write) + if stats.totalGuidedErrors > 0 { + pluralSuffix := "file needs" + if stats.totalGuidedErrors > 1 { + pluralSuffix = "files need" + } + fmt.Fprintf(os.Stderr, "%s\n", console.FormatErrorMessage(fmt.Sprintf("%d %s a manual fix", stats.totalGuidedErrors, pluralSuffix))) + return &ExitCodeError{Code: 2} + } + + if stats.totalProcessingErrors > 0 { + return &ExitCodeError{Code: 1} + } + + return nil +} + +type runFixCommandStats struct { + totalFixed int + totalFiles int + totalGuidedErrors int + totalProcessingErrors int + workflowsNeedingFixes []workflowFixInfo +} + +func runFixCommandWorkflowFiles(workflowIDs []string, verbose bool, workflowDir string) ([]string, error) { + if workflowDir == "" { + workflowDir = constants.GetWorkflowDir() + fixLog.Printf("Using default workflow directory: %s", workflowDir) + } else { + workflowDir = filepath.Clean(workflowDir) + fixLog.Printf("Using custom workflow directory: %s", workflowDir) + } + if len(workflowIDs) == 0 { + return getMarkdownWorkflowFiles(workflowDir) + } + var files []string + for _, workflowID := range workflowIDs { + file, err := resolveWorkflowFileInDir(workflowID, verbose, workflowDir) + if err != nil { + return nil, err + } + files = append(files, file) + } + return files, nil +} + +func runFixCommandProcessFiles(files []string, codemods []Codemod, write bool, verbose bool) runFixCommandStats { + var stats runFixCommandStats for _, file := range files { fixLog.Printf("Processing file: %s", file) - fixed, appliedFixes, err := processWorkflowFileWithInfo(file, codemods, write, verbose) if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatErrorMessage(fmt.Sprintf("Error processing %s: %v", filepath.Base(file), err))) - var guidedErr *GuidedError - if errors.As(err, &guidedErr) { - totalGuidedErrors++ - totalFiles++ - } else { - totalProcessingErrors++ - } + runFixCommandRecordError(&stats, file, err) continue } - - totalFiles++ + stats.totalFiles++ if fixed { - totalFixed++ + stats.totalFixed++ if !write { - workflowsNeedingFixes = append(workflowsNeedingFixes, workflowFixInfo{ - File: filepath.Base(file), - Fixes: appliedFixes, - }) + stats.workflowsNeedingFixes = append(stats.workflowsNeedingFixes, workflowFixInfo{File: filepath.Base(file), Fixes: appliedFixes}) } } } + return stats +} + +func runFixCommandRecordError(stats *runFixCommandStats, file string, err error) { + fmt.Fprintf(os.Stderr, "%s\n", console.FormatErrorMessage(fmt.Sprintf("Error processing %s: %v", filepath.Base(file), err))) + var guidedErr *GuidedError + if errors.As(err, &guidedErr) { + stats.totalGuidedErrors++ + stats.totalFiles++ + } else { + stats.totalProcessingErrors++ + } +} +func runFixCommandUpdateSupportFiles(write bool, verbose bool) { // Update prompt and skill files (similar to init command) // This ensures the latest templates are always used fixLog.Print("Updating prompt and skill files") - - // Update dispatcher skill if err := ensureAgenticWorkflowsDispatcher(verbose, false); err != nil { fixLog.Printf("Failed to update dispatcher skill: %v", err) fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to update dispatcher skill: %v", err))) @@ -206,86 +229,80 @@ func runFixCommand(workflowIDs []string, write bool, verbose bool, workflowDir s fixLog.Printf("Failed to update agentic workflows custom agent: %v", err) fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to update agentic workflows custom agent: %v", err))) } - - // Delete old template files from pkg/cli/templates/ (only with --write) if write { - fixLog.Print("Cleaning up old template files") - if err := deleteOldTemplateFiles(verbose); err != nil { - fixLog.Printf("Failed to delete old template files: %v", err) - fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete old template files: %v", err))) - } + runFixCommandCleanupGeneratedFiles(verbose) } +} - // Delete old agent files if write flag is set - if write { - fixLog.Print("Deleting old agent files") - if err := deleteLegacyAgentFiles(verbose); err != nil { - fixLog.Printf("Failed to delete old agent files: %v", err) - fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete old agent files: %v", err))) - } +func runFixCommandCleanupGeneratedFiles(verbose bool) { + fixLog.Print("Cleaning up old template files") + if err := deleteOldTemplateFiles(verbose); err != nil { + fixLog.Printf("Failed to delete old template files: %v", err) + fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete old template files: %v", err))) + } + fixLog.Print("Deleting old agent files") + if err := deleteLegacyAgentFiles(verbose); err != nil { + fixLog.Printf("Failed to delete old agent files: %v", err) + fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete old agent files: %v", err))) } +} - // Delete deprecated schema file if it exists +func runFixCommandDeleteDeprecatedSchema(write bool, verbose bool) { schemaPath := filepath.Join(".github", "aw", "schemas", "agentic-workflow.json") - if fileutil.FileExists(schemaPath) { - fixLog.Printf("Found deprecated schema file at %s", schemaPath) - if write { - if err := os.Remove(schemaPath); err != nil { - fixLog.Printf("Failed to delete schema file: %v", err) - fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete deprecated schema file: %v", err))) - } else { - fixLog.Print("Deleted deprecated schema file") - if verbose { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatSuccessMessage("Deleted deprecated .github/aw/schemas/agentic-workflow.json")) - } - } - } else { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Would delete deprecated .github/aw/schemas/agentic-workflow.json")) - } + if !fileutil.FileExists(schemaPath) { + return + } + fixLog.Printf("Found deprecated schema file at %s", schemaPath) + if !write { + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Would delete deprecated .github/aw/schemas/agentic-workflow.json")) + return } + if err := os.Remove(schemaPath); err != nil { + fixLog.Printf("Failed to delete schema file: %v", err) + fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete deprecated schema file: %v", err))) + return + } + fixLog.Print("Deleted deprecated schema file") + if verbose { + fmt.Fprintf(os.Stderr, "%s\n", console.FormatSuccessMessage("Deleted deprecated .github/aw/schemas/agentic-workflow.json")) + } +} - // Print summary +func runFixCommandPrintSummary(stats runFixCommandStats, write bool) { fmt.Fprintln(os.Stderr, "") if write { - if totalFixed > 0 { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatSuccessMessage(fmt.Sprintf("✓ Fixed %d of %d workflow files", totalFixed, totalFiles))) - } else if totalGuidedErrors == 0 && totalProcessingErrors == 0 { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("✓ No fixes needed")) - } + runFixCommandPrintWriteSummary(stats) } else { - if totalFixed > 0 { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Would fix %d of %d workflow files", totalFixed, totalFiles))) - fmt.Fprintln(os.Stderr, "") - - // Output as agent prompt - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("To fix these issues, run:")) - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, " gh aw fix --write") - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Or fix them individually:")) - fmt.Fprintln(os.Stderr, "") - for _, wf := range workflowsNeedingFixes { - fmt.Fprintf(os.Stderr, " gh aw fix %s --write\n", strings.TrimSuffix(wf.File, ".md")) - } - } else if totalGuidedErrors == 0 && totalProcessingErrors == 0 { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("✓ No fixes needed")) - } + runFixCommandPrintDryRunSummary(stats) } +} - if totalGuidedErrors > 0 { - pluralSuffix := "file needs" - if totalGuidedErrors > 1 { - pluralSuffix = "files need" - } - fmt.Fprintf(os.Stderr, "%s\n", console.FormatErrorMessage(fmt.Sprintf("%d %s a manual fix", totalGuidedErrors, pluralSuffix))) - return &ExitCodeError{Code: 2} +func runFixCommandPrintWriteSummary(stats runFixCommandStats) { + if stats.totalFixed > 0 { + fmt.Fprintf(os.Stderr, "%s\n", console.FormatSuccessMessage(fmt.Sprintf("✓ Fixed %d of %d workflow files", stats.totalFixed, stats.totalFiles))) + } else if stats.totalGuidedErrors == 0 && stats.totalProcessingErrors == 0 { + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("✓ No fixes needed")) } +} - if totalProcessingErrors > 0 { - return &ExitCodeError{Code: 1} +func runFixCommandPrintDryRunSummary(stats runFixCommandStats) { + if stats.totalFixed == 0 { + if stats.totalGuidedErrors == 0 && stats.totalProcessingErrors == 0 { + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("✓ No fixes needed")) + } + return + } + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Would fix %d of %d workflow files", stats.totalFixed, stats.totalFiles))) + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("To fix these issues, run:")) + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, " gh aw fix --write") + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Or fix them individually:")) + fmt.Fprintln(os.Stderr, "") + for _, wf := range stats.workflowsNeedingFixes { + fmt.Fprintf(os.Stderr, " gh aw fix %s --write\n", strings.TrimSuffix(wf.File, ".md")) } - - return nil } // workflowFixInfo tracks workflow files that need fixes @@ -307,15 +324,38 @@ func processWorkflowFileWithInfo(filePath string, codemods []Codemod, write bool originalContent := string(content) currentContent := originalContent - // Track what was applied + // Apply each codemod + currentContent, appliedCodemods, hasChanges, err := processWorkflowFileWithInfoApplyCodemods(currentContent, codemods) + if err != nil { + return false, nil, err + } + + // If no changes, report and return + if !hasChanges { + if verbose { + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf(" %s - no fixes needed", filepath.Base(filePath)))) + } + return false, nil, nil + } + + // Report changes + fileName := filepath.Base(filePath) + if write { + if err := processWorkflowFileWithInfoWrite(filePath, fileName, currentContent, appliedCodemods, verbose); err != nil { + return false, nil, err + } + } else { + processWorkflowFileWithInfoDryRun(fileName, appliedCodemods) + } + + return true, appliedCodemods, nil +} + +func processWorkflowFileWithInfoApplyCodemods(currentContent string, codemods []Codemod) (string, []string, bool, error) { var appliedCodemods []string var hasChanges bool - - // Apply each codemod for _, codemod := range codemods { fixLog.Printf("Attempting codemod: %s", codemod.ID) - - // Re-parse frontmatter for each codemod to get fresh state currentResult, err := parser.ExtractFrontmatterFromContent(currentContent) if err != nil { fixLog.Printf("Failed to parse frontmatter for codemod %s: %v", codemod.ID, err) @@ -327,9 +367,9 @@ func processWorkflowFileWithInfo(filePath string, codemods []Codemod, write bool fixLog.Printf("Codemod %s failed: %v", codemod.ID, err) wrappedErr := fmt.Errorf("codemod %s failed: %w", codemod.ID, err) if codemod.Guided { - return false, nil, &GuidedError{Cause: wrappedErr} + return "", nil, false, &GuidedError{Cause: wrappedErr} } - return false, nil, wrappedErr + return "", nil, false, wrappedErr } if applied { @@ -339,39 +379,29 @@ func processWorkflowFileWithInfo(filePath string, codemods []Codemod, write bool fixLog.Printf("Applied codemod: %s", codemod.ID) } } + return currentContent, appliedCodemods, hasChanges, nil +} - // If no changes, report and return - if !hasChanges { - if verbose { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf(" %s - no fixes needed", filepath.Base(filePath)))) - } - return false, nil, nil +func processWorkflowFileWithInfoWrite(filePath, fileName, currentContent string, appliedCodemods []string, verbose bool) error { + // Write the file with owner-only read/write permissions (0600) for security best practices + if err := os.WriteFile(filePath, []byte(currentContent), constants.FilePermSensitive); err != nil { + return fmt.Errorf("failed to write file: %w", err) } - - // Report changes - fileName := filepath.Base(filePath) - if write { - // Write the file with owner-only read/write permissions (0600) for security best practices - if err := os.WriteFile(filePath, []byte(currentContent), constants.FilePermSensitive); err != nil { - return false, nil, fmt.Errorf("failed to write file: %w", err) - } - - if err := scaffoldSerenaSharedWorkflowIfNeeded(filePath, appliedCodemods, currentContent, verbose); err != nil { - return false, nil, fmt.Errorf("failed to scaffold shared Serena workflow: %w", err) - } - - fmt.Fprintf(os.Stderr, "%s\n", console.FormatSuccessMessage(fileName)) - for _, codemodName := range appliedCodemods { - fmt.Fprintf(os.Stderr, " • %s\n", codemodName) - } - } else { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage("⚠ "+fileName)) - for _, codemodName := range appliedCodemods { - fmt.Fprintf(os.Stderr, " • %s\n", codemodName) - } + if err := scaffoldSerenaSharedWorkflowIfNeeded(filePath, appliedCodemods, currentContent, verbose); err != nil { + return fmt.Errorf("failed to scaffold shared Serena workflow: %w", err) + } + fmt.Fprintf(os.Stderr, "%s\n", console.FormatSuccessMessage(fileName)) + for _, codemodName := range appliedCodemods { + fmt.Fprintf(os.Stderr, " • %s\n", codemodName) } + return nil +} - return true, appliedCodemods, nil +func processWorkflowFileWithInfoDryRun(fileName string, appliedCodemods []string) { + fmt.Fprintf(os.Stderr, "%s\n", console.FormatWarningMessage("⚠ "+fileName)) + for _, codemodName := range appliedCodemods { + fmt.Fprintf(os.Stderr, " • %s\n", codemodName) + } } const scaffoldedSerenaSharedWorkflow = `--- diff --git a/pkg/cli/forecast.go b/pkg/cli/forecast.go index 0b83d4f2638..059a833ccb5 100644 --- a/pkg/cli/forecast.go +++ b/pkg/cli/forecast.go @@ -24,35 +24,21 @@ var forecastPeriodDays = map[string]int{ // RunForecast is the entry point for the forecast command. func RunForecast(config ForecastConfig) error { forecastRunLog.Printf("Running forecast: workflows=%v, days=%d, period=%s, eval=%v", config.WorkflowIDs, config.Days, config.Period, config.EvalMode) - if config.TimeoutMinutes < 0 { - return fmt.Errorf("invalid timeout value: %d; must be >= 0", config.TimeoutMinutes) + ctx, stop, err := runForecastContext(config) + if err != nil { + return err } - - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() - if config.TimeoutMinutes > 0 { - timeoutCtx, cancel := context.WithTimeout(ctx, time.Duration(config.TimeoutMinutes)*time.Minute) - defer cancel() - ctx = timeoutCtx - } // Emit experimental warning so users know this command is not yet stable. // Per R-IMPL-040: the warning MUST NOT be emitted when --json is specified, // as JSON callers are assumed to be automated pipelines that handle warnings separately. - if !config.JSONOutput { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("forecast is an experimental command and may change without notice")) - } + runForecastExperimentalWarning(config) // Validate period. - periodDays, ok := forecastPeriodDays[config.Period] - if !ok { - return fmt.Errorf("invalid period %q: must be 'week' or 'month'", config.Period) - } - if config.Days != 7 && config.Days != 30 { - return fmt.Errorf("invalid days value: %d; must be 7 or 30", config.Days) - } - if config.SampleSize <= 0 { - config.SampleSize = 100 + periodDays, err := runForecastValidate(&config) + if err != nil { + return err } // Resolve the list of workflow IDs to forecast. @@ -66,16 +52,89 @@ func RunForecast(config ForecastConfig) error { } now := time.Now() + startDate, validationStartDate, validationEndDate := runForecastDates(config, now, periodDays) + + runForecastPrintStart(workflowIDs, config) + spinner := runForecastSpinner(config) + + results := make([]ForecastWorkflowResult, 0, len(workflowIDs)) + if err := runForecastWorkflows(runForecastWorkflowsParams{ + Ctx: ctx, + WorkflowIDs: workflowIDs, + StartDate: startDate, + ValidationStartDate: validationStartDate, + ValidationEndDate: validationEndDate, + Config: config, + PeriodDays: periodDays, + Spinner: spinner, + Now: now, + Results: &results, + }); err != nil { + return err + } + + if !config.Verbose { + spinner.Stop() + } + + // Sort results by Monte Carlo P50 (or point estimate when MC unavailable) descending. + runForecastSortResults(results) + + return runForecastRender(results, config, now) +} + +func runForecastContext(config ForecastConfig) (context.Context, context.CancelFunc, error) { + if config.TimeoutMinutes < 0 { + return nil, nil, fmt.Errorf("invalid timeout value: %d; must be >= 0", config.TimeoutMinutes) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + if config.TimeoutMinutes == 0 { + return ctx, stop, nil + } + timeoutCtx, cancel := context.WithTimeout(ctx, time.Duration(config.TimeoutMinutes)*time.Minute) + return timeoutCtx, func() { + cancel() + stop() + }, nil +} + +func runForecastValidate(config *ForecastConfig) (int, error) { + periodDays, ok := forecastPeriodDays[config.Period] + if !ok { + return 0, fmt.Errorf("invalid period %q: must be 'week' or 'month'", config.Period) + } + if config.Days != 7 && config.Days != 30 { + return 0, fmt.Errorf("invalid days value: %d; must be 7 or 30", config.Days) + } + if config.SampleSize <= 0 { + config.SampleSize = 100 + } + return periodDays, nil +} + +func runForecastExperimentalWarning(config ForecastConfig) { + if !config.JSONOutput { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("forecast is an experimental command and may change without notice")) + } +} - // In eval mode, shift the entire date range back by one period so we can - // compare the forecast against the actual runs in the most recent period. - // - // ┌──────────────────────────────────────────────────────────────────┐ - // │ [anchor - days ... anchor] training │ [anchor ... now] val │ - // └──────────────────────────────────────────────────────────────────┘ - // anchor = now - periodDays - // - // Normal mode: startDate = now - days (no anchor shift). +func runForecastPrintStart(workflowIDs []string, config ForecastConfig) { + if !config.Verbose && !config.JSONOutput { + label := fmt.Sprintf("Forecasting %d workflow(s) using %d-day history → projecting per %s", + len(workflowIDs), config.Days, config.Period) + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(label)) + } +} + +func runForecastSpinner(config ForecastConfig) *console.SpinnerWrapper { + spinner := console.NewSpinner("Sampling workflow run history…") + if !config.Verbose { + spinner.Start() + } + return spinner +} + +func runForecastDates(config ForecastConfig, now time.Time, periodDays int) (string, string, string) { var anchor time.Time var validationStartDate, validationEndDate string if config.EvalMode { @@ -89,71 +148,77 @@ func RunForecast(config ForecastConfig) error { startDate := now.AddDate(0, 0, -config.Days).Format("2006-01-02") if config.EvalMode { - // Training window ends at the anchor, not now. startDate = anchor.AddDate(0, 0, -config.Days).Format("2006-01-02") } + return startDate, validationStartDate, validationEndDate +} - if !config.Verbose && !config.JSONOutput { - label := fmt.Sprintf("Forecasting %d workflow(s) using %d-day history → projecting per %s", - len(workflowIDs), config.Days, config.Period) - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(label)) - } - - spinner := console.NewSpinner("Sampling workflow run history…") - if !config.Verbose { - spinner.Start() - } +type runForecastWorkflowsParams struct { + Ctx context.Context + WorkflowIDs []string + StartDate string + ValidationStartDate string + ValidationEndDate string + Config ForecastConfig + PeriodDays int + Spinner *console.SpinnerWrapper + Now time.Time + Results *[]ForecastWorkflowResult +} - results := make([]ForecastWorkflowResult, 0, len(workflowIDs)) - for _, wfID := range workflowIDs { - if err := ctx.Err(); err != nil { - if !config.Verbose { - spinner.Stop() - } - emitPartialForecastResults(results, config, now) - return normalizeForecastRunError(err, config) +func runForecastWorkflows(p runForecastWorkflowsParams) error { + for _, wfID := range p.WorkflowIDs { + if err := runForecastCheckContext(p.Ctx, p.Config, p.Spinner, *p.Results, p.Now); err != nil { + return err } - if !config.Verbose { - spinner.UpdateMessage(fmt.Sprintf("Sampling %s…", wfID)) + if !p.Config.Verbose { + p.Spinner.UpdateMessage(fmt.Sprintf("Sampling %s…", wfID)) } - - // forecastWorkflow uses the shifted startDate; in eval mode we also pass the - // anchor so the function knows where the training window ends. - result, err := forecastWorkflow(ctx, wfID, startDate, config, periodDays) + result, err := forecastWorkflow(p.Ctx, wfID, p.StartDate, p.Config, p.PeriodDays) if err != nil { - // context.Canceled typically indicates user interruption (Ctrl-C), while - // context.DeadlineExceeded indicates the configured forecast timeout. - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - if !config.Verbose { - spinner.Stop() - } - emitPartialForecastResults(results, config, now) - return normalizeForecastRunError(err, config) - } - if !config.Verbose { - spinner.Stop() - } - fmt.Fprintln(os.Stderr, console.FormatWarningMessage( - fmt.Sprintf("Skipping %s: %v", wfID, err))) - if !config.Verbose { - spinner.Start() + if handledErr := runForecastHandleWorkflowError(err, wfID, p.Config, p.Spinner, *p.Results, p.Now); handledErr != nil { + return handledErr } continue } - - // In eval mode, fetch the validation-window runs and attach evaluation metrics. - if config.EvalMode { - result.Evaluation = evaluateForecast(ctx, wfID, result, validationStartDate, validationEndDate, config) + if p.Config.EvalMode { + result.Evaluation = evaluateForecast(p.Ctx, wfID, result, p.ValidationStartDate, p.ValidationEndDate, p.Config) } + *p.Results = append(*p.Results, result) + } + return nil +} - results = append(results, result) +func runForecastCheckContext(ctx context.Context, config ForecastConfig, spinner *console.SpinnerWrapper, results []ForecastWorkflowResult, now time.Time) error { + if err := ctx.Err(); err != nil { + if !config.Verbose { + spinner.Stop() + } + emitPartialForecastResults(results, config, now) + return normalizeForecastRunError(err, config) } + return nil +} +func runForecastHandleWorkflowError(err error, wfID string, config ForecastConfig, spinner *console.SpinnerWrapper, results []ForecastWorkflowResult, now time.Time) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + if !config.Verbose { + spinner.Stop() + } + emitPartialForecastResults(results, config, now) + return normalizeForecastRunError(err, config) + } if !config.Verbose { spinner.Stop() } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: %v", wfID, err))) + if !config.Verbose { + spinner.Start() + } + return nil +} - // Sort results by Monte Carlo P50 (or point estimate when MC unavailable) descending. +func runForecastSortResults(results []ForecastWorkflowResult) { slices.SortFunc(results, func(a, b ForecastWorkflowResult) int { pi := a.ProjectedAIC if mc := a.MonteCarlo; mc != nil { @@ -171,14 +236,15 @@ func RunForecast(config ForecastConfig) error { } return 0 }) +} +func runForecastRender(results []ForecastWorkflowResult, config ForecastConfig, now time.Time) error { output := ForecastResult{ Period: config.Period, AsOf: now.UTC().Format(time.RFC3339), EvalMode: config.EvalMode, Workflows: results, } - if config.JSONOutput { return renderForecastJSON(output) } diff --git a/pkg/cli/forecast_command.go b/pkg/cli/forecast_command.go index 573c76d07e6..755b65fa5be 100644 --- a/pkg/cli/forecast_command.go +++ b/pkg/cli/forecast_command.go @@ -34,9 +34,22 @@ type ForecastConfig struct { // NewForecastCommand creates the forecast command. func NewForecastCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "forecast [workflow]...", - Short: "[EXPERIMENTAL] Forecast AI Credit (AIC) usage for agentic workflows", - Long: `[EXPERIMENTAL] Forecast AI Credit (AIC) usage for agentic workflows by sampling + Use: "forecast [workflow]...", + Short: "[EXPERIMENTAL] Forecast AI Credit (AIC) usage for agentic workflows", + Long: newForecastCommandLong(), + Example: newForecastCommandExample(), + Args: cobra.ArbitraryArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return newForecastCommandRun(cmd, args) + }, + } + + newForecastCommandRegisterFlags(cmd) + return cmd +} + +func newForecastCommandLong() string { + return `[EXPERIMENTAL] Forecast AI Credit (AIC) usage for agentic workflows by sampling recent run history and projecting forward on a per-week or per-month basis. The forecaster downloads a sample of recent completed workflow runs and derives @@ -63,47 +76,24 @@ Backtesting (--eval): P10–P90 confidence interval. Use this to validate the model before relying on forward projections. -` + WorkflowIDExplanation, - Example: ` ` + string(constants.CLIExtensionPrefix) + ` forecast # Forecast all workflows (monthly) - ` + string(constants.CLIExtensionPrefix) + ` forecast ci-doctor # Forecast a specific workflow - ` + string(constants.CLIExtensionPrefix) + ` forecast ci-doctor daily-news # Compare two workflows - ` + string(constants.CLIExtensionPrefix) + ` forecast --period week # Weekly projections - ` + string(constants.CLIExtensionPrefix) + ` forecast --days 7 # Use 7-day history window - ` + string(constants.CLIExtensionPrefix) + ` forecast --sample 50 # Sample up to 50 runs per workflow - ` + string(constants.CLIExtensionPrefix) + ` forecast --timeout 10 # Stop gracefully after 10 minutes - ` + string(constants.CLIExtensionPrefix) + ` forecast --json # Machine-readable JSON output - ` + string(constants.CLIExtensionPrefix) + ` forecast --repo owner/repo # Forecast in another repository - ` + string(constants.CLIExtensionPrefix) + ` forecast --eval # Backtest: evaluate forecast quality against past data`, - Args: cobra.ArbitraryArgs, - RunE: func(cmd *cobra.Command, args []string) error { - days, _ := cmd.Flags().GetInt("days") - period, _ := cmd.Flags().GetString("period") - jsonOutput, _ := cmd.Flags().GetBool("json") - verbose, _ := cmd.Flags().GetBool("verbose") - repoOverride, _ := cmd.Flags().GetString("repo") - sampleSize, _ := cmd.Flags().GetInt("sample") - evalMode, _ := cmd.Flags().GetBool("eval") - timeoutMinutes, _ := cmd.Flags().GetInt("timeout") - - forecastRunLog.Printf("Forecast command invoked: workflow_count=%d, days=%d, period=%s, sample_size=%d, eval=%v, timeout_minutes=%d, json=%v, repo=%q", - len(args), days, period, sampleSize, evalMode, timeoutMinutes, jsonOutput, repoOverride) - - config := ForecastConfig{ - WorkflowIDs: args, - Days: days, - Period: period, - JSONOutput: jsonOutput, - Verbose: verbose, - RepoOverride: repoOverride, - SampleSize: sampleSize, - EvalMode: evalMode, - TimeoutMinutes: timeoutMinutes, - } - - return RunForecast(config) - }, - } +` + WorkflowIDExplanation +} + +func newForecastCommandExample() string { + prefix := string(constants.CLIExtensionPrefix) + return ` ` + prefix + ` forecast # Forecast all workflows (monthly) + ` + prefix + ` forecast ci-doctor # Forecast a specific workflow + ` + prefix + ` forecast ci-doctor daily-news # Compare two workflows + ` + prefix + ` forecast --period week # Weekly projections + ` + prefix + ` forecast --days 7 # Use 7-day history window + ` + prefix + ` forecast --sample 50 # Sample up to 50 runs per workflow + ` + prefix + ` forecast --timeout 10 # Stop gracefully after 10 minutes + ` + prefix + ` forecast --json # Machine-readable JSON output + ` + prefix + ` forecast --repo owner/repo # Forecast in another repository + ` + prefix + ` forecast --eval # Backtest: evaluate forecast quality against past data` +} +func newForecastCommandRegisterFlags(cmd *cobra.Command) { cmd.Flags().Int("days", 30, "Historical window in days to sample run history (allowed values: 7, 30)") cmd.Flags().String("period", "month", "Aggregation period for projections: week or month") cmd.Flags().Int("sample", 100, "Maximum number of completed runs to sample per workflow") @@ -116,6 +106,34 @@ Backtesting (--eval): _ = cmd.RegisterFlagCompletionFunc("days", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { return []string{"7", "30"}, cobra.ShellCompDirectiveNoFileComp }) +} - return cmd +func newForecastCommandRun(cmd *cobra.Command, args []string) error { + config := newForecastCommandConfig(cmd, args) + forecastRunLog.Printf("Forecast command invoked: workflow_count=%d, days=%d, period=%s, sample_size=%d, eval=%v, timeout_minutes=%d, json=%v, repo=%q", + len(args), config.Days, config.Period, config.SampleSize, config.EvalMode, config.TimeoutMinutes, config.JSONOutput, config.RepoOverride) + return RunForecast(config) +} + +func newForecastCommandConfig(cmd *cobra.Command, args []string) ForecastConfig { + days, _ := cmd.Flags().GetInt("days") + period, _ := cmd.Flags().GetString("period") + jsonOutput, _ := cmd.Flags().GetBool("json") + verbose, _ := cmd.Flags().GetBool("verbose") + repoOverride, _ := cmd.Flags().GetString("repo") + sampleSize, _ := cmd.Flags().GetInt("sample") + evalMode, _ := cmd.Flags().GetBool("eval") + timeoutMinutes, _ := cmd.Flags().GetInt("timeout") + + return ForecastConfig{ + WorkflowIDs: args, + Days: days, + Period: period, + JSONOutput: jsonOutput, + Verbose: verbose, + RepoOverride: repoOverride, + SampleSize: sampleSize, + EvalMode: evalMode, + TimeoutMinutes: timeoutMinutes, + } } diff --git a/pkg/cli/forecast_compute.go b/pkg/cli/forecast_compute.go index 0c17d99f036..dca160718ee 100644 --- a/pkg/cli/forecast_compute.go +++ b/pkg/cli/forecast_compute.go @@ -35,21 +35,58 @@ func forecastWorkflow(ctx context.Context, workflowName, startDate string, confi HistoryDays: config.Days, } + forecastWorkflowLoadMeta(workflowName, config, &result) + + runs, _, err := listRunsWithBackoff(ctx, forecastWorkflowRunOptions(workflowName, startDate, config), result.WorkflowID) + if err != nil { + if gitutil.IsRateLimitError(err.Error()) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage( + fmt.Sprintf("Skipping %s: GitHub API rate limit exceeded", result.WorkflowID))) + return result, nil + } + return result, err + } + + // Only use completed runs for metric computation. + completed := forecastWorkflowCompletedRuns(runs) + if len(completed) == 0 { + forecastRunLog.Printf("No completed runs found for %s in last %d days", workflowName, config.Days) + return result, nil + } + + // Compute per-run averages and collect individual run samples. + totalAIC, totalDurSec, successCount, aicObservations := forecastWorkflowSamples(ctx, workflowName, completed, config, &result) + + n := len(aicObservations) + result.SampledRuns = n + if n == 0 { + forecastRunLog.Printf("No non-zero AIC run samples found for %s in last %d days", workflowName, config.Days) + return result, nil + } + + forecastWorkflowComputeMetrics(&result, totalAIC, totalDurSec, successCount, aicObservations, config, periodDays) + + // Populate experiment variant fractions from run history when metadata has variants. + result.ExperimentVariants = computeVariantFractions(result.ExperimentVariants, completed) + + return result, nil +} + +func forecastWorkflowLoadMeta(workflowName string, config ForecastConfig, result *ForecastWorkflowResult) { // Load frontmatter metadata (triggers, concurrency, experiments). meta := loadWorkflowMeta(workflowName, config.Verbose) result.ActiveTriggers = meta.activeTriggers result.ConcurrencyLimit = meta.concurrencyLimit result.ExperimentVariants = meta.variants result.Engines = meta.engines +} - // Determine the API name used to filter workflow runs (prefer lock file name). +func forecastWorkflowRunOptions(workflowName, startDate string, config ForecastConfig) ListWorkflowRunsOptions { apiName := workflowName if lockFile, err := workflow.GetWorkflowLockFileName(workflowName); err == nil { apiName = lockFile } - - // Fetch completed runs from the history window. - opts := ListWorkflowRunsOptions{ + return ListWorkflowRunsOptions{ WorkflowName: apiName, Status: "completed", StartDate: startDate, @@ -58,18 +95,9 @@ func forecastWorkflow(ctx context.Context, workflowName, startDate string, confi RepoOverride: config.RepoOverride, Verbose: config.Verbose, } +} - runs, _, err := listRunsWithBackoff(ctx, opts, result.WorkflowID) - if err != nil { - if gitutil.IsRateLimitError(err.Error()) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage( - fmt.Sprintf("Skipping %s: GitHub API rate limit exceeded", result.WorkflowID))) - return result, nil - } - return result, err - } - - // Only use completed runs for metric computation. +func forecastWorkflowCompletedRuns(runs []WorkflowRun) []WorkflowRun { completed := make([]WorkflowRun, 0, len(runs)) for _, r := range runs { if isCompletedNonSkippedRun(r) { @@ -81,12 +109,10 @@ func forecastWorkflow(ctx context.Context, workflowName, startDate string, confi completed = append(completed, r) } } - if len(completed) == 0 { - forecastRunLog.Printf("No completed runs found for %s in last %d days", workflowName, config.Days) - return result, nil - } + return completed +} - // Compute per-run averages and collect individual run samples. +func forecastWorkflowSamples(ctx context.Context, workflowName string, completed []WorkflowRun, config ForecastConfig, result *ForecastWorkflowResult) (float64, float64, int, []int) { var totalAIC float64 var totalDurSec float64 successCount := 0 @@ -104,66 +130,66 @@ func forecastWorkflow(ctx context.Context, workflowName, startDate string, confi } totalAIC += runAIC totalDurSec += r.Duration.Seconds() - // Monte Carlo currently samples integer observations; keep milli-AIC precision - // so sub-1 AIC runs are represented without losing granularity. aicObservations = append(aicObservations, int(math.Round(runAIC*1000))) if r.Conclusion == "success" { successCount++ } - sample := ForecastRunSample{RunID: r.DatabaseID, AIC: roundForecastAIC(runAIC)} - if !r.StartedAt.IsZero() { - sample.Date = r.StartedAt.Format("2006-01-02") - } - if r.URL != "" { - sample.RunURL = r.URL - } - samples = append(samples, sample) + samples = append(samples, forecastWorkflowSample(r, runAIC)) } result.RunSamples = samples - if result.WorkflowPath == "" { - for _, r := range completed { - if r.WorkflowPath != "" { - result.WorkflowPath = r.WorkflowPath - break - } - } + forecastWorkflowSetPath(result, completed) + return totalAIC, totalDurSec, successCount, aicObservations +} + +func forecastWorkflowSample(r WorkflowRun, runAIC float64) ForecastRunSample { + sample := ForecastRunSample{RunID: r.DatabaseID, AIC: roundForecastAIC(runAIC)} + if !r.StartedAt.IsZero() { + sample.Date = r.StartedAt.Format("2006-01-02") + } + if r.URL != "" { + sample.RunURL = r.URL } + return sample +} - n := len(aicObservations) - result.SampledRuns = n - if n == 0 { - forecastRunLog.Printf("No non-zero AIC run samples found for %s in last %d days", workflowName, config.Days) - return result, nil +func forecastWorkflowSetPath(result *ForecastWorkflowResult, completed []WorkflowRun) { + if result.WorkflowPath != "" { + return + } + for _, r := range completed { + if r.WorkflowPath != "" { + result.WorkflowPath = r.WorkflowPath + return + } } +} +func forecastWorkflowComputeMetrics(result *ForecastWorkflowResult, totalAIC, totalDurSec float64, successCount int, aicObservations []int, config ForecastConfig, periodDays int) { + n := len(aicObservations) result.AvgAIC = roundForecastAIC(totalAIC / float64(n)) result.AvgDurationSeconds = totalDurSec / float64(n) result.SuccessRate = float64(successCount) / float64(n) + forecastWorkflowPercentiles(result, aicObservations) - // Compute P50 and P95 of individual run AIC (per-run percentiles, not period totals). - sortedAIC := make([]int, len(aicObservations)) - copy(sortedAIC, aicObservations) - sort.Ints(sortedAIC) - result.P50AIC = roundForecastAIC(float64(percentileInt(sortedAIC, 50)) / 1000) - result.P95AIC = roundForecastAIC(float64(percentileInt(sortedAIC, 95)) / 1000) - - // Compute observed run frequency: runs per calendar day over the history window, - // scaled to the projection period. observedRunsPerDay := float64(n) / float64(config.Days) result.ObservedRunsPerPeriod = observedRunsPerDay * float64(periodDays) - - // Point estimates for weekly (7-day) and monthly (30-day) projections. weeklyRuns := observedRunsPerDay * 7 monthlyRuns := observedRunsPerDay * 30 result.WeeklyProjectedAIC = roundForecastAIC(weeklyRuns * result.AvgAIC) result.MonthlyProjectedAIC = roundForecastAIC(monthlyRuns * result.AvgAIC) - - // Projected token usage (point estimate using simple means) for the configured period. result.ProjectedAIC = roundForecastAIC(result.ObservedRunsPerPeriod * result.AvgAIC) + forecastWorkflowMonteCarlo(result, aicObservations, successCount, weeklyRuns, monthlyRuns) +} + +func forecastWorkflowPercentiles(result *ForecastWorkflowResult, aicObservations []int) { + sortedAIC := make([]int, len(aicObservations)) + copy(sortedAIC, aicObservations) + sort.Ints(sortedAIC) + result.P50AIC = roundForecastAIC(float64(percentileInt(sortedAIC, 50)) / 1000) + result.P95AIC = roundForecastAIC(float64(percentileInt(sortedAIC, 95)) / 1000) +} - // Monte Carlo simulation: model run-count (Poisson), per-run token usage - // (bootstrap), and per-run success (Bernoulli) to produce P10/P50/P90 ranges. - // Two independent RNGs ensure the weekly and monthly simulations are uncorrelated. +func forecastWorkflowMonteCarlo(result *ForecastWorkflowResult, aicObservations []int, successCount int, weeklyRuns, monthlyRuns float64) { seed := time.Now().UnixNano() rng := rand.New(rand.NewSource(seed)) //nolint:gosec // non-cryptographic simulation RNG rng2 := rand.New(rand.NewSource(seed + 1)) //nolint:gosec @@ -171,11 +197,6 @@ func forecastWorkflow(ctx context.Context, workflowName, startDate string, confi result.MonteCarlo = runMonteCarlo(aicObservations, successCount, result.ObservedRunsPerPeriod, rng) result.WeeklyMonteCarlo = runMonteCarlo(aicObservations, successCount, weeklyRuns, rng2) result.MonthlyMonteCarlo = runMonteCarlo(aicObservations, successCount, monthlyRuns, rng3) - - // Populate experiment variant fractions from run history when metadata has variants. - result.ExperimentVariants = computeVariantFractions(result.ExperimentVariants, completed) - - return result, nil } // loadCachedRunAIC looks up a locally-cached RunSummary for the given @@ -250,18 +271,10 @@ func forecastDownloadUsageArtifact(ctx context.Context, runID int64, outputDir s shouldLogProgress := IsRunningInCI() || verbose // Check if the requested artifacts are already on disk (cache hit from actions/cache restore). - if fileutil.DirExists(outputDir) && !fileutil.IsDirEmpty(outputDir) { - missing := findMissingFilterEntries(artifactFilter, outputDir) - if len(missing) == 0 { - forecastRunLog.Printf("Usage artifact already on disk for run %d, skipping download", runID) - if shouldLogProgress { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage( - fmt.Sprintf("Usage artifact already present for run %d, skipping download", runID))) - } - return nil - } - forecastRunLog.Printf("Usage artifact partially missing for run %d: %v; downloading missing entries", runID, missing) - artifactFilter = missing + if updatedFilter, done := forecastDownloadUsageArtifactCacheHit(runID, outputDir, artifactFilter, shouldLogProgress); done { + return nil + } else { + artifactFilter = updatedFilter } if err := os.MkdirAll(outputDir, constants.DirPermPublic); err != nil { @@ -269,24 +282,10 @@ func forecastDownloadUsageArtifact(ctx context.Context, runID int64, outputDir s } // List available artifacts for the run to find which match the filter. - artifactNames, listErr := listRunArtifactNames(ctx, runID, owner, repo, hostname, verbose) - if listErr != nil { - forecastRunLog.Printf("Failed to list artifacts for run %d: %v", runID, listErr) - if fileutil.IsDirEmpty(outputDir) { - _ = os.RemoveAll(outputDir) - } - return fmt.Errorf("failed to list artifacts for run %d: %w", runID, listErr) - } - - var downloadableNames []string - for _, name := range artifactNames { - if !isDockerBuildArtifact(name) && artifactMatchesFilter(name, artifactFilter) { - downloadableNames = append(downloadableNames, name) - } + downloadableNames, err := forecastDownloadUsageArtifactNames(ctx, runID, outputDir, verbose, owner, repo, hostname, artifactFilter) + if err != nil { + return err } - - forecastRunLog.Printf("Run %d: listed artifacts=%v, filter=%v, downloadable=%v", runID, artifactNames, artifactFilter, downloadableNames) - if len(downloadableNames) == 0 { // No usage artifact — clean up empty directory and report. if fileutil.IsDirEmpty(outputDir) { @@ -313,6 +312,43 @@ func forecastDownloadUsageArtifact(ctx context.Context, runID int64, outputDir s return nil } +func forecastDownloadUsageArtifactCacheHit(runID int64, outputDir string, artifactFilter []string, shouldLogProgress bool) ([]string, bool) { + if !fileutil.DirExists(outputDir) || fileutil.IsDirEmpty(outputDir) { + return artifactFilter, false + } + missing := findMissingFilterEntries(artifactFilter, outputDir) + if len(missing) == 0 { + forecastRunLog.Printf("Usage artifact already on disk for run %d, skipping download", runID) + if shouldLogProgress { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage( + fmt.Sprintf("Usage artifact already present for run %d, skipping download", runID))) + } + return artifactFilter, true + } + forecastRunLog.Printf("Usage artifact partially missing for run %d: %v; downloading missing entries", runID, missing) + return missing, false +} + +func forecastDownloadUsageArtifactNames(ctx context.Context, runID int64, outputDir string, verbose bool, owner, repo, hostname string, artifactFilter []string) ([]string, error) { + artifactNames, listErr := listRunArtifactNames(ctx, runID, owner, repo, hostname, verbose) + if listErr != nil { + forecastRunLog.Printf("Failed to list artifacts for run %d: %v", runID, listErr) + if fileutil.IsDirEmpty(outputDir) { + _ = os.RemoveAll(outputDir) + } + return nil, fmt.Errorf("failed to list artifacts for run %d: %w", runID, listErr) + } + + var downloadableNames []string + for _, name := range artifactNames { + if !isDockerBuildArtifact(name) && artifactMatchesFilter(name, artifactFilter) { + downloadableNames = append(downloadableNames, name) + } + } + forecastRunLog.Printf("Run %d: listed artifacts=%v, filter=%v, downloadable=%v", runID, artifactNames, artifactFilter, downloadableNames) + return downloadableNames, nil +} + // emitPartialForecastResults outputs whatever workflow results have been collected so // far when the forecast computation is interrupted (timeout or user cancellation). // Partial results are only meaningful when at least one workflow has been fully @@ -372,34 +408,14 @@ func isCompletedNonSkippedRun(r WorkflowRun) bool { func evaluateForecast(ctx context.Context, workflowName string, forecast ForecastWorkflowResult, validationStartDate, validationEndDate string, config ForecastConfig) *ForecastEvaluation { // Compute the actual ISO-8601 training start date by subtracting HistoryDays // from the validation start (= anchor). - var trainingStartDate string - if t, err := time.Parse("2006-01-02", validationStartDate); err == nil { - trainingStartDate = t.AddDate(0, 0, -forecast.HistoryDays).Format("2006-01-02") - } else { - trainingStartDate = validationStartDate - } eval := &ForecastEvaluation{ - TrainingStartDate: trainingStartDate, + TrainingStartDate: evaluateForecastTrainingStart(validationStartDate, forecast.HistoryDays), TrainingEndDate: validationStartDate, ValidationEndDate: validationEndDate, } - // Determine the API name used to filter workflow runs. - apiName := workflowName - if lockFile, err := workflow.GetWorkflowLockFileName(workflowName); err == nil { - apiName = lockFile - } - // Fetch completed runs in the validation window. - opts := ListWorkflowRunsOptions{ - WorkflowName: apiName, - Status: "completed", - StartDate: validationStartDate, - Limit: config.SampleSize, - TargetCount: config.SampleSize, - RepoOverride: config.RepoOverride, - Verbose: config.Verbose, - } + opts := forecastWorkflowRunOptions(workflowName, validationStartDate, config) opts.Context = ctx runs, _, err := listWorkflowRunsWithPagination(opts) if err != nil { @@ -408,14 +424,27 @@ func evaluateForecast(ctx context.Context, workflowName string, forecast Forecas } // Filter to completed runs that fall within the validation window. + evaluateForecastActuals(ctx, eval, runs, validationStartDate, config) + + // Compute error metrics against P50 (falls back to point estimate). + evaluateForecastMetrics(eval, forecast) + return eval +} + +func evaluateForecastTrainingStart(validationStartDate string, historyDays int) string { + if t, err := time.Parse("2006-01-02", validationStartDate); err == nil { + return t.AddDate(0, 0, -historyDays).Format("2006-01-02") + } + return validationStartDate +} + +func evaluateForecastActuals(ctx context.Context, eval *ForecastEvaluation, runs []WorkflowRun, validationStartDate string, config ForecastConfig) { validationEnd := time.Now() validationStart, _ := time.Parse("2006-01-02", validationStartDate) for _, r := range runs { if !isCompletedNonSkippedRun(r) { continue } - // Skip runs with no timestamp — we cannot verify they belong to the - // validation window, so including them would introduce undefined bias. if r.StartedAt.IsZero() { continue } @@ -425,8 +454,9 @@ func evaluateForecast(ctx context.Context, workflowName string, forecast Forecas eval.ActualRuns++ eval.ActualAIC += forecastLoadCachedRunAIC(ctx, r.DatabaseID, config.Verbose) } +} - // Compute error metrics against P50 (falls back to point estimate). +func evaluateForecastMetrics(eval *ForecastEvaluation, forecast ForecastWorkflowResult) { p50 := forecast.ProjectedAIC p10 := forecast.ProjectedAIC p90 := forecast.ProjectedAIC @@ -441,6 +471,4 @@ func evaluateForecast(ctx context.Context, workflowName string, forecast Forecas eval.P50ErrorPct = eval.P50ErrorAbs / p50 * 100 } eval.InCI = eval.ActualAIC >= p10 && eval.ActualAIC <= p90 - - return eval } diff --git a/pkg/cli/forecast_montecarlo.go b/pkg/cli/forecast_montecarlo.go index 1d91a414403..874e778c832 100644 --- a/pkg/cli/forecast_montecarlo.go +++ b/pkg/cli/forecast_montecarlo.go @@ -116,8 +116,17 @@ func runMonteCarlo(aicObservations []int, successCount int, observedRunsPerPerio gammaShape := float64(n) + 0.5 gammaScale := observedRunsPerPeriod / float64(n) - simAICMilli := make([]int, monteCarloIterations) + simAICMilli := runMonteCarloSimulate(aicObservations, successRate, gammaShape, gammaScale, rng) + + // Sort for percentile computation. + sort.Ints(simAICMilli) + return runMonteCarloSummary(n, simAICMilli) +} + +func runMonteCarloSimulate(aicObservations []int, successRate, gammaShape, gammaScale float64, rng *rand.Rand) []int { + n := len(aicObservations) + simAICMilli := make([]int, monteCarloIterations) for i := range monteCarloIterations { // Draw run-count rate from posterior Gamma (accounts for estimation uncertainty in λ). lambdaTrial := gammaSample(rng, gammaShape) * gammaScale @@ -133,13 +142,12 @@ func runMonteCarlo(aicObservations []int, successCount int, observedRunsPerPerio // Bootstrap: sample AIC from the empirical distribution. totalAICMilli += aicObservations[rng.Intn(n)] } - simAICMilli[i] = totalAICMilli } + return simAICMilli +} - // Sort for percentile computation. - sort.Ints(simAICMilli) - +func runMonteCarloSummary(n int, simAICMilli []int) *ForecastMonteCarloSummary { meanMilli, stddevMilli := meanStdDevInt(simAICMilli) mean := roundForecastAIC(float64(meanMilli) / 1000) stddev := roundForecastAIC(stddevMilli / 1000) diff --git a/pkg/cli/forecast_render.go b/pkg/cli/forecast_render.go index 64632c7cd3b..238b171517d 100644 --- a/pkg/cli/forecast_render.go +++ b/pkg/cli/forecast_render.go @@ -34,69 +34,83 @@ func renderForecastTable(output ForecastResult, config ForecastConfig) error { fmt.Fprintln(os.Stderr, "") anyUnreliable := false - var totalWeeklyP50, totalMonthlyP50 float64 - rows := make([]forecastTableRow, 0, len(output.Workflows)+1) - for _, wf := range output.Workflows { - unreliableMark := "" + rows, totalWeeklyP50, totalMonthlyP50, anyUnreliable := renderForecastTableRows(output.Workflows) - weeklyP50 := wf.WeeklyProjectedAIC - if mc := wf.WeeklyMonteCarlo; mc != nil { - weeklyP50 = mc.P50ProjectedAIC - if !mc.IsReliable { - anyUnreliable = true - unreliableMark = "*" - } - } - monthlyP50 := wf.MonthlyProjectedAIC - if mc := wf.MonthlyMonteCarlo; mc != nil { - monthlyP50 = mc.P50ProjectedAIC - } + forecastRenderLog.Printf("Forecast aggregates: total_weekly_p50=%.3f, total_monthly_p50=%.3f, any_unreliable=%v", totalWeeklyP50, totalMonthlyP50, anyUnreliable) + + // Append a totals row when more than one workflow is present. + rows = renderForecastTableAppendTotal(rows, len(output.Workflows), totalWeeklyP50, totalMonthlyP50) + + fmt.Fprint(os.Stderr, console.RenderStruct(rows)) + fmt.Fprintln(os.Stderr, "") + + renderForecastTableDetails(output, anyUnreliable) + return nil +} + +func renderForecastTableRows(workflows []ForecastWorkflowResult) ([]forecastTableRow, float64, float64, bool) { + anyUnreliable := false + var totalWeeklyP50, totalMonthlyP50 float64 + rows := make([]forecastTableRow, 0, len(workflows)+1) + for _, wf := range workflows { + row, weeklyP50, monthlyP50, unreliable := renderForecastTableRow(wf) + anyUnreliable = anyUnreliable || unreliable totalWeeklyP50 += weeklyP50 totalMonthlyP50 += monthlyP50 - - row := forecastTableRow{ - Workflow: wf.WorkflowID + unreliableMark, - Engines: formatEngineList(wf.Engines), - Runs: wf.SampledRuns, - P50PerRun: formatForecastAIC(wf.P50AIC), - P95PerRun: formatForecastAIC(wf.P95AIC), - WeeklyP50: formatForecastAIC(weeklyP50), - MonthlyP50: formatForecastAIC(monthlyP50), - SuccessRate: formatForecastPercent(wf.SuccessRate, wf.SampledRuns > 0), - Triggers: formatTriggerList(wf.ActiveTriggers), - } rows = append(rows, row) } + return rows, totalWeeklyP50, totalMonthlyP50, anyUnreliable +} - forecastRenderLog.Printf("Forecast aggregates: total_weekly_p50=%.3f, total_monthly_p50=%.3f, any_unreliable=%v", totalWeeklyP50, totalMonthlyP50, anyUnreliable) - - // Append a totals row when more than one workflow is present. - if len(output.Workflows) > 1 { - rows = append(rows, forecastTableRow{ - Workflow: "TOTAL", - WeeklyP50: formatForecastAIC(totalWeeklyP50), - MonthlyP50: formatForecastAIC(totalMonthlyP50), - }) +func renderForecastTableRow(wf ForecastWorkflowResult) (forecastTableRow, float64, float64, bool) { + unreliableMark := "" + weeklyP50 := wf.WeeklyProjectedAIC + unreliable := false + if mc := wf.WeeklyMonteCarlo; mc != nil { + weeklyP50 = mc.P50ProjectedAIC + if !mc.IsReliable { + unreliable = true + unreliableMark = "*" + } } + monthlyP50 := wf.MonthlyProjectedAIC + if mc := wf.MonthlyMonteCarlo; mc != nil { + monthlyP50 = mc.P50ProjectedAIC + } + return forecastTableRow{ + Workflow: wf.WorkflowID + unreliableMark, + Engines: formatEngineList(wf.Engines), + Runs: wf.SampledRuns, + P50PerRun: formatForecastAIC(wf.P50AIC), + P95PerRun: formatForecastAIC(wf.P95AIC), + WeeklyP50: formatForecastAIC(weeklyP50), + MonthlyP50: formatForecastAIC(monthlyP50), + SuccessRate: formatForecastPercent(wf.SuccessRate, wf.SampledRuns > 0), + Triggers: formatTriggerList(wf.ActiveTriggers), + }, weeklyP50, monthlyP50, unreliable +} - fmt.Fprint(os.Stderr, console.RenderStruct(rows)) - fmt.Fprintln(os.Stderr, "") +func renderForecastTableAppendTotal(rows []forecastTableRow, workflowCount int, totalWeeklyP50, totalMonthlyP50 float64) []forecastTableRow { + if workflowCount <= 1 { + return rows + } + return append(rows, forecastTableRow{ + Workflow: "TOTAL", + WeeklyP50: formatForecastAIC(totalWeeklyP50), + MonthlyP50: formatForecastAIC(totalMonthlyP50), + }) +} - // Show detailed per-run samples section. +func renderForecastTableDetails(output ForecastResult, anyUnreliable bool) { printRunSamplesSection(output.Workflows) - - // Show experiment variant details when present. for _, wf := range output.Workflows { if len(wf.ExperimentVariants) > 0 { printVariantBreakdown(wf) } } - - // Show backtesting evaluation table in --eval mode. if output.EvalMode { printEvalBreakdown(output.Workflows) } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage( fmt.Sprintf("P50/Run = per-run median AIC; P95/Run = 95th-percentile per-run AIC; Weekly/Monthly = projected P50 from %d-trial Monte Carlo simulation.", monteCarloIterations))) if anyUnreliable { @@ -105,7 +119,6 @@ func renderForecastTable(output ForecastResult, config ForecastConfig) error { } fmt.Fprintln(os.Stderr, console.FormatInfoMessage( fmt.Sprintf("Run '%s forecast --json' for full Monte Carlo output including P10/P90 confidence intervals.", string(constants.CLIExtensionPrefix)))) - return nil } // printRunSamplesSection prints a detailed table of the sampled runs used in the forecast, diff --git a/pkg/cli/frontmatter_editor.go b/pkg/cli/frontmatter_editor.go index ce9bd131506..e5177fbe602 100644 --- a/pkg/cli/frontmatter_editor.go +++ b/pkg/cli/frontmatter_editor.go @@ -28,81 +28,72 @@ func UpdateFieldInFrontmatter(content, fieldName, fieldValue string) (string, er // Try to preserve original frontmatter formatting by manually updating the field if len(result.FrontmatterLines) > 0 { - frontmatterEditorLog.Printf("Using raw frontmatter lines for field update (%d lines)", len(result.FrontmatterLines)) - // Look for existing field in the raw lines - fieldUpdated := false - skipChildren := false - fieldIndentLevel := 0 - newFrontmatterLines := make([]string, 0, len(result.FrontmatterLines)) - - for _, line := range result.FrontmatterLines { - trimmedLine := strings.TrimSpace(line) - - // If we just updated the field, skip its child lines (block mapping values) - if skipChildren { - currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) - if currentIndent > fieldIndentLevel { - // This line is a child of the replaced field — drop it - continue - } - // No longer in the child block - skipChildren = false - } + return updateFieldInFrontmatterRaw(result, fieldName, fieldValue), nil + } - // Check if this line contains our field - if !fieldUpdated && strings.HasPrefix(trimmedLine, fieldName+":") { - // Preserve the original indentation and comments - leadingSpace := line[:len(line)-len(strings.TrimLeft(line, " \t"))] - - // Check if there's a comment on the same line - commentIndex := strings.Index(line, "#") - var comment string - if commentIndex > strings.Index(line, ":") && commentIndex != -1 { - comment = line[commentIndex:] - } - - // Update the field value while preserving formatting - var updatedLine string - if comment != "" { - updatedLine = fmt.Sprintf("%s%s: %s %s", leadingSpace, fieldName, fieldValue, comment) - } else { - updatedLine = fmt.Sprintf("%s%s: %s", leadingSpace, fieldName, fieldValue) - } - newFrontmatterLines = append(newFrontmatterLines, updatedLine) - fieldUpdated = true - // Track the indent level so we can skip any child lines that follow - fieldIndentLevel = len(leadingSpace) - skipChildren = true - frontmatterEditorLog.Printf("Updated existing field %s", fieldName) + // Fallback to marshal-based approach if no raw lines are available + return updateFieldInFrontmatterFallback(result, fieldName, fieldValue) +} + +func updateFieldInFrontmatterRaw(result *parser.FrontmatterResult, fieldName, fieldValue string) string { + frontmatterEditorLog.Printf("Using raw frontmatter lines for field update (%d lines)", len(result.FrontmatterLines)) + newFrontmatterLines := make([]string, 0, len(result.FrontmatterLines)) + fieldUpdated := false + skipChildren := false + fieldIndentLevel := 0 + + for _, line := range result.FrontmatterLines { + if skipChildren { + currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) + if currentIndent > fieldIndentLevel { continue } - - newFrontmatterLines = append(newFrontmatterLines, line) + skipChildren = false } - - // If field wasn't found in the raw lines, add it at the end - if !fieldUpdated { - newField := fmt.Sprintf("%s: %s", fieldName, fieldValue) - newFrontmatterLines = append(newFrontmatterLines, newField) - frontmatterEditorLog.Printf("Added new field %s at end of frontmatter", fieldName) + trimmedLine := strings.TrimSpace(line) + if !fieldUpdated && strings.HasPrefix(trimmedLine, fieldName+":") { + updatedLine, indent := updateFieldInFrontmatterLine(line, fieldName, fieldValue) + newFrontmatterLines = append(newFrontmatterLines, updatedLine) + fieldUpdated = true + fieldIndentLevel = indent + skipChildren = true + frontmatterEditorLog.Printf("Updated existing field %s", fieldName) + continue } + newFrontmatterLines = append(newFrontmatterLines, line) + } - // Reconstruct the file with preserved formatting - var lines []string - lines = append(lines, "---") - lines = append(lines, newFrontmatterLines...) - lines = append(lines, "---") - if result.Markdown != "" { - // Add empty line before markdown content to match original format - lines = append(lines, "") - lines = append(lines, result.Markdown) - } + if !fieldUpdated { + newFrontmatterLines = append(newFrontmatterLines, fmt.Sprintf("%s: %s", fieldName, fieldValue)) + frontmatterEditorLog.Printf("Added new field %s at end of frontmatter", fieldName) + } + return updateFieldInFrontmatterReconstruct(newFrontmatterLines, result.Markdown) +} - return strings.Join(lines, "\n"), nil +func updateFieldInFrontmatterLine(line, fieldName, fieldValue string) (string, int) { + leadingSpace := line[:len(line)-len(strings.TrimLeft(line, " \t"))] + commentIndex := strings.Index(line, "#") + var comment string + if commentIndex > strings.Index(line, ":") && commentIndex != -1 { + comment = line[commentIndex:] + } + if comment != "" { + return fmt.Sprintf("%s%s: %s %s", leadingSpace, fieldName, fieldValue, comment), len(leadingSpace) } + return fmt.Sprintf("%s%s: %s", leadingSpace, fieldName, fieldValue), len(leadingSpace) +} - // Fallback to marshal-based approach if no raw lines are available - return updateFieldInFrontmatterFallback(result, fieldName, fieldValue) +func updateFieldInFrontmatterReconstruct(frontmatterLines []string, markdown string) string { + var lines []string + lines = append(lines, "---") + lines = append(lines, frontmatterLines...) + lines = append(lines, "---") + if markdown != "" { + // Add empty line before markdown content to match original format + lines = append(lines, "") + lines = append(lines, markdown) + } + return strings.Join(lines, "\n") } // updateFieldInFrontmatterFallback implements the original behavior as a fallback @@ -229,85 +220,80 @@ func RemoveFieldFromOnTrigger(content, fieldName string) (string, error) { // Work with raw frontmatter lines to preserve formatting if len(result.FrontmatterLines) > 0 { - frontmatterEditorLog.Printf("Using raw frontmatter lines to remove field (%d lines)", len(result.FrontmatterLines)) - - frontmatterLines := make([]string, 0, len(result.FrontmatterLines)) - inOnBlock := false - onIndentLevel := 0 - skipNextLine := false - fieldIndentLevel := 0 - - for i := range len(result.FrontmatterLines) { - line := result.FrontmatterLines[i] - trimmedLine := strings.TrimSpace(line) - - // Skip if this is a continuation line that should be skipped - if skipNextLine { - // Check if this line is indented more than the field we're removing - // If so, skip it (it's a continuation of the removed field) - currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) - if currentIndent > fieldIndentLevel { - continue - } - skipNextLine = false - } + frontmatterLines := removeFieldFromOnTriggerRaw(result.FrontmatterLines, fieldName) + frontmatterEditorLog.Printf("Successfully removed field %s from 'on' trigger", fieldName) + return removeFieldFromOnTriggerReconstruct(frontmatterLines, result.Markdown), nil + } + + // This should rarely happen since we already checked for frontmatter existence + frontmatterEditorLog.Printf("No raw frontmatter lines available, returning content unchanged") + return content, nil +} - // Detect the start of the 'on:' block (must be just "on:" without inline value) - if !inOnBlock && (trimmedLine == "on:" || trimmedLine == `"on":` || - strings.HasPrefix(trimmedLine, "on: #") || strings.HasPrefix(trimmedLine, `"on": #`)) { - inOnBlock = true - onIndentLevel = len(line) - len(strings.TrimLeft(line, " \t")) +func removeFieldFromOnTriggerRaw(lines []string, fieldName string) []string { + frontmatterEditorLog.Printf("Using raw frontmatter lines to remove field (%d lines)", len(lines)) + frontmatterLines := make([]string, 0, len(lines)) + inOnBlock := false + onIndentLevel := 0 + skipNextLine := false + fieldIndentLevel := 0 + for i := range len(lines) { + line := lines[i] + trimmedLine := strings.TrimSpace(line) + if skipNextLine { + currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) + if currentIndent > fieldIndentLevel { + continue + } + skipNextLine = false + } + if !inOnBlock && removeFieldFromOnTriggerIsOnLine(trimmedLine) { + inOnBlock = true + onIndentLevel = len(line) - len(strings.TrimLeft(line, " \t")) + frontmatterLines = append(frontmatterLines, line) + continue + } + if inOnBlock { + currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) + if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") && currentIndent <= onIndentLevel { + inOnBlock = false frontmatterLines = append(frontmatterLines, line) continue } - - // If we're in the 'on' block, check if this is the field to remove - if inOnBlock { - currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) - - // Check if we've exited the 'on' block - if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") && currentIndent <= onIndentLevel { - inOnBlock = false - frontmatterLines = append(frontmatterLines, line) - continue - } - - // Check if this is the field to remove (exact match) - if trimmedLine == fieldName+":" || - strings.HasPrefix(trimmedLine, fieldName+": ") || - strings.HasPrefix(trimmedLine, fieldName+":\t") { - frontmatterEditorLog.Printf("Found field %s to remove at line %d", fieldName, i+1) - // Track the indentation of the field being removed - fieldIndentLevel = currentIndent - // Skip this line - // Also mark that we should skip continuation lines if the value is multiline - skipNextLine = true - continue - } + if removeFieldFromOnTriggerIsField(trimmedLine, fieldName) { + frontmatterEditorLog.Printf("Found field %s to remove at line %d", fieldName, i+1) + fieldIndentLevel = currentIndent + skipNextLine = true + continue } - - // Keep this line - frontmatterLines = append(frontmatterLines, line) } + frontmatterLines = append(frontmatterLines, line) + } + return frontmatterLines +} - // Reconstruct the file with preserved formatting - var lines []string - lines = append(lines, "---") - lines = append(lines, frontmatterLines...) - lines = append(lines, "---") - if result.Markdown != "" { - // Add empty line before markdown content to match original format - lines = append(lines, "") - lines = append(lines, result.Markdown) - } +func removeFieldFromOnTriggerIsOnLine(trimmedLine string) bool { + return trimmedLine == "on:" || trimmedLine == `"on":` || + strings.HasPrefix(trimmedLine, "on: #") || strings.HasPrefix(trimmedLine, `"on": #`) +} - frontmatterEditorLog.Printf("Successfully removed field %s from 'on' trigger", fieldName) - return strings.Join(lines, "\n"), nil - } +func removeFieldFromOnTriggerIsField(trimmedLine, fieldName string) bool { + return trimmedLine == fieldName+":" || + strings.HasPrefix(trimmedLine, fieldName+": ") || + strings.HasPrefix(trimmedLine, fieldName+":\t") +} - // This should rarely happen since we already checked for frontmatter existence - frontmatterEditorLog.Printf("No raw frontmatter lines available, returning content unchanged") - return content, nil +func removeFieldFromOnTriggerReconstruct(frontmatterLines []string, markdown string) string { + var lines []string + lines = append(lines, "---") + lines = append(lines, frontmatterLines...) + lines = append(lines, "---") + if markdown != "" { + // Add empty line before markdown content to match original format + lines = append(lines, "") + lines = append(lines, markdown) + } + return strings.Join(lines, "\n") } // SetFieldInOnTrigger sets a field value in the 'on' trigger object in the frontmatter. @@ -335,26 +321,7 @@ func SetFieldInOnTrigger(content, fieldName, fieldValue string) (string, error) // No 'on' field exists, need to create it // Add the 'on:' block with the field at the beginning of frontmatter if len(result.FrontmatterLines) > 0 { - frontmatterEditorLog.Printf("Creating 'on' block with field %s", fieldName) - - // Create new frontmatter lines with 'on:' block at the start - frontmatterLines := make([]string, 0, len(result.FrontmatterLines)+2) - frontmatterLines = append(frontmatterLines, "on:") - frontmatterLines = append(frontmatterLines, fmt.Sprintf(" %s: %s", fieldName, fieldValue)) - frontmatterLines = append(frontmatterLines, result.FrontmatterLines...) - - // Reconstruct the file - var lines []string - lines = append(lines, "---") - lines = append(lines, frontmatterLines...) - lines = append(lines, "---") - if result.Markdown != "" { - lines = append(lines, "") - lines = append(lines, result.Markdown) - } - - frontmatterEditorLog.Printf("Successfully created 'on' block with field %s", fieldName) - return strings.Join(lines, "\n"), nil + return setFieldInOnTriggerCreateBlock(result, fieldName, fieldValue), nil } // No frontmatter lines, cannot create 'on' block @@ -370,107 +337,13 @@ func SetFieldInOnTrigger(content, fieldName, fieldValue string) (string, error) // Work with raw frontmatter lines to preserve formatting if len(result.FrontmatterLines) > 0 { - frontmatterEditorLog.Printf("Using raw frontmatter lines to set field (%d lines)", len(result.FrontmatterLines)) - - frontmatterLines := make([]string, 0, len(result.FrontmatterLines)) - inOnBlock := false - onIndentLevel := 0 - fieldUpdated := false - - for i := range len(result.FrontmatterLines) { - line := result.FrontmatterLines[i] - trimmedLine := strings.TrimSpace(line) - - // Detect the start of the 'on:' block (must be just "on:" without inline value) - if !inOnBlock && (trimmedLine == "on:" || trimmedLine == `"on":` || - strings.HasPrefix(trimmedLine, "on: #") || strings.HasPrefix(trimmedLine, `"on": #`)) { - inOnBlock = true - onIndentLevel = len(line) - len(strings.TrimLeft(line, " \t")) - frontmatterLines = append(frontmatterLines, line) - continue - } - - // If we're in the 'on' block - if inOnBlock { - currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) - - // Check if we've exited the 'on' block - if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") && currentIndent <= onIndentLevel { - inOnBlock = false - - // If we didn't update the field yet, add it before exiting the block - if !fieldUpdated { - // Calculate the appropriate indentation (one level deeper than 'on:') - indent := strings.Repeat(" ", onIndentLevel+4) - newField := fmt.Sprintf("%s%s: %s", indent, fieldName, fieldValue) - frontmatterLines = append(frontmatterLines, newField) - fieldUpdated = true - frontmatterEditorLog.Printf("Added new field %s to 'on' block", fieldName) - } - - frontmatterLines = append(frontmatterLines, line) - continue - } - - // Check if this is the field to update (exact match) - if trimmedLine == fieldName+":" || - strings.HasPrefix(trimmedLine, fieldName+": ") || - strings.HasPrefix(trimmedLine, fieldName+":\t") { - // Preserve the original indentation and comments - leadingSpace := line[:len(line)-len(strings.TrimLeft(line, " \t"))] - - // Check if there's a comment on the same line, after the field separator - fieldSep := fieldName + ":" - fieldSepIndex := strings.Index(line, fieldSep) - commentIndex := strings.Index(line, "#") - var comment string - if fieldSepIndex != -1 && commentIndex > fieldSepIndex { - comment = line[commentIndex:] - } - - // Update the field value while preserving formatting - if comment != "" { - frontmatterLines = append(frontmatterLines, fmt.Sprintf("%s%s: %s %s", leadingSpace, fieldName, fieldValue, comment)) - } else { - frontmatterLines = append(frontmatterLines, fmt.Sprintf("%s%s: %s", leadingSpace, fieldName, fieldValue)) - } - fieldUpdated = true - frontmatterEditorLog.Printf("Updated existing field %s in 'on' block", fieldName) - continue - } - } - - // Keep this line - frontmatterLines = append(frontmatterLines, line) - } - - // If we were still in the 'on' block at the end of the frontmatter and didn't update the field - if inOnBlock && !fieldUpdated { - // Add the field at the end of the 'on' block - indent := strings.Repeat(" ", onIndentLevel+4) - newField := fmt.Sprintf("%s%s: %s", indent, fieldName, fieldValue) - frontmatterLines = append(frontmatterLines, newField) - fieldUpdated = true - frontmatterEditorLog.Printf("Added new field %s at end of 'on' block", fieldName) - } - + frontmatterLines, fieldUpdated := setFieldInOnTriggerRaw(result.FrontmatterLines, fieldName, fieldValue) if !fieldUpdated { return "", fmt.Errorf("failed to set field %s in 'on' trigger", fieldName) } - // Reconstruct the file with preserved formatting - var lines []string - lines = append(lines, "---") - lines = append(lines, frontmatterLines...) - lines = append(lines, "---") - if result.Markdown != "" { - // Add empty line before markdown content to match original format - lines = append(lines, "") - lines = append(lines, result.Markdown) - } - frontmatterEditorLog.Printf("Successfully set field %s in 'on' trigger", fieldName) - return strings.Join(lines, "\n"), nil + return setFieldInOnTriggerReconstruct(frontmatterLines, result.Markdown), nil } // This should rarely happen since we already checked for frontmatter existence @@ -478,6 +351,96 @@ func SetFieldInOnTrigger(content, fieldName, fieldValue string) (string, error) return "", errors.New("no frontmatter lines available to modify") } +func setFieldInOnTriggerCreateBlock(result *parser.FrontmatterResult, fieldName, fieldValue string) string { + frontmatterEditorLog.Printf("Creating 'on' block with field %s", fieldName) + frontmatterLines := make([]string, 0, len(result.FrontmatterLines)+2) + frontmatterLines = append(frontmatterLines, "on:") + frontmatterLines = append(frontmatterLines, fmt.Sprintf(" %s: %s", fieldName, fieldValue)) + frontmatterLines = append(frontmatterLines, result.FrontmatterLines...) + frontmatterEditorLog.Printf("Successfully created 'on' block with field %s", fieldName) + return setFieldInOnTriggerReconstruct(frontmatterLines, result.Markdown) +} + +func setFieldInOnTriggerRaw(lines []string, fieldName, fieldValue string) ([]string, bool) { + frontmatterEditorLog.Printf("Using raw frontmatter lines to set field (%d lines)", len(lines)) + frontmatterLines := make([]string, 0, len(lines)) + inOnBlock := false + onIndentLevel := 0 + fieldUpdated := false + for i := range len(lines) { + line := lines[i] + trimmedLine := strings.TrimSpace(line) + if !inOnBlock && removeFieldFromOnTriggerIsOnLine(trimmedLine) { + inOnBlock = true + onIndentLevel = len(line) - len(strings.TrimLeft(line, " \t")) + frontmatterLines = append(frontmatterLines, line) + continue + } + if inOnBlock { + currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) + if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") && currentIndent <= onIndentLevel { + inOnBlock = false + frontmatterLines, fieldUpdated = setFieldInOnTriggerAppendNew(frontmatterLines, fieldUpdated, onIndentLevel, fieldName, fieldValue, false) + frontmatterLines = append(frontmatterLines, line) + continue + } + if removeFieldFromOnTriggerIsField(trimmedLine, fieldName) { + frontmatterLines = append(frontmatterLines, setFieldInOnTriggerUpdatedLine(line, fieldName, fieldValue)) + fieldUpdated = true + frontmatterEditorLog.Printf("Updated existing field %s in 'on' block", fieldName) + continue + } + } + frontmatterLines = append(frontmatterLines, line) + } + if inOnBlock && !fieldUpdated { + frontmatterLines, fieldUpdated = setFieldInOnTriggerAppendNew(frontmatterLines, fieldUpdated, onIndentLevel, fieldName, fieldValue, true) + } + return frontmatterLines, fieldUpdated +} + +func setFieldInOnTriggerAppendNew(lines []string, fieldUpdated bool, onIndentLevel int, fieldName, fieldValue string, atEnd bool) ([]string, bool) { + if fieldUpdated { + return lines, fieldUpdated + } + indent := strings.Repeat(" ", onIndentLevel+4) + lines = append(lines, fmt.Sprintf("%s%s: %s", indent, fieldName, fieldValue)) + if atEnd { + frontmatterEditorLog.Printf("Added new field %s at end of 'on' block", fieldName) + } else { + frontmatterEditorLog.Printf("Added new field %s to 'on' block", fieldName) + } + return lines, true +} + +func setFieldInOnTriggerUpdatedLine(line, fieldName, fieldValue string) string { + leadingSpace := line[:len(line)-len(strings.TrimLeft(line, " \t"))] + fieldSep := fieldName + ":" + fieldSepIndex := strings.Index(line, fieldSep) + commentIndex := strings.Index(line, "#") + var comment string + if fieldSepIndex != -1 && commentIndex > fieldSepIndex { + comment = line[commentIndex:] + } + if comment != "" { + return fmt.Sprintf("%s%s: %s %s", leadingSpace, fieldName, fieldValue, comment) + } + return fmt.Sprintf("%s%s: %s", leadingSpace, fieldName, fieldValue) +} + +func setFieldInOnTriggerReconstruct(frontmatterLines []string, markdown string) string { + var lines []string + lines = append(lines, "---") + lines = append(lines, frontmatterLines...) + lines = append(lines, "---") + if markdown != "" { + // Add empty line before markdown content to match original format + lines = append(lines, "") + lines = append(lines, markdown) + } + return strings.Join(lines, "\n") +} + // UpdateScheduleInOnBlock updates the "schedule" sub-key inside the "on:" block mapping in // the workflow frontmatter. It replaces the existing schedule value—whether a scalar // (schedule: daily) or a list (schedule:\n - cron: "0 9 * * *")—with a new scalar @@ -494,75 +457,68 @@ func UpdateScheduleInOnBlock(content, scheduleExpr string) (string, error) { return "", errors.New("no frontmatter lines available to modify") } - frontmatterLines := make([]string, 0, len(result.FrontmatterLines)) + frontmatterLines, scheduleFound := updateScheduleInOnBlockRaw(result.FrontmatterLines, scheduleExpr) + if !scheduleFound { + return "", errors.New("schedule key not found inside on: block") + } + + return updateScheduleInOnBlockReconstruct(frontmatterLines, result.Markdown), nil +} + +func updateScheduleInOnBlockRaw(lines []string, scheduleExpr string) ([]string, bool) { + frontmatterLines := make([]string, 0, len(lines)) inOnBlock := false onIndentLevel := 0 scheduleFound := false skipScheduleChildren := false scheduleIndentLevel := 0 - - for _, line := range result.FrontmatterLines { + for _, line := range lines { trimmedLine := strings.TrimSpace(line) currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) - - // Detect the start of the 'on:' block (bare "on:" key, with optional trailing comment). - if !inOnBlock && - (trimmedLine == "on:" || trimmedLine == `"on":` || - strings.HasPrefix(trimmedLine, "on: #") || strings.HasPrefix(trimmedLine, `"on": #`)) { + if !inOnBlock && removeFieldFromOnTriggerIsOnLine(trimmedLine) { inOnBlock = true onIndentLevel = currentIndent frontmatterLines = append(frontmatterLines, line) continue } - if inOnBlock { - // Drop child lines of the schedule: key (e.g., "- cron: daily"). - if skipScheduleChildren { - if currentIndent > scheduleIndentLevel { - continue - } - skipScheduleChildren = false + if skipScheduleChildren && currentIndent > scheduleIndentLevel { + continue } - - // Exit the on: block when a non-empty, non-comment line appears at or above on: indent. + skipScheduleChildren = false if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") && currentIndent <= onIndentLevel { inOnBlock = false frontmatterLines = append(frontmatterLines, line) continue } - - // Replace the schedule: line (handles both scalar and bare-key forms). - if !scheduleFound && - (trimmedLine == "schedule:" || - strings.HasPrefix(trimmedLine, "schedule: ") || - strings.HasPrefix(trimmedLine, "schedule:\t")) { - leadingSpace := line[:currentIndent] - frontmatterLines = append(frontmatterLines, fmt.Sprintf("%sschedule: %s", leadingSpace, scheduleExpr)) + if !scheduleFound && updateScheduleInOnBlockIsSchedule(trimmedLine) { + frontmatterLines = append(frontmatterLines, fmt.Sprintf("%sschedule: %s", line[:currentIndent], scheduleExpr)) scheduleFound = true scheduleIndentLevel = currentIndent skipScheduleChildren = true frontmatterEditorLog.Printf("Updated schedule in on: block to %q", scheduleExpr) continue } - - frontmatterLines = append(frontmatterLines, line) - continue } - frontmatterLines = append(frontmatterLines, line) } + return frontmatterLines, scheduleFound +} - if !scheduleFound { - return "", errors.New("schedule key not found inside on: block") - } +func updateScheduleInOnBlockIsSchedule(trimmedLine string) bool { + return trimmedLine == "schedule:" || + strings.HasPrefix(trimmedLine, "schedule: ") || + strings.HasPrefix(trimmedLine, "schedule:\t") +} +func updateScheduleInOnBlockReconstruct(frontmatterLines []string, markdown string) string { var lines []string lines = append(lines, "---") lines = append(lines, frontmatterLines...) lines = append(lines, "---") - if result.Markdown != "" { + if markdown != "" { lines = append(lines, "") - lines = append(lines, result.Markdown) + lines = append(lines, markdown) } - return strings.Join(lines, "\n"), nil + return strings.Join(lines, "\n") } diff --git a/pkg/cli/gateway_logs_mcp.go b/pkg/cli/gateway_logs_mcp.go index 9b4fcb28165..aa7f1297387 100644 --- a/pkg/cli/gateway_logs_mcp.go +++ b/pkg/cli/gateway_logs_mcp.go @@ -51,54 +51,60 @@ func extractMCPToolUsageData(logDir string, verbose bool) (*MCPToolUsageData, er mcpData.GuardPolicySummary = buildGuardPolicySummary(gatewayMetrics) } + gatewayLogPath, usingRPCMessages, err := extractMCPToolUsageDataLogPath(logDir) + if err != nil { + return nil, err + } + + if err := extractMCPToolUsageDataToolCalls(logDir, gatewayLogPath, usingRPCMessages, mcpData); err != nil { + return nil, err + } + + // Build summary statistics from aggregated metrics + buildMCPSummaryStats(gatewayMetrics, mcpData) + gatewayLogsLog.Printf("Built MCP summary: %d tool summaries, %d server stats", len(mcpData.Summary), len(mcpData.Servers)) + + return mcpData, nil +} + +func extractMCPToolUsageDataLogPath(logDir string) (string, bool, error) { // Read the log file again to get individual tool call records. // Prefer gateway.jsonl; fall back to rpc-messages.jsonl when not available. gatewayLogPath := filepath.Join(logDir, "gateway.jsonl") - usingRPCMessages := false - - if _, err := os.Stat(gatewayLogPath); os.IsNotExist(err) { - mcpLogsPath := filepath.Join(logDir, "mcp-logs", "gateway.jsonl") - if _, err := os.Stat(mcpLogsPath); os.IsNotExist(err) { - // Fall back to rpc-messages.jsonl - rpcPath := findRPCMessagesPath(logDir) - if rpcPath == "" { - return nil, errors.New("gateway.jsonl not found") - } - gatewayLogPath = rpcPath - usingRPCMessages = true - } else { - gatewayLogPath = mcpLogsPath - } + if _, err := os.Stat(gatewayLogPath); err == nil { + return gatewayLogPath, false, nil + } + mcpLogsPath := filepath.Join(logDir, "mcp-logs", "gateway.jsonl") + if _, err := os.Stat(mcpLogsPath); err == nil { + return mcpLogsPath, false, nil } + rpcPath := findRPCMessagesPath(logDir) + if rpcPath == "" { + return "", false, errors.New("gateway.jsonl not found") + } + return rpcPath, true, nil +} +func extractMCPToolUsageDataToolCalls(logDir, gatewayLogPath string, usingRPCMessages bool, mcpData *MCPToolUsageData) error { if usingRPCMessages { gatewayLogsLog.Printf("Reading tool calls from rpc-messages.jsonl: %s", gatewayLogPath) - // Build tool call records from rpc-messages.jsonl toolCalls, err := buildToolCallsFromRPCMessages(gatewayLogPath) if err != nil { - return nil, fmt.Errorf("failed to read rpc-messages.jsonl: %w", err) + return fmt.Errorf("failed to read rpc-messages.jsonl: %w", err) } - // Correlate tool calls with effective-token deltas from token-usage.jsonl tokenUsageFile := findTokenUsageFile(logDir) - toolCalls = correlateToolCallsWithTokenDelta(toolCalls, tokenUsageFile) - mcpData.ToolCalls = toolCalls - gatewayLogsLog.Printf("Loaded %d tool calls from rpc-messages.jsonl", len(toolCalls)) - } else { - gatewayLogsLog.Printf("Reading tool calls from gateway.jsonl: %s", gatewayLogPath) - if err := extractToolCallsFromGatewayLog(gatewayLogPath, mcpData); err != nil { - return nil, err - } - // Correlate tool calls with effective-token deltas from token-usage.jsonl - tokenUsageFile := findTokenUsageFile(logDir) - mcpData.ToolCalls = correlateToolCallsWithTokenDelta(mcpData.ToolCalls, tokenUsageFile) - gatewayLogsLog.Printf("Loaded %d tool calls from gateway.jsonl", len(mcpData.ToolCalls)) + mcpData.ToolCalls = correlateToolCallsWithTokenDelta(toolCalls, tokenUsageFile) + gatewayLogsLog.Printf("Loaded %d tool calls from rpc-messages.jsonl", len(mcpData.ToolCalls)) + return nil } - - // Build summary statistics from aggregated metrics - buildMCPSummaryStats(gatewayMetrics, mcpData) - gatewayLogsLog.Printf("Built MCP summary: %d tool summaries, %d server stats", len(mcpData.Summary), len(mcpData.Servers)) - - return mcpData, nil + gatewayLogsLog.Printf("Reading tool calls from gateway.jsonl: %s", gatewayLogPath) + if err := extractToolCallsFromGatewayLog(gatewayLogPath, mcpData); err != nil { + return err + } + tokenUsageFile := findTokenUsageFile(logDir) + mcpData.ToolCalls = correlateToolCallsWithTokenDelta(mcpData.ToolCalls, tokenUsageFile) + gatewayLogsLog.Printf("Loaded %d tool calls from gateway.jsonl", len(mcpData.ToolCalls)) + return nil } // extractToolCallsFromGatewayLog reads gateway.jsonl and appends tool call records to mcpData. @@ -123,46 +129,7 @@ func extractToolCallsFromGatewayLog(gatewayLogPath string, mcpData *MCPToolUsage continue // Skip malformed lines } - // Only process tool call events - if entry.Event == "tool_call" || entry.Event == "rpc_call" || entry.Event == "request" { - toolName := entry.ToolName - if toolName == "" { - toolName = entry.Method - } - - // Skip entries without tool information - if entry.ServerName == "" || toolName == "" { - continue - } - - // Derive status from available fields when not explicitly set. - // Post-OTel-collector migrations may omit the "status" string field, - // relying instead on "error" or "level" to signal failures. - status := entry.Status - if status == "" { - if entry.Error != "" || entry.Level == "error" { - status = "error" - } else { - status = "success" - } - } - - // Create individual tool call record - toolCall := MCPToolCall{ - Timestamp: entry.Timestamp, - ServerName: entry.ServerName, - ToolName: toolName, - Method: entry.Method, - InputSize: entry.InputSize, - OutputSize: entry.OutputSize, - Status: status, - Error: entry.Error, - } - - if entry.Duration > 0 { - toolCall.Duration = timeutil.FormatDuration(time.Duration(entry.Duration * float64(time.Millisecond))) - } - + if toolCall, ok := extractToolCallsFromGatewayLogEntry(entry); ok { mcpData.ToolCalls = append(mcpData.ToolCalls, toolCall) } } @@ -173,67 +140,100 @@ func extractToolCallsFromGatewayLog(gatewayLogPath string, mcpData *MCPToolUsage return nil } -// buildMCPSummaryStats populates mcpData.Summary and mcpData.Servers from aggregated gateway metrics. -func buildMCPSummaryStats(gatewayMetrics *GatewayMetrics, mcpData *MCPToolUsageData) { - for serverName, serverMetrics := range gatewayMetrics.Servers { - // Server-level stats - serverStats := MCPServerStats{ - ServerName: serverName, - RequestCount: serverMetrics.RequestCount, - ToolCallCount: serverMetrics.ToolCallCount, - TotalInputSize: 0, - TotalOutputSize: 0, - ErrorCount: serverMetrics.ErrorCount, - } +func extractToolCallsFromGatewayLogEntry(entry GatewayLogEntry) (MCPToolCall, bool) { + // Only process tool call events + if entry.Event != "tool_call" && entry.Event != "rpc_call" && entry.Event != "request" { + return MCPToolCall{}, false + } + toolName := entry.ToolName + if toolName == "" { + toolName = entry.Method + } + if entry.ServerName == "" || toolName == "" { + return MCPToolCall{}, false + } - if serverMetrics.RequestCount > 0 { - avgDur := serverMetrics.TotalDuration / float64(serverMetrics.RequestCount) - serverStats.AvgDuration = timeutil.FormatDuration(time.Duration(avgDur * float64(time.Millisecond))) - } + toolCall := MCPToolCall{ + Timestamp: entry.Timestamp, + ServerName: entry.ServerName, + ToolName: toolName, + Method: entry.Method, + InputSize: entry.InputSize, + OutputSize: entry.OutputSize, + Status: extractToolCallsFromGatewayLogStatus(entry), + Error: entry.Error, + } + if entry.Duration > 0 { + toolCall.Duration = timeutil.FormatDuration(time.Duration(entry.Duration * float64(time.Millisecond))) + } + return toolCall, true +} - // Tool-level stats - for toolName, toolMetrics := range serverMetrics.Tools { - summary := MCPToolSummary{ - ServerName: serverName, - ToolName: toolName, - CallCount: toolMetrics.CallCount, - TotalInputSize: toolMetrics.TotalInputSize, - TotalOutputSize: toolMetrics.TotalOutputSize, - MaxInputSize: 0, // Will be calculated below - MaxOutputSize: 0, // Will be calculated below - ErrorCount: toolMetrics.ErrorCount, - } +func extractToolCallsFromGatewayLogStatus(entry GatewayLogEntry) string { + if entry.Status != "" { + return entry.Status + } + if entry.Error != "" || entry.Level == "error" { + return "error" + } + return "success" +} - if toolMetrics.AvgDuration > 0 { - summary.AvgDuration = timeutil.FormatDuration(time.Duration(toolMetrics.AvgDuration * float64(time.Millisecond))) - } - if toolMetrics.MaxDuration > 0 { - summary.MaxDuration = timeutil.FormatDuration(time.Duration(toolMetrics.MaxDuration * float64(time.Millisecond))) - } +// buildMCPSummaryStats populates mcpData.Summary and mcpData.Servers from aggregated gateway metrics. +func buildMCPSummaryStats(gatewayMetrics *GatewayMetrics, mcpData *MCPToolUsageData) { + for serverName, serverMetrics := range gatewayMetrics.Servers { + serverStats := buildMCPSummaryStatsServer(serverName, serverMetrics, mcpData) + mcpData.Servers = append(mcpData.Servers, serverStats) + } - // Calculate max input/output sizes from individual tool calls - for _, tc := range mcpData.ToolCalls { - if tc.ServerName == serverName && tc.ToolName == toolName { - if tc.InputSize > summary.MaxInputSize { - summary.MaxInputSize = tc.InputSize - } - if tc.OutputSize > summary.MaxOutputSize { - summary.MaxOutputSize = tc.OutputSize - } - } - } + buildMCPSummaryStatsSort(mcpData) +} - mcpData.Summary = append(mcpData.Summary, summary) +func buildMCPSummaryStatsServer(serverName string, serverMetrics *GatewayServerMetrics, mcpData *MCPToolUsageData) MCPServerStats { + serverStats := MCPServerStats{ + ServerName: serverName, + RequestCount: serverMetrics.RequestCount, + ToolCallCount: serverMetrics.ToolCallCount, + ErrorCount: serverMetrics.ErrorCount, + } + if serverMetrics.RequestCount > 0 { + avgDur := serverMetrics.TotalDuration / float64(serverMetrics.RequestCount) + serverStats.AvgDuration = timeutil.FormatDuration(time.Duration(avgDur * float64(time.Millisecond))) + } + for toolName, toolMetrics := range serverMetrics.Tools { + summary := buildMCPSummaryStatsTool(serverName, toolName, toolMetrics, mcpData.ToolCalls) + mcpData.Summary = append(mcpData.Summary, summary) + serverStats.TotalInputSize += toolMetrics.TotalInputSize + serverStats.TotalOutputSize += toolMetrics.TotalOutputSize + } + return serverStats +} - // Update server totals - serverStats.TotalInputSize += toolMetrics.TotalInputSize - serverStats.TotalOutputSize += toolMetrics.TotalOutputSize +func buildMCPSummaryStatsTool(serverName, toolName string, toolMetrics *GatewayToolMetrics, toolCalls []MCPToolCall) MCPToolSummary { + summary := MCPToolSummary{ + ServerName: serverName, + ToolName: toolName, + CallCount: toolMetrics.CallCount, + TotalInputSize: toolMetrics.TotalInputSize, + TotalOutputSize: toolMetrics.TotalOutputSize, + ErrorCount: toolMetrics.ErrorCount, + } + if toolMetrics.AvgDuration > 0 { + summary.AvgDuration = timeutil.FormatDuration(time.Duration(toolMetrics.AvgDuration * float64(time.Millisecond))) + } + if toolMetrics.MaxDuration > 0 { + summary.MaxDuration = timeutil.FormatDuration(time.Duration(toolMetrics.MaxDuration * float64(time.Millisecond))) + } + for _, tc := range toolCalls { + if tc.ServerName == serverName && tc.ToolName == toolName { + summary.MaxInputSize = max(summary.MaxInputSize, tc.InputSize) + summary.MaxOutputSize = max(summary.MaxOutputSize, tc.OutputSize) } - - mcpData.Servers = append(mcpData.Servers, serverStats) } + return summary +} - // Sort summaries by server name, then tool name +func buildMCPSummaryStatsSort(mcpData *MCPToolUsageData) { slices.SortFunc(mcpData.Summary, func(a, b MCPToolSummary) int { if a.ServerName != b.ServerName { if a.ServerName < b.ServerName { @@ -251,7 +251,6 @@ func buildMCPSummaryStats(gatewayMetrics *GatewayMetrics, mcpData *MCPToolUsageD } }) - // Sort servers by name slices.SortFunc(mcpData.Servers, func(a, b MCPServerStats) int { switch { case a.ServerName < b.ServerName: diff --git a/pkg/cli/gateway_logs_parsing.go b/pkg/cli/gateway_logs_parsing.go index f96b1e7b311..1230a643bf2 100644 --- a/pkg/cli/gateway_logs_parsing.go +++ b/pkg/cli/gateway_logs_parsing.go @@ -18,27 +18,12 @@ import ( // parseGatewayLogs parses a gateway.jsonl file and extracts metrics. // Falls back to rpc-messages.jsonl (canonical fallback) when gateway.jsonl is not present. func parseGatewayLogs(logDir string, verbose bool) (*GatewayMetrics, error) { - // Try root directory first (for older logs where gateway.jsonl was in the root) - gatewayLogPath := filepath.Join(logDir, "gateway.jsonl") - - // Check if gateway.jsonl exists in root - if _, err := os.Stat(gatewayLogPath); os.IsNotExist(err) { - // Try mcp-logs subdirectory (new path after artifact download) - // Gateway logs are uploaded from /tmp/gh-aw/mcp-logs/gateway.jsonl and the common parent - // /tmp/gh-aw/ is stripped during artifact upload, resulting in mcp-logs/gateway.jsonl after download - mcpLogsPath := filepath.Join(logDir, "mcp-logs", "gateway.jsonl") - if _, err := os.Stat(mcpLogsPath); os.IsNotExist(err) { - // Fall back to rpc-messages.jsonl (canonical fallback when gateway.jsonl is missing) - rpcPath := findRPCMessagesPath(logDir) - if rpcPath != "" { - gatewayLogsLog.Printf("gateway.jsonl not found; falling back to rpc-messages.jsonl: %s", rpcPath) - return parseRPCMessages(rpcPath, verbose) - } - gatewayLogsLog.Printf("gateway.jsonl not found at: %s or %s", gatewayLogPath, mcpLogsPath) - return nil, errors.New("gateway.jsonl not found") - } - gatewayLogPath = mcpLogsPath - gatewayLogsLog.Printf("Found gateway.jsonl in mcp-logs subdirectory") + gatewayLogPath, rpcPath, err := parseGatewayLogsPath(logDir) + if err != nil { + return nil, err + } + if rpcPath != "" { + return parseRPCMessages(rpcPath, verbose) } gatewayLogsLog.Printf("Parsing gateway.jsonl from: %s", gatewayLogPath) @@ -93,133 +78,159 @@ func parseGatewayLogs(logDir string, verbose bool) (*GatewayMetrics, error) { return metrics, nil } +func parseGatewayLogsPath(logDir string) (string, string, error) { + // Try root directory first (for older logs where gateway.jsonl was in the root) + gatewayLogPath := filepath.Join(logDir, "gateway.jsonl") + if _, err := os.Stat(gatewayLogPath); err == nil { + return gatewayLogPath, "", nil + } + + // Try mcp-logs subdirectory (new path after artifact download) + // Gateway logs are uploaded from /tmp/gh-aw/mcp-logs/gateway.jsonl and the common parent + // /tmp/gh-aw/ is stripped during artifact upload, resulting in mcp-logs/gateway.jsonl after download + mcpLogsPath := filepath.Join(logDir, "mcp-logs", "gateway.jsonl") + if _, err := os.Stat(mcpLogsPath); err == nil { + gatewayLogsLog.Printf("Found gateway.jsonl in mcp-logs subdirectory") + return mcpLogsPath, "", nil + } + + // Fall back to rpc-messages.jsonl (canonical fallback when gateway.jsonl is missing) + rpcPath := findRPCMessagesPath(logDir) + if rpcPath != "" { + gatewayLogsLog.Printf("gateway.jsonl not found; falling back to rpc-messages.jsonl: %s", rpcPath) + return "", rpcPath, nil + } + gatewayLogsLog.Printf("gateway.jsonl not found at: %s or %s", gatewayLogPath, mcpLogsPath) + return "", "", errors.New("gateway.jsonl not found") +} + // processGatewayLogEntry processes a single log entry and updates metrics func processGatewayLogEntry(entry *GatewayLogEntry, metrics *GatewayMetrics, verbose bool) { - // Parse timestamp for time range (supports both RFC3339 and RFC3339Nano) - if entry.Timestamp != "" { - t, err := time.Parse(time.RFC3339Nano, entry.Timestamp) - if err != nil { - t, err = time.Parse(time.RFC3339, entry.Timestamp) - } - if err == nil { - if metrics.StartTime.IsZero() || t.Before(metrics.StartTime) { - metrics.StartTime = t - } - if metrics.EndTime.IsZero() || t.After(metrics.EndTime) { - metrics.EndTime = t - } - } - } + processGatewayLogEntryTimeRange(entry, metrics) // Handle DIFC_FILTERED events if entry.Type == "DIFC_FILTERED" { - metrics.TotalFiltered++ - // DIFC_FILTERED events use server_id; fall back to server_name for compatibility - serverKey := entry.ServerID - if serverKey == "" { - serverKey = entry.ServerName - } - if serverKey != "" { - server := getOrCreateServer(metrics, serverKey) - server.FilteredCount++ - } - metrics.FilteredEvents = append(metrics.FilteredEvents, DifcFilteredEvent{ - Timestamp: entry.Timestamp, - ServerID: serverKey, - ToolName: entry.ToolName, - Description: entry.Description, - Reason: entry.Reason, - SecrecyTags: entry.SecrecyTags, - IntegrityTags: entry.IntegrityTags, - AuthorAssociation: entry.AuthorAssociation, - AuthorLogin: entry.AuthorLogin, - HTMLURL: entry.HTMLURL, - Number: entry.Number, - }) + processGatewayLogEntryFiltered(entry, metrics) return } // Handle GUARD_POLICY_BLOCKED events from gateway.jsonl if entry.Type == "GUARD_POLICY_BLOCKED" { - metrics.TotalGuardBlocked++ - serverKey := entry.ServerID - if serverKey == "" { - serverKey = entry.ServerName - } - if serverKey != "" { - server := getOrCreateServer(metrics, serverKey) - server.GuardPolicyBlocked++ - } - metrics.GuardPolicyEvents = append(metrics.GuardPolicyEvents, GuardPolicyEvent{ - Timestamp: entry.Timestamp, - ServerID: serverKey, - ToolName: entry.ToolName, - Reason: entry.Reason, - Message: entry.Message, - Details: entry.Description, - }) + processGatewayLogEntryGuardPolicy(entry, metrics) return } // Track errors. Include entries where level is "error" to handle gateway log formats // that omit the "status" field post-OTel-collector migration. - if entry.Status == "error" || entry.Error != "" || entry.Level == "error" { - metrics.TotalErrors++ - if entry.ServerName != "" { - server := getOrCreateServer(metrics, entry.ServerName) - server.ErrorCount++ - - if entry.ToolName != "" { - tool := getOrCreateTool(server, entry.ToolName) - tool.ErrorCount++ - } - } - } + processGatewayLogEntryError(entry, metrics) // Process based on event type switch entry.Event { case "request", "tool_call", "rpc_call": - metrics.TotalRequests++ + processGatewayLogEntryRequest(entry, metrics) + } +} - if entry.ServerName != "" { - server := getOrCreateServer(metrics, entry.ServerName) - server.RequestCount++ +func processGatewayLogEntryTimeRange(entry *GatewayLogEntry, metrics *GatewayMetrics) { + if entry.Timestamp == "" { + return + } + t, err := time.Parse(time.RFC3339Nano, entry.Timestamp) + if err != nil { + t, err = time.Parse(time.RFC3339, entry.Timestamp) + } + if err == nil { + if metrics.StartTime.IsZero() || t.Before(metrics.StartTime) { + metrics.StartTime = t + } + if metrics.EndTime.IsZero() || t.After(metrics.EndTime) { + metrics.EndTime = t + } + } +} - if entry.Duration > 0 { - server.TotalDuration += entry.Duration - metrics.TotalDuration += entry.Duration - } +func processGatewayLogEntryFiltered(entry *GatewayLogEntry, metrics *GatewayMetrics) { + metrics.TotalFiltered++ + serverKey := entry.ServerID + if serverKey == "" { + serverKey = entry.ServerName + } + if serverKey != "" { + server := getOrCreateServer(metrics, serverKey) + server.FilteredCount++ + } + metrics.FilteredEvents = append(metrics.FilteredEvents, DifcFilteredEvent{ + Timestamp: entry.Timestamp, ServerID: serverKey, ToolName: entry.ToolName, + Description: entry.Description, Reason: entry.Reason, SecrecyTags: entry.SecrecyTags, + IntegrityTags: entry.IntegrityTags, AuthorAssociation: entry.AuthorAssociation, + AuthorLogin: entry.AuthorLogin, HTMLURL: entry.HTMLURL, Number: entry.Number, + }) +} - // Track tool calls - if entry.ToolName != "" || entry.Method != "" { - toolName := entry.ToolName - if toolName == "" { - toolName = entry.Method - } - - metrics.TotalToolCalls++ - server.ToolCallCount++ - - tool := getOrCreateTool(server, toolName) - tool.CallCount++ - - if entry.Duration > 0 { - tool.TotalDuration += entry.Duration - if tool.MaxDuration == 0 || entry.Duration > tool.MaxDuration { - tool.MaxDuration = entry.Duration - } - if tool.MinDuration == 0 || entry.Duration < tool.MinDuration { - tool.MinDuration = entry.Duration - } - } - - if entry.InputSize > 0 { - tool.TotalInputSize += entry.InputSize - } - if entry.OutputSize > 0 { - tool.TotalOutputSize += entry.OutputSize - } - } +func processGatewayLogEntryGuardPolicy(entry *GatewayLogEntry, metrics *GatewayMetrics) { + metrics.TotalGuardBlocked++ + serverKey := entry.ServerID + if serverKey == "" { + serverKey = entry.ServerName + } + if serverKey != "" { + server := getOrCreateServer(metrics, serverKey) + server.GuardPolicyBlocked++ + } + metrics.GuardPolicyEvents = append(metrics.GuardPolicyEvents, GuardPolicyEvent{ + Timestamp: entry.Timestamp, ServerID: serverKey, ToolName: entry.ToolName, + Reason: entry.Reason, Message: entry.Message, Details: entry.Description, + }) +} + +func processGatewayLogEntryError(entry *GatewayLogEntry, metrics *GatewayMetrics) { + if entry.Status != "error" && entry.Error == "" && entry.Level != "error" { + return + } + metrics.TotalErrors++ + if entry.ServerName != "" { + server := getOrCreateServer(metrics, entry.ServerName) + server.ErrorCount++ + if entry.ToolName != "" { + tool := getOrCreateTool(server, entry.ToolName) + tool.ErrorCount++ + } + } +} + +func processGatewayLogEntryRequest(entry *GatewayLogEntry, metrics *GatewayMetrics) { + metrics.TotalRequests++ + if entry.ServerName == "" { + return + } + server := getOrCreateServer(metrics, entry.ServerName) + server.RequestCount++ + if entry.Duration > 0 { + server.TotalDuration += entry.Duration + metrics.TotalDuration += entry.Duration + } + processGatewayLogEntryTool(entry, metrics, server) +} + +func processGatewayLogEntryTool(entry *GatewayLogEntry, metrics *GatewayMetrics, server *GatewayServerMetrics) { + toolName := entry.ToolName + if toolName == "" { + toolName = entry.Method + } + if toolName == "" { + return + } + metrics.TotalToolCalls++ + server.ToolCallCount++ + tool := getOrCreateTool(server, toolName) + tool.CallCount++ + if entry.Duration > 0 { + tool.TotalDuration += entry.Duration + tool.MaxDuration = max(tool.MaxDuration, entry.Duration) + if tool.MinDuration == 0 || entry.Duration < tool.MinDuration { + tool.MinDuration = entry.Duration } } + tool.TotalInputSize += max(0, entry.InputSize) + tool.TotalOutputSize += max(0, entry.OutputSize) } diff --git a/pkg/cli/gateway_logs_render.go b/pkg/cli/gateway_logs_render.go index d9e25350c96..04fb442368e 100644 --- a/pkg/cli/gateway_logs_render.go +++ b/pkg/cli/gateway_logs_render.go @@ -32,157 +32,134 @@ func renderGatewayMetricsTable(metrics *GatewayMetrics, verbose bool) string { output.WriteString("\n\n") // Summary statistics - fmt.Fprintf(&output, "Total Requests: %d\n", metrics.TotalRequests) - fmt.Fprintf(&output, "Total Tool Calls: %d\n", metrics.TotalToolCalls) - fmt.Fprintf(&output, "Total Errors: %d\n", metrics.TotalErrors) + renderGatewayMetricsTableSummary(&output, metrics) + + // Server metrics table + if len(metrics.Servers) > 0 { + renderGatewayMetricsTableServers(&output, metrics) + } + + // DIFC filtered events table + if len(metrics.FilteredEvents) > 0 { + renderGatewayMetricsTableFiltered(&output, metrics) + } + + // Guard policy events table + if len(metrics.GuardPolicyEvents) > 0 { + renderGatewayMetricsTableGuardPolicy(&output, metrics) + } + + // Tool metrics table (if verbose) + if verbose { + renderGatewayMetricsTableTools(&output, metrics) + } + + return output.String() +} + +func renderGatewayMetricsTableSummary(output *strings.Builder, metrics *GatewayMetrics) { + fmt.Fprintf(output, "Total Requests: %d\n", metrics.TotalRequests) + fmt.Fprintf(output, "Total Tool Calls: %d\n", metrics.TotalToolCalls) + fmt.Fprintf(output, "Total Errors: %d\n", metrics.TotalErrors) if metrics.TotalFiltered > 0 { - fmt.Fprintf(&output, "Total DIFC Filtered: %d\n", metrics.TotalFiltered) + fmt.Fprintf(output, "Total DIFC Filtered: %d\n", metrics.TotalFiltered) } if metrics.TotalGuardBlocked > 0 { - fmt.Fprintf(&output, "Total Guard Policy Blocked: %d\n", metrics.TotalGuardBlocked) + fmt.Fprintf(output, "Total Guard Policy Blocked: %d\n", metrics.TotalGuardBlocked) } - fmt.Fprintf(&output, "Servers: %d\n", len(metrics.Servers)) - + fmt.Fprintf(output, "Servers: %d\n", len(metrics.Servers)) if !metrics.StartTime.IsZero() && !metrics.EndTime.IsZero() { duration := metrics.EndTime.Sub(metrics.StartTime) - fmt.Fprintf(&output, "Time Range: %s\n", duration.Round(time.Second)) + fmt.Fprintf(output, "Time Range: %s\n", duration.Round(time.Second)) } - output.WriteString("\n") +} - // Server metrics table - if len(metrics.Servers) > 0 { - // Sort servers by request count - serverNames := getSortedServerNames(metrics) - - hasFiltered := metrics.TotalFiltered > 0 - hasGuardPolicy := metrics.TotalGuardBlocked > 0 - serverRows := make([][]string, 0, len(serverNames)) - for _, serverName := range serverNames { - server := metrics.Servers[serverName] - avgTime := 0.0 - if server.RequestCount > 0 { - avgTime = server.TotalDuration / float64(server.RequestCount) - } - row := []string{ - serverName, - strconv.Itoa(server.RequestCount), - strconv.Itoa(server.ToolCallCount), - fmt.Sprintf("%.0fms", avgTime), - strconv.Itoa(server.ErrorCount), - } - if hasFiltered { - row = append(row, strconv.Itoa(server.FilteredCount)) - } - if hasGuardPolicy { - row = append(row, strconv.Itoa(server.GuardPolicyBlocked)) - } - serverRows = append(serverRows, row) - } +func renderGatewayMetricsTableServers(output *strings.Builder, metrics *GatewayMetrics) { + serverNames := getSortedServerNames(metrics) + hasFiltered := metrics.TotalFiltered > 0 + hasGuardPolicy := metrics.TotalGuardBlocked > 0 + serverRows := make([][]string, 0, len(serverNames)) + for _, serverName := range serverNames { + serverRows = append(serverRows, renderGatewayMetricsTableServerRow(serverName, metrics.Servers[serverName], hasFiltered, hasGuardPolicy)) + } - headers := []string{"Server", "Requests", "Tool Calls", "Avg Time", "Errors"} - if hasFiltered { - headers = append(headers, "Filtered") - } - if hasGuardPolicy { - headers = append(headers, "Guard Blocked") - } - output.WriteString(console.RenderTable(console.TableConfig{ - Title: "Server Usage", - Headers: headers, - Rows: serverRows, - })) + headers := []string{"Server", "Requests", "Tool Calls", "Avg Time", "Errors"} + if hasFiltered { + headers = append(headers, "Filtered") + } + if hasGuardPolicy { + headers = append(headers, "Guard Blocked") } + output.WriteString(console.RenderTable(console.TableConfig{Title: "Server Usage", Headers: headers, Rows: serverRows})) +} - // DIFC filtered events table - if len(metrics.FilteredEvents) > 0 { - output.WriteString("\n") - filteredRows := make([][]string, 0, len(metrics.FilteredEvents)) - for _, fe := range metrics.FilteredEvents { - reason := stringutil.Truncate(fe.Reason, 80) - filteredRows = append(filteredRows, []string{ - fe.ServerID, - fe.ToolName, - fe.AuthorLogin, - reason, - }) +func renderGatewayMetricsTableServerRow(serverName string, server *GatewayServerMetrics, hasFiltered, hasGuardPolicy bool) []string { + avgTime := 0.0 + if server.RequestCount > 0 { + avgTime = server.TotalDuration / float64(server.RequestCount) + } + row := []string{serverName, strconv.Itoa(server.RequestCount), strconv.Itoa(server.ToolCallCount), fmt.Sprintf("%.0fms", avgTime), strconv.Itoa(server.ErrorCount)} + if hasFiltered { + row = append(row, strconv.Itoa(server.FilteredCount)) + } + if hasGuardPolicy { + row = append(row, strconv.Itoa(server.GuardPolicyBlocked)) + } + return row +} + +func renderGatewayMetricsTableFiltered(output *strings.Builder, metrics *GatewayMetrics) { + output.WriteString("\n") + filteredRows := make([][]string, 0, len(metrics.FilteredEvents)) + for _, fe := range metrics.FilteredEvents { + filteredRows = append(filteredRows, []string{fe.ServerID, fe.ToolName, fe.AuthorLogin, stringutil.Truncate(fe.Reason, 80)}) + } + output.WriteString(console.RenderTable(console.TableConfig{ + Title: "DIFC Filtered Events", Headers: []string{"Server", "Tool", "User", "Reason"}, Rows: filteredRows, + })) +} + +func renderGatewayMetricsTableGuardPolicy(output *strings.Builder, metrics *GatewayMetrics) { + output.WriteString("\n") + guardRows := make([][]string, 0, len(metrics.GuardPolicyEvents)) + for _, gpe := range metrics.GuardPolicyEvents { + repo := gpe.Repository + if repo == "" { + repo = "-" } - output.WriteString(console.RenderTable(console.TableConfig{ - Title: "DIFC Filtered Events", - Headers: []string{"Server", "Tool", "User", "Reason"}, - Rows: filteredRows, - })) + guardRows = append(guardRows, []string{gpe.ServerID, gpe.ToolName, gpe.Reason, stringutil.Truncate(gpe.Message, 60), repo}) } + output.WriteString(console.RenderTable(console.TableConfig{ + Title: "Guard Policy Blocked Events", Headers: []string{"Server", "Tool", "Reason", "Message", "Repository"}, Rows: guardRows, + })) +} - // Guard policy events table - if len(metrics.GuardPolicyEvents) > 0 { - output.WriteString("\n") - guardRows := make([][]string, 0, len(metrics.GuardPolicyEvents)) - for _, gpe := range metrics.GuardPolicyEvents { - message := stringutil.Truncate(gpe.Message, 60) - repo := gpe.Repository - if repo == "" { - repo = "-" - } - guardRows = append(guardRows, []string{ - gpe.ServerID, - gpe.ToolName, - gpe.Reason, - message, - repo, - }) +func renderGatewayMetricsTableTools(output *strings.Builder, metrics *GatewayMetrics) { + output.WriteString("\n") + output.WriteString("Tool Usage Details:\n") + for _, serverName := range getSortedServerNames(metrics) { + server := metrics.Servers[serverName] + if len(server.Tools) == 0 { + continue } output.WriteString(console.RenderTable(console.TableConfig{ - Title: "Guard Policy Blocked Events", - Headers: []string{"Server", "Tool", "Reason", "Message", "Repository"}, - Rows: guardRows, + Title: serverName, Headers: []string{"Tool", "Calls", "Avg Time", "Max Time", "Errors"}, Rows: renderGatewayMetricsTableToolRows(server), })) } +} - // Tool metrics table (if verbose) - if verbose { - output.WriteString("\n") - output.WriteString("Tool Usage Details:\n") - - for _, serverName := range getSortedServerNames(metrics) { - server := metrics.Servers[serverName] - if len(server.Tools) == 0 { - continue - } - - // Sort tools by call count - toolNames := sliceutil.MapKeys(server.Tools) - slices.SortFunc(toolNames, func(a, b string) int { - if server.Tools[a].CallCount > server.Tools[b].CallCount { - return -1 - } - if server.Tools[a].CallCount < server.Tools[b].CallCount { - return 1 - } - return 0 - }) - - toolRows := make([][]string, 0, len(toolNames)) - for _, toolName := range toolNames { - tool := server.Tools[toolName] - toolRows = append(toolRows, []string{ - toolName, - strconv.Itoa(tool.CallCount), - fmt.Sprintf("%.0fms", tool.AvgDuration), - fmt.Sprintf("%.0fms", tool.MaxDuration), - strconv.Itoa(tool.ErrorCount), - }) - } - - output.WriteString(console.RenderTable(console.TableConfig{ - Title: serverName, - Headers: []string{"Tool", "Calls", "Avg Time", "Max Time", "Errors"}, - Rows: toolRows, - })) - } +func renderGatewayMetricsTableToolRows(server *GatewayServerMetrics) [][]string { + toolNames := sliceutil.MapKeys(server.Tools) + slices.SortFunc(toolNames, func(a, b string) int { + return server.Tools[b].CallCount - server.Tools[a].CallCount + }) + toolRows := make([][]string, 0, len(toolNames)) + for _, toolName := range toolNames { + tool := server.Tools[toolName] + toolRows = append(toolRows, []string{toolName, strconv.Itoa(tool.CallCount), fmt.Sprintf("%.0fms", tool.AvgDuration), fmt.Sprintf("%.0fms", tool.MaxDuration), strconv.Itoa(tool.ErrorCount)}) } - - return output.String() + return toolRows } // getSortedServerNames returns server names sorted by request count @@ -235,42 +212,7 @@ func displayAggregatedGatewayMetrics(processedRuns []ProcessedRun, outputDir str aggregated.FilteredEvents = append(aggregated.FilteredEvents, runMetrics.FilteredEvents...) aggregated.GuardPolicyEvents = append(aggregated.GuardPolicyEvents, runMetrics.GuardPolicyEvents...) - // Merge server metrics - for serverName, serverMetrics := range runMetrics.Servers { - aggServer := getOrCreateServer(aggregated, serverName) - aggServer.RequestCount += serverMetrics.RequestCount - aggServer.ToolCallCount += serverMetrics.ToolCallCount - aggServer.TotalDuration += serverMetrics.TotalDuration - aggServer.ErrorCount += serverMetrics.ErrorCount - aggServer.FilteredCount += serverMetrics.FilteredCount - aggServer.GuardPolicyBlocked += serverMetrics.GuardPolicyBlocked - - // Merge tool metrics - for toolName, toolMetrics := range serverMetrics.Tools { - aggTool := getOrCreateTool(aggServer, toolName) - aggTool.CallCount += toolMetrics.CallCount - aggTool.TotalDuration += toolMetrics.TotalDuration - aggTool.ErrorCount += toolMetrics.ErrorCount - aggTool.TotalInputSize += toolMetrics.TotalInputSize - aggTool.TotalOutputSize += toolMetrics.TotalOutputSize - - // Update max/min durations - if toolMetrics.MaxDuration > aggTool.MaxDuration { - aggTool.MaxDuration = toolMetrics.MaxDuration - } - if aggTool.MinDuration == 0 || (toolMetrics.MinDuration > 0 && toolMetrics.MinDuration < aggTool.MinDuration) { - aggTool.MinDuration = toolMetrics.MinDuration - } - } - } - - // Update time range - if aggregated.StartTime.IsZero() || (!runMetrics.StartTime.IsZero() && runMetrics.StartTime.Before(aggregated.StartTime)) { - aggregated.StartTime = runMetrics.StartTime - } - if aggregated.EndTime.IsZero() || (!runMetrics.EndTime.IsZero() && runMetrics.EndTime.After(aggregated.EndTime)) { - aggregated.EndTime = runMetrics.EndTime - } + displayAggregatedGatewayMetricsMerge(aggregated, runMetrics) } // Only display if we found gateway metrics @@ -293,3 +235,37 @@ func displayAggregatedGatewayMetrics(processedRuns []ProcessedRun, outputDir str } } } + +func displayAggregatedGatewayMetricsMerge(aggregated, runMetrics *GatewayMetrics) { + for serverName, serverMetrics := range runMetrics.Servers { + aggServer := getOrCreateServer(aggregated, serverName) + aggServer.RequestCount += serverMetrics.RequestCount + aggServer.ToolCallCount += serverMetrics.ToolCallCount + aggServer.TotalDuration += serverMetrics.TotalDuration + aggServer.ErrorCount += serverMetrics.ErrorCount + aggServer.FilteredCount += serverMetrics.FilteredCount + aggServer.GuardPolicyBlocked += serverMetrics.GuardPolicyBlocked + displayAggregatedGatewayMetricsMergeTools(aggServer, serverMetrics) + } + if aggregated.StartTime.IsZero() || (!runMetrics.StartTime.IsZero() && runMetrics.StartTime.Before(aggregated.StartTime)) { + aggregated.StartTime = runMetrics.StartTime + } + if aggregated.EndTime.IsZero() || (!runMetrics.EndTime.IsZero() && runMetrics.EndTime.After(aggregated.EndTime)) { + aggregated.EndTime = runMetrics.EndTime + } +} + +func displayAggregatedGatewayMetricsMergeTools(aggServer, serverMetrics *GatewayServerMetrics) { + for toolName, toolMetrics := range serverMetrics.Tools { + aggTool := getOrCreateTool(aggServer, toolName) + aggTool.CallCount += toolMetrics.CallCount + aggTool.TotalDuration += toolMetrics.TotalDuration + aggTool.ErrorCount += toolMetrics.ErrorCount + aggTool.TotalInputSize += toolMetrics.TotalInputSize + aggTool.TotalOutputSize += toolMetrics.TotalOutputSize + aggTool.MaxDuration = max(aggTool.MaxDuration, toolMetrics.MaxDuration) + if aggTool.MinDuration == 0 || (toolMetrics.MinDuration > 0 && toolMetrics.MinDuration < aggTool.MinDuration) { + aggTool.MinDuration = toolMetrics.MinDuration + } + } +} diff --git a/pkg/cli/gateway_logs_rpc.go b/pkg/cli/gateway_logs_rpc.go index c39443a3b98..b3e40aa71eb 100644 --- a/pkg/cli/gateway_logs_rpc.go +++ b/pkg/cli/gateway_logs_rpc.go @@ -59,168 +59,164 @@ func parseRPCMessages(logPath string, verbose bool) (*GatewayMetrics, error) { continue } - // Update time range - if entry.Timestamp != "" { - if t, err := time.Parse(time.RFC3339Nano, entry.Timestamp); err == nil { - if metrics.StartTime.IsZero() || t.Before(metrics.StartTime) { - metrics.StartTime = t - } - if metrics.EndTime.IsZero() || t.After(metrics.EndTime) { - metrics.EndTime = t - } - } - } + parseRPCMessagesTimeRange(&entry, metrics) if entry.ServerID == "" { continue } - switch { - case entry.Type == "DIFC_FILTERED": - // DIFC integrity/secrecy filter event — not a REQUEST or RESPONSE - metrics.TotalFiltered++ - server := getOrCreateServer(metrics, entry.ServerID) - server.FilteredCount++ - metrics.FilteredEvents = append(metrics.FilteredEvents, DifcFilteredEvent{ - Timestamp: entry.Timestamp, - ServerID: entry.ServerID, - ToolName: entry.ToolName, - Description: entry.Description, - Reason: entry.Reason, - SecrecyTags: entry.SecrecyTags, - IntegrityTags: entry.IntegrityTags, - AuthorAssociation: entry.AuthorAssociation, - AuthorLogin: entry.AuthorLogin, - HTMLURL: entry.HTMLURL, - Number: entry.Number, - }) + parseRPCMessagesEntry(&entry, metrics, pendingRequests) + } - case entry.Direction == "OUT" && entry.Type == "REQUEST": - // Outgoing request from AI engine to MCP server - var req rpcRequestPayload - if err := json.Unmarshal(entry.Payload, &req); err != nil { - continue - } - if req.Method != "tools/call" { - continue - } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading rpc-messages.jsonl: %w", err) + } - // Extract tool name - var params rpcToolCallParams - if err := json.Unmarshal(req.Params, ¶ms); err != nil || params.Name == "" { - continue - } + calculateGatewayAggregates(metrics) - metrics.TotalRequests++ - server := getOrCreateServer(metrics, entry.ServerID) - server.RequestCount++ - metrics.TotalToolCalls++ - server.ToolCallCount++ - - tool := getOrCreateTool(server, params.Name) - tool.CallCount++ - - // Store pending request for duration calculation - if req.ID != nil && entry.Timestamp != "" { - if t, err := time.Parse(time.RFC3339Nano, entry.Timestamp); err == nil { - key := fmt.Sprintf("%s/%v", entry.ServerID, req.ID) - pendingRequests[key] = &rpcPendingRequest{ - ServerID: entry.ServerID, - ToolName: params.Name, - Timestamp: t, - } - } - } + gatewayLogsLog.Printf("Successfully parsed rpc-messages.jsonl: %d servers, %d total requests", + len(metrics.Servers), metrics.TotalRequests) - case entry.Direction == "IN" && entry.Type == "RESPONSE": - // Incoming response from MCP server to AI engine - var resp rpcResponsePayload - if err := json.Unmarshal(entry.Payload, &resp); err != nil { - continue - } + return metrics, nil +} - // Track errors and detect guard policy blocks - if resp.Error != nil { - metrics.TotalErrors++ - server := getOrCreateServer(metrics, entry.ServerID) - server.ErrorCount++ - - // Detect guard policy enforcement errors - if isGuardPolicyErrorCode(resp.Error.Code) { - metrics.TotalGuardBlocked++ - server.GuardPolicyBlocked++ - - // Determine tool name from pending request if available - toolName := "" - if resp.ID != nil { - key := fmt.Sprintf("%s/%v", entry.ServerID, resp.ID) - if pending, ok := pendingRequests[key]; ok { - toolName = pending.ToolName - } - } - - reason := guardPolicyReasonFromCode(resp.Error.Code) - if resp.Error.Data != nil && resp.Error.Data.Reason != "" { - reason = resp.Error.Data.Reason - } - - evt := GuardPolicyEvent{ - Timestamp: entry.Timestamp, - ServerID: entry.ServerID, - ToolName: toolName, - ErrorCode: resp.Error.Code, - Reason: reason, - Message: resp.Error.Message, - } - if resp.Error.Data != nil { - evt.Details = resp.Error.Data.Details - evt.Repository = resp.Error.Data.Repository - } - metrics.GuardPolicyEvents = append(metrics.GuardPolicyEvents, evt) - } - } +func parseRPCMessagesTimeRange(entry *RPCMessageEntry, metrics *GatewayMetrics) { + if entry.Timestamp == "" { + return + } + if t, err := time.Parse(time.RFC3339Nano, entry.Timestamp); err == nil { + if metrics.StartTime.IsZero() || t.Before(metrics.StartTime) { + metrics.StartTime = t + } + if metrics.EndTime.IsZero() || t.After(metrics.EndTime) { + metrics.EndTime = t + } + } +} - // Calculate duration by matching with pending request - if resp.ID != nil && entry.Timestamp != "" { - key := fmt.Sprintf("%s/%v", entry.ServerID, resp.ID) - if pending, ok := pendingRequests[key]; ok { - delete(pendingRequests, key) - if t, err := time.Parse(time.RFC3339Nano, entry.Timestamp); err == nil { - durationMs := float64(t.Sub(pending.Timestamp).Milliseconds()) - if durationMs >= 0 { - server := getOrCreateServer(metrics, entry.ServerID) - server.TotalDuration += durationMs - metrics.TotalDuration += durationMs - - tool := getOrCreateTool(server, pending.ToolName) - tool.TotalDuration += durationMs - if tool.MaxDuration == 0 || durationMs > tool.MaxDuration { - tool.MaxDuration = durationMs - } - if tool.MinDuration == 0 || durationMs < tool.MinDuration { - tool.MinDuration = durationMs - } - - if resp.Error != nil { - tool.ErrorCount++ - } - } - } - } - } +func parseRPCMessagesEntry(entry *RPCMessageEntry, metrics *GatewayMetrics, pendingRequests map[string]*rpcPendingRequest) { + switch { + case entry.Type == "DIFC_FILTERED": + parseRPCMessagesFiltered(entry, metrics) + case entry.Direction == "OUT" && entry.Type == "REQUEST": + parseRPCMessagesRequest(entry, metrics, pendingRequests) + case entry.Direction == "IN" && entry.Type == "RESPONSE": + parseRPCMessagesResponse(entry, metrics, pendingRequests) + } +} + +func parseRPCMessagesFiltered(entry *RPCMessageEntry, metrics *GatewayMetrics) { + metrics.TotalFiltered++ + server := getOrCreateServer(metrics, entry.ServerID) + server.FilteredCount++ + metrics.FilteredEvents = append(metrics.FilteredEvents, DifcFilteredEvent{ + Timestamp: entry.Timestamp, ServerID: entry.ServerID, ToolName: entry.ToolName, + Description: entry.Description, Reason: entry.Reason, SecrecyTags: entry.SecrecyTags, + IntegrityTags: entry.IntegrityTags, AuthorAssociation: entry.AuthorAssociation, + AuthorLogin: entry.AuthorLogin, HTMLURL: entry.HTMLURL, Number: entry.Number, + }) +} + +func parseRPCMessagesRequest(entry *RPCMessageEntry, metrics *GatewayMetrics, pendingRequests map[string]*rpcPendingRequest) { + var req rpcRequestPayload + if err := json.Unmarshal(entry.Payload, &req); err != nil || req.Method != "tools/call" { + return + } + var params rpcToolCallParams + if err := json.Unmarshal(req.Params, ¶ms); err != nil || params.Name == "" { + return + } + + metrics.TotalRequests++ + server := getOrCreateServer(metrics, entry.ServerID) + server.RequestCount++ + metrics.TotalToolCalls++ + server.ToolCallCount++ + tool := getOrCreateTool(server, params.Name) + tool.CallCount++ + + if req.ID != nil && entry.Timestamp != "" { + if t, err := time.Parse(time.RFC3339Nano, entry.Timestamp); err == nil { + key := fmt.Sprintf("%s/%v", entry.ServerID, req.ID) + pendingRequests[key] = &rpcPendingRequest{ServerID: entry.ServerID, ToolName: params.Name, Timestamp: t} } } +} - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading rpc-messages.jsonl: %w", err) +func parseRPCMessagesResponse(entry *RPCMessageEntry, metrics *GatewayMetrics, pendingRequests map[string]*rpcPendingRequest) { + var resp rpcResponsePayload + if err := json.Unmarshal(entry.Payload, &resp); err != nil { + return + } + if resp.Error != nil { + parseRPCMessagesResponseError(entry, metrics, pendingRequests, &resp) } + if resp.ID != nil && entry.Timestamp != "" { + parseRPCMessagesResponseDuration(entry, metrics, pendingRequests, &resp) + } +} - calculateGatewayAggregates(metrics) +func parseRPCMessagesResponseError(entry *RPCMessageEntry, metrics *GatewayMetrics, pendingRequests map[string]*rpcPendingRequest, resp *rpcResponsePayload) { + metrics.TotalErrors++ + server := getOrCreateServer(metrics, entry.ServerID) + server.ErrorCount++ + if !isGuardPolicyErrorCode(resp.Error.Code) { + return + } + metrics.TotalGuardBlocked++ + server.GuardPolicyBlocked++ + evt := parseRPCMessagesGuardPolicyEvent(entry, pendingRequests, resp) + metrics.GuardPolicyEvents = append(metrics.GuardPolicyEvents, evt) +} - gatewayLogsLog.Printf("Successfully parsed rpc-messages.jsonl: %d servers, %d total requests", - len(metrics.Servers), metrics.TotalRequests) +func parseRPCMessagesGuardPolicyEvent(entry *RPCMessageEntry, pendingRequests map[string]*rpcPendingRequest, resp *rpcResponsePayload) GuardPolicyEvent { + toolName := "" + if resp.ID != nil { + key := fmt.Sprintf("%s/%v", entry.ServerID, resp.ID) + if pending, ok := pendingRequests[key]; ok { + toolName = pending.ToolName + } + } + reason := guardPolicyReasonFromCode(resp.Error.Code) + if resp.Error.Data != nil && resp.Error.Data.Reason != "" { + reason = resp.Error.Data.Reason + } + evt := GuardPolicyEvent{Timestamp: entry.Timestamp, ServerID: entry.ServerID, ToolName: toolName, ErrorCode: resp.Error.Code, Reason: reason, Message: resp.Error.Message} + if resp.Error.Data != nil { + evt.Details = resp.Error.Data.Details + evt.Repository = resp.Error.Data.Repository + } + return evt +} - return metrics, nil +func parseRPCMessagesResponseDuration(entry *RPCMessageEntry, metrics *GatewayMetrics, pendingRequests map[string]*rpcPendingRequest, resp *rpcResponsePayload) { + key := fmt.Sprintf("%s/%v", entry.ServerID, resp.ID) + pending, ok := pendingRequests[key] + if !ok { + return + } + delete(pendingRequests, key) + if t, err := time.Parse(time.RFC3339Nano, entry.Timestamp); err == nil { + durationMs := float64(t.Sub(pending.Timestamp).Milliseconds()) + if durationMs >= 0 { + parseRPCMessagesApplyDuration(entry.ServerID, pending.ToolName, durationMs, resp.Error != nil, metrics) + } + } +} + +func parseRPCMessagesApplyDuration(serverID, toolName string, durationMs float64, hadError bool, metrics *GatewayMetrics) { + server := getOrCreateServer(metrics, serverID) + server.TotalDuration += durationMs + metrics.TotalDuration += durationMs + tool := getOrCreateTool(server, toolName) + tool.TotalDuration += durationMs + tool.MaxDuration = max(tool.MaxDuration, durationMs) + if tool.MinDuration == 0 || durationMs < tool.MinDuration { + tool.MinDuration = durationMs + } + if hadError { + tool.ErrorCount++ + } } // findRPCMessagesPath returns the path to rpc-messages.jsonl if it exists, or "" if not found. @@ -241,6 +237,19 @@ func findRPCMessagesPath(logDir string) string { // buildToolCallsFromRPCMessages reads rpc-messages.jsonl and builds MCPToolCall records. // Duration is computed by pairing outgoing requests with incoming responses. // Input/output sizes are not available in rpc-messages.jsonl and will be 0. +type buildToolCallsFromRPCMessagesPendingCall struct { + serverID string + toolName string + timestamp time.Time +} + +type buildToolCallsFromRPCMessagesRawEntry struct { + entry RPCMessageEntry + req rpcRequestPayload + resp rpcResponsePayload + valid bool +} + func buildToolCallsFromRPCMessages(logPath string) ([]MCPToolCall, error) { file, err := os.Open(logPath) if err != nil { @@ -248,26 +257,35 @@ func buildToolCallsFromRPCMessages(logPath string) ([]MCPToolCall, error) { } defer file.Close() - type pendingCall struct { - serverID string - toolName string - timestamp time.Time + pending := make(map[string]*buildToolCallsFromRPCMessagesPendingCall) // key: "/" + entries, err := buildToolCallsFromRPCMessagesRead(file) + if err != nil { + return nil, err } - pending := make(map[string]*pendingCall) // key: "/" - // Collect requests first to pair with responses - type rawEntry struct { - entry RPCMessageEntry - req rpcRequestPayload - resp rpcResponsePayload - valid bool - } - var entries []rawEntry + // Second pass: build MCPToolCall records. + // Declared before first pass so requests without IDs can be appended immediately. + var toolCalls []MCPToolCall + processedKeys := make(map[string]struct { + }) + + // First pass: index outgoing tool-call requests by (serverID, id) + buildToolCallsFromRPCMessagesIndexRequests(entries, pending, &toolCalls) + + // Second pass: pair responses with pending requests to compute durations + buildToolCallsFromRPCMessagesPairResponses(entries, pending, processedKeys, &toolCalls) + + // Emit any requests that never received a response + buildToolCallsFromRPCMessagesUnknown(pending, processedKeys, &toolCalls) + + return toolCalls, nil +} +func buildToolCallsFromRPCMessagesRead(file *os.File) ([]buildToolCallsFromRPCMessagesRawEntry, error) { + var entries []buildToolCallsFromRPCMessagesRawEntry scanner := bufio.NewScanner(file) buf := make([]byte, maxScannerBufferSize) scanner.Buffer(buf, maxScannerBufferSize) - for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" { @@ -277,19 +295,15 @@ func buildToolCallsFromRPCMessages(logPath string) ([]MCPToolCall, error) { if err := json.Unmarshal([]byte(line), &e); err != nil { continue } - entries = append(entries, rawEntry{entry: e, valid: true}) + entries = append(entries, buildToolCallsFromRPCMessagesRawEntry{entry: e, valid: true}) } if err := scanner.Err(); err != nil { return nil, fmt.Errorf("error reading rpc-messages.jsonl: %w", err) } + return entries, nil +} - // Second pass: build MCPToolCall records. - // Declared before first pass so requests without IDs can be appended immediately. - var toolCalls []MCPToolCall - processedKeys := make(map[string]struct { - }) - - // First pass: index outgoing tool-call requests by (serverID, id) +func buildToolCallsFromRPCMessagesIndexRequests(entries []buildToolCallsFromRPCMessagesRawEntry, pending map[string]*buildToolCallsFromRPCMessagesPendingCall, toolCalls *[]MCPToolCall) { for i := range entries { e := &entries[i] if e.entry.Direction != "OUT" || e.entry.Type != "REQUEST" { @@ -303,15 +317,7 @@ func buildToolCallsFromRPCMessages(logPath string) ([]MCPToolCall, error) { continue } if e.req.ID == nil { - // Requests without an ID cannot be matched to responses. - // Emit the tool call immediately with "unknown" status so it appears - // in the tool_calls list (same as parseRPCMessages counts it in the summary). - toolCalls = append(toolCalls, MCPToolCall{ - Timestamp: e.entry.Timestamp, - ServerName: e.entry.ServerID, - ToolName: params.Name, - Status: "unknown", - }) + *toolCalls = append(*toolCalls, MCPToolCall{Timestamp: e.entry.Timestamp, ServerName: e.entry.ServerID, ToolName: params.Name, Status: "unknown"}) continue } t, err := time.Parse(time.RFC3339Nano, e.entry.Timestamp) @@ -319,66 +325,50 @@ func buildToolCallsFromRPCMessages(logPath string) ([]MCPToolCall, error) { continue } key := fmt.Sprintf("%s/%v", e.entry.ServerID, e.req.ID) - pending[key] = &pendingCall{ - serverID: e.entry.ServerID, - toolName: params.Name, - timestamp: t, - } + pending[key] = &buildToolCallsFromRPCMessagesPendingCall{serverID: e.entry.ServerID, toolName: params.Name, timestamp: t} } +} - // Second pass: pair responses with pending requests to compute durations +func buildToolCallsFromRPCMessagesPairResponses(entries []buildToolCallsFromRPCMessagesRawEntry, pending map[string]*buildToolCallsFromRPCMessagesPendingCall, processedKeys map[string]struct{}, toolCalls *[]MCPToolCall) { for i := range entries { e := &entries[i] - switch { - case e.entry.Direction == "OUT" && e.entry.Type == "REQUEST": - // Outgoing tool-call request – we'll emit the record when we see the response - // (or after if no response found) - case e.entry.Direction == "IN" && e.entry.Type == "RESPONSE": - if err := json.Unmarshal(e.entry.Payload, &e.resp); err != nil { - continue - } - if e.resp.ID == nil { - continue - } - key := fmt.Sprintf("%s/%v", e.entry.ServerID, e.resp.ID) - p, ok := pending[key] - if !ok { - continue - } - processedKeys[key] = struct { - }{} - - call := MCPToolCall{ - Timestamp: p.timestamp.Format(time.RFC3339Nano), - ServerName: p.serverID, - ToolName: p.toolName, - Status: "success", - } - if e.resp.Error != nil { - call.Status = "error" - call.Error = e.resp.Error.Message - } - if t, err := time.Parse(time.RFC3339Nano, e.entry.Timestamp); err == nil { - d := t.Sub(p.timestamp) - if d >= 0 { - call.Duration = timeutil.FormatDuration(d) - } - } - toolCalls = append(toolCalls, call) + if e.entry.Direction != "IN" || e.entry.Type != "RESPONSE" { + continue } + if err := json.Unmarshal(e.entry.Payload, &e.resp); err != nil || e.resp.ID == nil { + continue + } + key := fmt.Sprintf("%s/%v", e.entry.ServerID, e.resp.ID) + p, ok := pending[key] + if !ok { + continue + } + processedKeys[key] = struct{}{} + *toolCalls = append(*toolCalls, buildToolCallsFromRPCMessagesCall(e, p)) } +} - // Emit any requests that never received a response +func buildToolCallsFromRPCMessagesCall(e *buildToolCallsFromRPCMessagesRawEntry, p *buildToolCallsFromRPCMessagesPendingCall) MCPToolCall { + call := MCPToolCall{Timestamp: p.timestamp.Format(time.RFC3339Nano), ServerName: p.serverID, ToolName: p.toolName, Status: "success"} + if e.resp.Error != nil { + call.Status = "error" + call.Error = e.resp.Error.Message + } + if t, err := time.Parse(time.RFC3339Nano, e.entry.Timestamp); err == nil { + d := t.Sub(p.timestamp) + if d >= 0 { + call.Duration = timeutil.FormatDuration(d) + } + } + return call +} + +func buildToolCallsFromRPCMessagesUnknown(pending map[string]*buildToolCallsFromRPCMessagesPendingCall, processedKeys map[string]struct{}, toolCalls *[]MCPToolCall) { for key, p := range pending { if !setutil.Contains(processedKeys, key) { - toolCalls = append(toolCalls, MCPToolCall{ - Timestamp: p.timestamp.Format(time.RFC3339Nano), - ServerName: p.serverID, - ToolName: p.toolName, - Status: "unknown", + *toolCalls = append(*toolCalls, MCPToolCall{ + Timestamp: p.timestamp.Format(time.RFC3339Nano), ServerName: p.serverID, ToolName: p.toolName, Status: "unknown", }) } } - - return toolCalls, nil } diff --git a/pkg/cli/gateway_logs_timeline_render.go b/pkg/cli/gateway_logs_timeline_render.go index 0dc749745fe..d03f4bf485d 100644 --- a/pkg/cli/gateway_logs_timeline_render.go +++ b/pkg/cli/gateway_logs_timeline_render.go @@ -473,149 +473,207 @@ func renderUnifiedTimelineStream(events []UnifiedTimelineEvent) string { } isTerminal := tty.IsStdoutTerminal() + streamColor := renderUnifiedTimelineStreamColorFunc(isTerminal) + coloredMessageSnippet := renderUnifiedTimelineStreamMessageSnippetFunc(isTerminal) + var sb strings.Builder + inTurn := false + + for _, evt := range events { + renderUnifiedTimelineStreamEvent(&sb, evt, &inTurn, streamColor, coloredMessageSnippet) + } + + if inTurn { + sb.WriteString("\n") + } + + return sb.String() +} +func renderUnifiedTimelineStreamColorFunc(isTerminal bool) func(streamStyleRenderer, string) string { // streamColor wraps text with a lipgloss style only when output is a TTY so // that piped output stays clean of ANSI escape codes. - streamColor := func(s streamStyleRenderer, text string) string { + return func(s streamStyleRenderer, text string) string { if isTerminal { return s.Render(text) } return text } +} +func renderUnifiedTimelineStreamMessageSnippetFunc(isTerminal bool) func(string, string, streamStyleRenderer, streamStyleRenderer) string { // coloredMessageSnippet renders the first few lines of message content. // lineStyle and truncStyle are applied only when output is a TTY. - coloredMessageSnippet := func(content, indent string, lineStyle, truncStyle streamStyleRenderer) string { + return func(content, indent string, lineStyle, truncStyle streamStyleRenderer) string { ls, ts := streamStyleRenderer(noopStyleRenderer{}), streamStyleRenderer(noopStyleRenderer{}) if isTerminal { ls, ts = lineStyle, truncStyle } return renderMessageSnippet(content, indent, ls, ts) } +} - var sb strings.Builder - inTurn := false +func renderUnifiedTimelineStreamEvent( + sb *strings.Builder, + evt UnifiedTimelineEvent, + inTurn *bool, + streamColor func(streamStyleRenderer, string) string, + coloredMessageSnippet func(string, string, streamStyleRenderer, streamStyleRenderer) string, +) { + switch evt.Kind { + case TimelineKindAgentTurn, TimelineKindAssistantMessage, TimelineKindReasoning: + renderUnifiedTimelineStreamMessageEvent(sb, evt, inTurn, streamColor, coloredMessageSnippet) + case TimelineKindAgentToolStart, TimelineKindAgentToolDone, TimelineKindToolCall: + renderUnifiedTimelineStreamToolEvent(sb, evt, streamColor) + case TimelineKindNetworkAllowed, TimelineKindNetworkBlocked: + renderUnifiedTimelineStreamNetworkEvent(sb, evt, streamColor) + case TimelineKindDIFCFiltered, TimelineKindGuardPolicyBlocked, TimelineKindSteering: + renderUnifiedTimelineStreamPolicyEvent(sb, evt, streamColor) + default: + fmt.Fprintf(sb, " · [%s] %s %s\n", formatTimelineTime(evt), string(evt.Kind), timelineSourceLabel(evt.Source)) + } +} - for _, evt := range events { - ts := formatTimelineTime(evt) +func renderUnifiedTimelineStreamMessageEvent( + sb *strings.Builder, + evt UnifiedTimelineEvent, + inTurn *bool, + streamColor func(streamStyleRenderer, string) string, + coloredMessageSnippet func(string, string, streamStyleRenderer, streamStyleRenderer) string, +) { + switch evt.Kind { + case TimelineKindAgentTurn: + if *inTurn { + sb.WriteString("\n") + } + *inTurn = true + // Turn headers are bold purple (Command), timestamp muted. + turnLabel := streamColor(styles.Command, fmt.Sprintf("> Turn %d", evt.TurnIndex)) + tsLabel := streamColor(styles.LineNumber, "["+formatTimelineTime(evt)+"]") + fmt.Fprintf(sb, "%s %s\n", turnLabel, tsLabel) + // Show the first few lines of the user's message in muted style. + sb.WriteString(coloredMessageSnippet(evt.MessageContent, " ", styles.ContextLine, styles.LineNumber)) + case TimelineKindAssistantMessage: + icon := streamColor(styles.Info, timelineEventIcon(TimelineKindAssistantMessage)) + fmt.Fprintf(sb, " %s\n", icon) + // Show assistant response snippet in standard foreground, muted truncation. + sb.WriteString(coloredMessageSnippet(evt.MessageContent, " ", styles.ContextLine, styles.LineNumber)) + case TimelineKindReasoning: + icon := streamColor(styles.Verbose, timelineEventIcon(TimelineKindReasoning)) + fmt.Fprintf(sb, " %s\n", icon) + // Show reasoning snippet in muted/verbose style. + sb.WriteString(coloredMessageSnippet(evt.MessageContent, " ", styles.Verbose, styles.LineNumber)) + } +} - switch evt.Kind { - case TimelineKindAgentTurn: - if inTurn { - sb.WriteString("\n") - } - inTurn = true - // Turn headers are bold purple (Command), timestamp muted. - turnLabel := streamColor(styles.Command, fmt.Sprintf("> Turn %d", evt.TurnIndex)) - tsLabel := streamColor(styles.LineNumber, "["+ts+"]") - fmt.Fprintf(&sb, "%s %s\n", turnLabel, tsLabel) - // Show the first few lines of the user's message in muted style. - sb.WriteString(coloredMessageSnippet(evt.MessageContent, " ", styles.ContextLine, styles.LineNumber)) - - case TimelineKindAssistantMessage: - icon := streamColor(styles.Info, timelineEventIcon(TimelineKindAssistantMessage)) - fmt.Fprintf(&sb, " %s\n", icon) - // Show assistant response snippet in standard foreground, muted truncation. - sb.WriteString(coloredMessageSnippet(evt.MessageContent, " ", styles.ContextLine, styles.LineNumber)) - - case TimelineKindReasoning: - icon := streamColor(styles.Verbose, timelineEventIcon(TimelineKindReasoning)) - fmt.Fprintf(&sb, " %s\n", icon) - // Show reasoning snippet in muted/verbose style. - sb.WriteString(coloredMessageSnippet(evt.MessageContent, " ", styles.Verbose, styles.LineNumber)) - - case TimelineKindAgentToolStart: - detail := formatStreamToolDetail(evt.ServerName, evt.ToolName) - // Tool start is yellow progress indicator. - icon := streamColor(styles.Progress, timelineEventIcon(TimelineKindAgentToolStart)) - fmt.Fprintf(&sb, " %s %s\n", icon, detail) - - case TimelineKindAgentToolDone: - detail := formatStreamToolDetail(evt.ServerName, evt.ToolName) - status := evt.Status - if status == "" { - if evt.Success { - status = "success" - } else { - status = "error" - } - } - // Completion icon and status are green on success, red on error. - if evt.Success || status == "success" { - icon := streamColor(styles.Success, timelineEventIcon(TimelineKindAgentToolDone)) - statusColored := streamColor(styles.Success, status) - fmt.Fprintf(&sb, " %s %s %s\n", icon, detail, statusColored) - } else { - icon := streamColor(styles.Error, timelineEventIcon(TimelineKindAgentToolDone)) - statusColored := streamColor(styles.Error, status) - fmt.Fprintf(&sb, " %s %s %s\n", icon, detail, statusColored) - } - - case TimelineKindToolCall: - detail := formatStreamToolDetail(evt.ServerName, evt.ToolName) - icon := streamColor(styles.ServerName, timelineEventIcon(TimelineKindToolCall)) - suffix := "" - if evt.Duration > 0 { - suffix = " " + streamColor(styles.LineNumber, fmt.Sprintf("%.0fms", evt.Duration)) - } else if evt.Error != "" { - suffix = " " + streamColor(styles.Error, "error: "+stringutil.Truncate(evt.Error, streamMaxAnnotationLen)) - } - fmt.Fprintf(&sb, " %s %s%s\n", icon, detail, suffix) - - case TimelineKindNetworkAllowed: - method := "" - if evt.HTTPMethod != "" { - method = " " + streamColor(styles.LineNumber, evt.HTTPMethod) - } - icon := streamColor(styles.Info, timelineEventIcon(TimelineKindNetworkAllowed)) - fmt.Fprintf(&sb, " %s %s%s\n", icon, evt.Host, method) - - case TimelineKindNetworkBlocked: - method := "" - if evt.HTTPMethod != "" { - method = " " + streamColor(styles.LineNumber, evt.HTTPMethod) - } - icon := streamColor(styles.Error, timelineEventIcon(TimelineKindNetworkBlocked)) - blocked := streamColor(styles.Error, "[blocked]") - fmt.Fprintf(&sb, " %s %s%s %s\n", icon, evt.Host, method, blocked) - - case TimelineKindDIFCFiltered: - detail := formatStreamToolDetail(evt.ServerName, evt.ToolName) - icon := streamColor(styles.Warning, timelineEventIcon(TimelineKindDIFCFiltered)) - reason := "" - if evt.Reason != "" { - reason = " " + streamColor(styles.Warning, stringutil.Truncate(evt.Reason, streamMaxAnnotationLen)) - } - fmt.Fprintf(&sb, " %s %s%s\n", icon, detail, reason) - - case TimelineKindGuardPolicyBlocked: - detail := formatStreamToolDetail(evt.ServerName, evt.ToolName) - icon := streamColor(styles.Error, timelineEventIcon(TimelineKindGuardPolicyBlocked)) - annotation := evt.Reason - if annotation == "" { - annotation = evt.Error - } - annotationStr := "" - if annotation != "" { - annotationStr = " " + streamColor(styles.Error, stringutil.Truncate(annotation, streamMaxAnnotationLen)) - } - fmt.Fprintf(&sb, " %s %s%s\n", icon, detail, annotationStr) - - case TimelineKindSteering: - icon := streamColor(styles.Warning, timelineEventIcon(TimelineKindSteering)) - msg := stringutil.Truncate(evt.Reason, streamMaxAnnotationLen) - fmt.Fprintf(&sb, " %s %s\n", icon, msg) - - default: - fmt.Fprintf(&sb, " · [%s] %s %s\n", ts, string(evt.Kind), timelineSourceLabel(evt.Source)) +func renderUnifiedTimelineStreamToolEvent( + sb *strings.Builder, + evt UnifiedTimelineEvent, + streamColor func(streamStyleRenderer, string) string, +) { + detail := formatStreamToolDetail(evt.ServerName, evt.ToolName) + switch evt.Kind { + case TimelineKindAgentToolStart: + // Tool start is yellow progress indicator. + icon := streamColor(styles.Progress, timelineEventIcon(TimelineKindAgentToolStart)) + fmt.Fprintf(sb, " %s %s\n", icon, detail) + case TimelineKindAgentToolDone: + renderUnifiedTimelineStreamToolDone(sb, evt, detail, streamColor) + case TimelineKindToolCall: + icon := streamColor(styles.ServerName, timelineEventIcon(TimelineKindToolCall)) + suffix := "" + if evt.Duration > 0 { + suffix = " " + streamColor(styles.LineNumber, fmt.Sprintf("%.0fms", evt.Duration)) + } else if evt.Error != "" { + suffix = " " + streamColor(styles.Error, "error: "+stringutil.Truncate(evt.Error, streamMaxAnnotationLen)) } + fmt.Fprintf(sb, " %s %s%s\n", icon, detail, suffix) } +} - if inTurn { - sb.WriteString("\n") +func renderUnifiedTimelineStreamToolDone( + sb *strings.Builder, + evt UnifiedTimelineEvent, + detail string, + streamColor func(streamStyleRenderer, string) string, +) { + status := evt.Status + if status == "" { + if evt.Success { + status = "success" + } else { + status = "error" + } + } + // Completion icon and status are green on success, red on error. + if evt.Success || status == "success" { + icon := streamColor(styles.Success, timelineEventIcon(TimelineKindAgentToolDone)) + statusColored := streamColor(styles.Success, status) + fmt.Fprintf(sb, " %s %s %s\n", icon, detail, statusColored) + return } + icon := streamColor(styles.Error, timelineEventIcon(TimelineKindAgentToolDone)) + statusColored := streamColor(styles.Error, status) + fmt.Fprintf(sb, " %s %s %s\n", icon, detail, statusColored) +} - return sb.String() +func renderUnifiedTimelineStreamNetworkEvent( + sb *strings.Builder, + evt UnifiedTimelineEvent, + streamColor func(streamStyleRenderer, string) string, +) { + method := "" + if evt.HTTPMethod != "" { + method = " " + streamColor(styles.LineNumber, evt.HTTPMethod) + } + if evt.Kind == TimelineKindNetworkAllowed { + icon := streamColor(styles.Info, timelineEventIcon(TimelineKindNetworkAllowed)) + fmt.Fprintf(sb, " %s %s%s\n", icon, evt.Host, method) + return + } + icon := streamColor(styles.Error, timelineEventIcon(TimelineKindNetworkBlocked)) + blocked := streamColor(styles.Error, "[blocked]") + fmt.Fprintf(sb, " %s %s%s %s\n", icon, evt.Host, method, blocked) +} + +func renderUnifiedTimelineStreamPolicyEvent( + sb *strings.Builder, + evt UnifiedTimelineEvent, + streamColor func(streamStyleRenderer, string) string, +) { + detail := formatStreamToolDetail(evt.ServerName, evt.ToolName) + switch evt.Kind { + case TimelineKindDIFCFiltered: + icon := streamColor(styles.Warning, timelineEventIcon(TimelineKindDIFCFiltered)) + reason := "" + if evt.Reason != "" { + reason = " " + streamColor(styles.Warning, stringutil.Truncate(evt.Reason, streamMaxAnnotationLen)) + } + fmt.Fprintf(sb, " %s %s%s\n", icon, detail, reason) + case TimelineKindGuardPolicyBlocked: + renderUnifiedTimelineStreamGuardBlocked(sb, evt, detail, streamColor) + case TimelineKindSteering: + icon := streamColor(styles.Warning, timelineEventIcon(TimelineKindSteering)) + msg := stringutil.Truncate(evt.Reason, streamMaxAnnotationLen) + fmt.Fprintf(sb, " %s %s\n", icon, msg) + } +} + +func renderUnifiedTimelineStreamGuardBlocked( + sb *strings.Builder, + evt UnifiedTimelineEvent, + detail string, + streamColor func(streamStyleRenderer, string) string, +) { + icon := streamColor(styles.Error, timelineEventIcon(TimelineKindGuardPolicyBlocked)) + annotation := evt.Reason + if annotation == "" { + annotation = evt.Error + } + annotationStr := "" + if annotation != "" { + annotationStr = " " + streamColor(styles.Error, stringutil.Truncate(annotation, streamMaxAnnotationLen)) + } + fmt.Fprintf(sb, " %s %s%s\n", icon, detail, annotationStr) } // ─── Top-level renderer ─────────────────────────────────────────────────────── @@ -628,86 +686,109 @@ func renderUnifiedTimeline(events []UnifiedTimelineEvent) string { return "" } + counts := renderUnifiedTimelineCounts(events) + var sb strings.Builder + renderUnifiedTimelineSummary(&sb, len(events), counts) + + // Build the table rows using per-kind primitives. + rows := make([][]string, 0, len(events)) + for _, evt := range events { + rows = append(rows, renderTimelineEventRow(evt)) + } + + sb.WriteString(console.RenderTable(console.TableConfig{ + Title: "Event Timeline", + Headers: []string{"Time", "Src", "Kind", "Detail", "Status"}, + Rows: rows, + })) + + return sb.String() +} + +type renderUnifiedTimelineEventCounts struct { + gateway, firewall, agent int + toolCalls, difcFiltered, guardBlocked int + netAllowed, netBlocked, steering int + agentTurns, agentToolStarts, agentToolDones, messages, think int +} + +func renderUnifiedTimelineCounts(events []UnifiedTimelineEvent) renderUnifiedTimelineEventCounts { // Tally event counts for the summary header. - var gwCount, fwCount, agCount int - var toolCalls, difcFiltered, guardBlocked, netAllowed, netBlocked, steeringCount int - var agentTurns, agentToolStarts, agentToolDones, assistantMessages, reasoningCount int + var counts renderUnifiedTimelineEventCounts for _, evt := range events { switch evt.Source { case TimelineSourceGateway: - gwCount++ - switch evt.Kind { - case TimelineKindToolCall: - toolCalls++ - case TimelineKindDIFCFiltered: - difcFiltered++ - case TimelineKindGuardPolicyBlocked: - guardBlocked++ - } + renderUnifiedTimelineCountGateway(evt, &counts) case TimelineSourceFirewall: - fwCount++ - switch evt.Kind { - case TimelineKindNetworkAllowed: - netAllowed++ - case TimelineKindNetworkBlocked: - netBlocked++ - case TimelineKindSteering: - steeringCount++ - } + renderUnifiedTimelineCountFirewall(evt, &counts) case TimelineSourceAgent: - agCount++ - switch evt.Kind { - case TimelineKindAgentTurn: - agentTurns++ - case TimelineKindAgentToolStart: - agentToolStarts++ - case TimelineKindAgentToolDone: - agentToolDones++ - case TimelineKindAssistantMessage: - assistantMessages++ - case TimelineKindReasoning: - reasoningCount++ - } + renderUnifiedTimelineCountAgent(evt, &counts) } } + return counts +} - var sb strings.Builder +func renderUnifiedTimelineCountGateway(evt UnifiedTimelineEvent, counts *renderUnifiedTimelineEventCounts) { + counts.gateway++ + switch evt.Kind { + case TimelineKindToolCall: + counts.toolCalls++ + case TimelineKindDIFCFiltered: + counts.difcFiltered++ + case TimelineKindGuardPolicyBlocked: + counts.guardBlocked++ + } +} +func renderUnifiedTimelineCountFirewall(evt UnifiedTimelineEvent, counts *renderUnifiedTimelineEventCounts) { + counts.firewall++ + switch evt.Kind { + case TimelineKindNetworkAllowed: + counts.netAllowed++ + case TimelineKindNetworkBlocked: + counts.netBlocked++ + case TimelineKindSteering: + counts.steering++ + } +} + +func renderUnifiedTimelineCountAgent(evt UnifiedTimelineEvent, counts *renderUnifiedTimelineEventCounts) { + counts.agent++ + switch evt.Kind { + case TimelineKindAgentTurn: + counts.agentTurns++ + case TimelineKindAgentToolStart: + counts.agentToolStarts++ + case TimelineKindAgentToolDone: + counts.agentToolDones++ + case TimelineKindAssistantMessage: + counts.messages++ + case TimelineKindReasoning: + counts.think++ + } +} + +func renderUnifiedTimelineSummary(sb *strings.Builder, total int, counts renderUnifiedTimelineEventCounts) { sb.WriteString("\n") sb.WriteString(console.FormatInfoMessage("Unified MCP + Firewall + Agent Event Timeline")) sb.WriteString("\n\n") - - fmt.Fprintf(&sb, "Total Events : %d\n", len(events)) - if gwCount > 0 { - fmt.Fprintf(&sb, " Gateway : %d (tool_calls=%d, difc_filtered=%d, guard_blocked=%d)\n", - gwCount, toolCalls, difcFiltered, guardBlocked) - } - if fwCount > 0 { - fwDetail := fmt.Sprintf("allowed=%d, blocked=%d", netAllowed, netBlocked) - if steeringCount > 0 { - fwDetail += fmt.Sprintf(", steering=%d", steeringCount) + fmt.Fprintf(sb, "Total Events : %d\n", total) + if counts.gateway > 0 { + fmt.Fprintf(sb, " Gateway : %d (tool_calls=%d, difc_filtered=%d, guard_blocked=%d)\n", + counts.gateway, counts.toolCalls, counts.difcFiltered, counts.guardBlocked) + } + if counts.firewall > 0 { + fwDetail := fmt.Sprintf("allowed=%d, blocked=%d", counts.netAllowed, counts.netBlocked) + if counts.steering > 0 { + fwDetail += fmt.Sprintf(", steering=%d", counts.steering) } - fmt.Fprintf(&sb, " Firewall : %d (%s)\n", fwCount, fwDetail) + fmt.Fprintf(sb, " Firewall : %d (%s)\n", counts.firewall, fwDetail) } - if agCount > 0 { - fmt.Fprintf(&sb, " Agent : %d (turns=%d, tool_start=%d, tool_done=%d, messages=%d, reasoning=%d)\n", - agCount, agentTurns, agentToolStarts, agentToolDones, assistantMessages, reasoningCount) + if counts.agent > 0 { + fmt.Fprintf(sb, " Agent : %d (turns=%d, tool_start=%d, tool_done=%d, messages=%d, reasoning=%d)\n", + counts.agent, counts.agentTurns, counts.agentToolStarts, counts.agentToolDones, counts.messages, counts.think) } sb.WriteString("\n") - - // Build the table rows using per-kind primitives. - rows := make([][]string, 0, len(events)) - for _, evt := range events { - rows = append(rows, renderTimelineEventRow(evt)) - } - - sb.WriteString(console.RenderTable(console.TableConfig{ - Title: "Event Timeline", - Headers: []string{"Time", "Src", "Kind", "Detail", "Status"}, - Rows: rows, - })) - - return sb.String() } // displayUnifiedTimeline collects all JSONL events from every processed run, merges them diff --git a/pkg/cli/generate_action_metadata_command.go b/pkg/cli/generate_action_metadata_command.go index c60b1e26c2f..0c3a350fc87 100644 --- a/pkg/cli/generate_action_metadata_command.go +++ b/pkg/cli/generate_action_metadata_command.go @@ -64,89 +64,118 @@ func GenerateActionMetadataCommand() error { generateActionMetadataLog.Print("Starting action metadata generation") // Select target JavaScript files (simple ones for proof of concept) - targetFiles := []string{ - "noop.cjs", - "minimize_comment.cjs", - "close_issue.cjs", - "close_pull_request.cjs", - "close_discussion.cjs", - } + targetFiles := generateActionMetadataCommandTargetFiles() fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("🔍 Generating actions for %d JavaScript modules...", len(targetFiles)))) generatedCount := 0 generateActionMetadataLog.Printf("Processing %d target JavaScript files in %s", len(targetFiles), jsDir) for _, filename := range targetFiles { - jsPath := filepath.Join(jsDir, filename) - generateActionMetadataLog.Printf("Processing file: %s", filename) - - // Read file content directly from filesystem - contentBytes, err := os.ReadFile(jsPath) + generated, err := generateActionMetadataCommandProcessFile(jsDir, actionsDir, filename) if err != nil { - generateActionMetadataLog.Printf("Skipping %s: failed to read file: %v", filename, err) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("⚠ Skipping %s: %s", filename, err))) - continue + return err + } + if generated { + generatedCount++ } - content := string(contentBytes) + } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("\n📦 Processing: "+filename)) + generateActionMetadataLog.Printf("Action metadata generation complete: generated=%d, total=%d", generatedCount, len(targetFiles)) + if generatedCount == 0 { + return errors.New("no actions were generated") + } - // Extract metadata - metadata, err := extractActionMetadata(filename, content) - if err != nil { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("✗ Failed to extract metadata from %s: %s", filename, err))) - continue - } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("\n✨ Successfully generated %d action(s)", generatedCount))) + generateActionMetadataCommandPrintNextSteps() - // Create action directory - actionDir := filepath.Join(actionsDir, metadata.ActionName) - if err := os.MkdirAll(actionDir, constants.DirPermPublic); err != nil { - return fmt.Errorf("failed to create directory %s: %w", actionDir, err) - } + return nil +} - // Create src directory - srcDir := filepath.Join(actionDir, "src") - if err := os.MkdirAll(srcDir, constants.DirPermPublic); err != nil { - return fmt.Errorf("failed to create directory %s: %w", srcDir, err) - } +func generateActionMetadataCommandTargetFiles() []string { + return []string{ + "noop.cjs", + "minimize_comment.cjs", + "close_issue.cjs", + "close_pull_request.cjs", + "close_discussion.cjs", + } +} - // Generate action.yml - if err := generateActionYml(actionDir, metadata); err != nil { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage("✗ Failed to generate action.yml: "+err.Error())) - continue - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" ✓ Generated action.yml")) +func generateActionMetadataCommandProcessFile(jsDir, actionsDir, filename string) (bool, error) { + jsPath := filepath.Join(jsDir, filename) + generateActionMetadataLog.Printf("Processing file: %s", filename) - // Generate README.md - if err := generateReadme(actionDir, metadata); err != nil { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage("✗ Failed to generate README.md: "+err.Error())) - continue - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" ✓ Generated README.md")) + // Read file content directly from filesystem + contentBytes, err := os.ReadFile(jsPath) + if err != nil { + generateActionMetadataLog.Printf("Skipping %s: failed to read file: %v", filename, err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("⚠ Skipping %s: %s", filename, err))) + return false, nil + } + content := string(contentBytes) - // Copy source file with owner-only read/write permissions (0600) for security best practices - srcPath := filepath.Join(srcDir, "index.js") - if err := os.WriteFile(srcPath, []byte(content), constants.FilePermSensitive); err != nil { - return fmt.Errorf("failed to write source file: %w", err) - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" ✓ Copied source to src/index.js")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("\n📦 Processing: "+filename)) + metadata, err := extractActionMetadata(filename, content) + if err != nil { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("✗ Failed to extract metadata from %s: %s", filename, err))) + return false, nil + } - generatedCount++ + actionDir, srcDir, err := generateActionMetadataCommandCreateDirs(actionsDir, metadata) + if err != nil { + return false, err + } + if !generateActionMetadataCommandWriteMetadata(actionDir, metadata) { + return false, nil } - generateActionMetadataLog.Printf("Action metadata generation complete: generated=%d, total=%d", generatedCount, len(targetFiles)) - if generatedCount == 0 { - return errors.New("no actions were generated") + // Copy source file with owner-only read/write permissions (0600) for security best practices + srcPath := filepath.Join(srcDir, "index.js") + if err := os.WriteFile(srcPath, []byte(content), constants.FilePermSensitive); err != nil { + return false, fmt.Errorf("failed to write source file: %w", err) + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" ✓ Copied source to src/index.js")) + return true, nil +} + +func generateActionMetadataCommandCreateDirs(actionsDir string, metadata *ActionMetadata) (string, string, error) { + // Create action directory + actionDir := filepath.Join(actionsDir, metadata.ActionName) + if err := os.MkdirAll(actionDir, constants.DirPermPublic); err != nil { + return "", "", fmt.Errorf("failed to create directory %s: %w", actionDir, err) } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("\n✨ Successfully generated %d action(s)", generatedCount))) + // Create src directory + srcDir := filepath.Join(actionDir, "src") + if err := os.MkdirAll(srcDir, constants.DirPermPublic); err != nil { + return "", "", fmt.Errorf("failed to create directory %s: %w", srcDir, err) + } + return actionDir, srcDir, nil +} + +func generateActionMetadataCommandWriteMetadata(actionDir string, metadata *ActionMetadata) bool { + // Generate action.yml + if err := generateActionYml(actionDir, metadata); err != nil { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage("✗ Failed to generate action.yml: "+err.Error())) + return false + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" ✓ Generated action.yml")) + + // Generate README.md + if err := generateReadme(actionDir, metadata); err != nil { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage("✗ Failed to generate README.md: "+err.Error())) + return false + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" ✓ Generated README.md")) + return true +} + +func generateActionMetadataCommandPrintNextSteps() { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("\nNext steps:")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" 1. Review the generated action.yml and README.md files")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" 2. Update dependency mapping in pkg/cli/actions_build_command.go")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" 3. Run 'make actions-build' to build the actions")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage(" 4. Test the actions in a workflow")) - - return nil } // extractActionMetadata extracts metadata from a JavaScript file @@ -380,55 +409,82 @@ func generateReadme(actionDir string, metadata *ActionMetadata) error { fmt.Fprintf(&content, "This action is generated from `pkg/workflow/js/%s` and provides functionality ", metadata.Filename) content.WriteString("for GitHub Agentic Workflows.\n\n") + generateReadmeUsage(&content, metadata) + generateReadmeInputs(&content, metadata) + generateReadmeOutputs(&content, metadata) + generateReadmeDependencies(&content, metadata) + generateReadmeDevelopment(&content, metadata) + + // Write to file with owner-only read/write permissions (0600) for security best practices + readmePath := filepath.Join(actionDir, "README.md") + if err := os.WriteFile(readmePath, []byte(content.String()), constants.FilePermSensitive); err != nil { + return fmt.Errorf("failed to write README.md: %w", err) + } + + return nil +} + +func generateReadmeUsage(content *strings.Builder, metadata *ActionMetadata) { // Usage section content.WriteString("## Usage\n\n") content.WriteString("```yaml\n") - fmt.Fprintf(&content, "- uses: ./actions/%s\n", metadata.ActionName) + fmt.Fprintf(content, "- uses: ./actions/%s\n", metadata.ActionName) if len(metadata.Inputs) > 0 { content.WriteString(" with:\n") for _, input := range metadata.Inputs { - fmt.Fprintf(&content, " %s: 'value' # %s\n", input.Name, input.Description) + fmt.Fprintf(content, " %s: 'value' # %s\n", input.Name, input.Description) } } content.WriteString("```\n\n") +} +func generateReadmeInputs(content *strings.Builder, metadata *ActionMetadata) { // Inputs section - if len(metadata.Inputs) > 0 { - content.WriteString("## Inputs\n\n") - for _, input := range metadata.Inputs { - fmt.Fprintf(&content, "### `%s`\n\n", input.Name) - fmt.Fprintf(&content, "**Description**: %s\n\n", input.Description) - fmt.Fprintf(&content, "**Required**: %t\n\n", input.Required) - if input.Default != "" { - fmt.Fprintf(&content, "**Default**: `%s`\n\n", input.Default) - } + if len(metadata.Inputs) == 0 { + return + } + content.WriteString("## Inputs\n\n") + for _, input := range metadata.Inputs { + fmt.Fprintf(content, "### `%s`\n\n", input.Name) + fmt.Fprintf(content, "**Description**: %s\n\n", input.Description) + fmt.Fprintf(content, "**Required**: %t\n\n", input.Required) + if input.Default != "" { + fmt.Fprintf(content, "**Default**: `%s`\n\n", input.Default) } } +} +func generateReadmeOutputs(content *strings.Builder, metadata *ActionMetadata) { // Outputs section - if len(metadata.Outputs) > 0 { - content.WriteString("## Outputs\n\n") - for _, output := range metadata.Outputs { - fmt.Fprintf(&content, "### `%s`\n\n", output.Name) - fmt.Fprintf(&content, "**Description**: %s\n\n", output.Description) - } + if len(metadata.Outputs) == 0 { + return + } + content.WriteString("## Outputs\n\n") + for _, output := range metadata.Outputs { + fmt.Fprintf(content, "### `%s`\n\n", output.Name) + fmt.Fprintf(content, "**Description**: %s\n\n", output.Description) } +} +func generateReadmeDependencies(content *strings.Builder, metadata *ActionMetadata) { // Dependencies section - if len(metadata.Dependencies) > 0 { - content.WriteString("## Dependencies\n\n") - content.WriteString("This action depends on the following JavaScript modules:\n\n") - for _, dep := range metadata.Dependencies { - fmt.Fprintf(&content, "- `%s`\n", dep) - } - content.WriteString("\n") + if len(metadata.Dependencies) == 0 { + return } + content.WriteString("## Dependencies\n\n") + content.WriteString("This action depends on the following JavaScript modules:\n\n") + for _, dep := range metadata.Dependencies { + fmt.Fprintf(content, "- `%s`\n", dep) + } + content.WriteString("\n") +} +func generateReadmeDevelopment(content *strings.Builder, metadata *ActionMetadata) { // Development section content.WriteString("## Development\n\n") content.WriteString("### Building\n\n") content.WriteString("To build this action, you need to:\n\n") - fmt.Fprintf(&content, "1. Update the dependency mapping in `pkg/cli/actions_build_command.go` for `%s`\n", metadata.ActionName) + fmt.Fprintf(content, "1. Update the dependency mapping in `pkg/cli/actions_build_command.go` for `%s`\n", metadata.ActionName) content.WriteString("2. Run `make actions-build` to bundle the JavaScript dependencies\n") content.WriteString("3. The bundled `index.js` will be generated and committed\n\n") @@ -439,18 +495,10 @@ func generateReadme(actionDir string, metadata *ActionMetadata) error { content.WriteString(" test:\n") content.WriteString(" runs-on: ubuntu-latest\n") content.WriteString(" steps:\n") - fmt.Fprintf(&content, " - uses: ./actions/%s\n", metadata.ActionName) + fmt.Fprintf(content, " - uses: ./actions/%s\n", metadata.ActionName) content.WriteString("```\n\n") // License content.WriteString("## License\n\n") content.WriteString("MIT\n") - - // Write to file with owner-only read/write permissions (0600) for security best practices - readmePath := filepath.Join(actionDir, "README.md") - if err := os.WriteFile(readmePath, []byte(content.String()), constants.FilePermSensitive); err != nil { - return fmt.Errorf("failed to write README.md: %w", err) - } - - return nil } diff --git a/pkg/cli/git.go b/pkg/cli/git.go index 8bb45af9acb..7754ac7e087 100644 --- a/pkg/cli/git.go +++ b/pkg/cli/git.go @@ -368,31 +368,7 @@ func ensureGitAttributes() (bool, error) { modified := false for _, required := range requiredEntries { - found := false - for i, line := range lines { - trimmedLine := strings.TrimSpace(line) - if trimmedLine == required { - found = true - break - } - // Check for old format entries that need updating - if strings.HasPrefix(trimmedLine, constants.WorkflowsLockYmlGlob) && required == lockYmlEntry { - gitLog.Print("Updating old .gitattributes entry format") - lines[i] = lockYmlEntry - found = true - modified = true - break - } - } - - if !found { - gitLog.Printf("Adding new .gitattributes entry: %s", required) - if len(lines) > 0 && lines[len(lines)-1] != "" { - lines = append(lines, "") - } - lines = append(lines, required) - modified = true - } + lines, modified = ensureGitAttributesRequiredEntry(lines, required, lockYmlEntry, modified) } if !modified { @@ -411,6 +387,35 @@ func ensureGitAttributes() (bool, error) { return true, nil } +func ensureGitAttributesRequiredEntry(lines []string, required, lockYmlEntry string, modified bool) ([]string, bool) { + found := false + for i, line := range lines { + trimmedLine := strings.TrimSpace(line) + if trimmedLine == required { + found = true + break + } + // Check for old format entries that need updating + if strings.HasPrefix(trimmedLine, constants.WorkflowsLockYmlGlob) && required == lockYmlEntry { + gitLog.Print("Updating old .gitattributes entry format") + lines[i] = lockYmlEntry + found = true + modified = true + break + } + } + + if !found { + gitLog.Printf("Adding new .gitattributes entry: %s", required) + if len(lines) > 0 && lines[len(lines)-1] != "" { + lines = append(lines, "") + } + lines = append(lines, required) + modified = true + } + return lines, modified +} + // stageGitAttributesIfChanged stages .gitattributes if it was modified func stageGitAttributesIfChanged() error { gitRoot, err := gitutil.FindGitRoot() @@ -599,62 +604,83 @@ func checkWorkflowFileStatus(workflowPath string) (*WorkflowFileStatus, error) { } // Make path relative to git root if it's absolute - var relPath string - if filepath.IsAbs(workflowPath) { - var err error - relPath, err = filepath.Rel(gitRoot, workflowPath) - if err != nil { - gitLog.Printf("Failed to make path relative: %v", err) - relPath = workflowPath - } - } else { - relPath = workflowPath - } + relPath := checkWorkflowFileStatusRelativePath(workflowPath, gitRoot) gitLog.Printf("Checking git status for: %s", relPath) // Check for modified or staged changes using git status --porcelain - cmd := exec.Command("git", "-C", gitRoot, "status", "--porcelain", relPath) - output, err := cmd.Output() - if err != nil { + if err := checkWorkflowFileStatusLocalChanges(status, gitRoot, relPath); err != nil { gitLog.Printf("Failed to check git status: %v", err) return status, nil // Ignore error, return empty status } + // Check for unpushed commits that affect this file + if err := checkWorkflowFileStatusUnpushed(status, gitRoot, relPath); err != nil { + gitLog.Printf("Failed to check unpushed commits: %v", err) + return status, nil // Ignore error, return current status + } + + return status, nil +} + +func checkWorkflowFileStatusRelativePath(workflowPath, gitRoot string) string { + if !filepath.IsAbs(workflowPath) { + return workflowPath + } + relPath, err := filepath.Rel(gitRoot, workflowPath) + if err != nil { + gitLog.Printf("Failed to make path relative: %v", err) + return workflowPath + } + return relPath +} + +func checkWorkflowFileStatusLocalChanges(status *WorkflowFileStatus, gitRoot, relPath string) error { + cmd := exec.Command("git", "-C", gitRoot, "status", "--porcelain", relPath) + output, err := cmd.Output() + if err != nil { + return err + } statusOutput := string(output) // Don't trim - the leading space is significant! - if statusOutput != "" { - gitLog.Printf("Git status output: %q", statusOutput) - // Parse the status line (format: XY filename) - // X = index (staged) status, Y = working tree (unstaged) status - // The format is exactly 2 characters followed by a space and then the filename - if len(statusOutput) >= 2 { - stagedStatus := statusOutput[0] - unstagedStatus := statusOutput[1] - - // Check if file is staged (first character is not space or ?) - if stagedStatus != ' ' && stagedStatus != '?' { - status.IsStaged = true - gitLog.Print("File has staged changes") - } + if statusOutput == "" { + return nil + } + gitLog.Printf("Git status output: %q", statusOutput) + // Parse the status line (format: XY filename) + // X = index (staged) status, Y = working tree (unstaged) status + // The format is exactly 2 characters followed by a space and then the filename + if len(statusOutput) >= 2 { + checkWorkflowFileStatusParsePorcelain(status, statusOutput) + } + return nil +} - // Check if file is modified in working tree (second character is M or other modification indicators) - if unstagedStatus == 'M' || unstagedStatus == 'D' || unstagedStatus == 'A' { - status.IsModified = true - gitLog.Print("File has unstaged modifications") - } - } +func checkWorkflowFileStatusParsePorcelain(status *WorkflowFileStatus, statusOutput string) { + stagedStatus := statusOutput[0] + unstagedStatus := statusOutput[1] + + // Check if file is staged (first character is not space or ?) + if stagedStatus != ' ' && stagedStatus != '?' { + status.IsStaged = true + gitLog.Print("File has staged changes") } - // Check for unpushed commits that affect this file + // Check if file is modified in working tree (second character is M or other modification indicators) + if unstagedStatus == 'M' || unstagedStatus == 'D' || unstagedStatus == 'A' { + status.IsModified = true + gitLog.Print("File has unstaged modifications") + } +} + +func checkWorkflowFileStatusUnpushed(status *WorkflowFileStatus, gitRoot, relPath string) error { // First, check if there's a remote tracking branch - cmd = exec.Command("git", "-C", gitRoot, "rev-parse", "--abbrev-ref", "@{u}") - output, err = cmd.Output() + cmd := exec.Command("git", "-C", gitRoot, "rev-parse", "--abbrev-ref", "@{u}") + output, err := cmd.Output() if err != nil { // No upstream branch configured, skip unpushed commits check gitLog.Print("No upstream branch configured") - return status, nil + return nil } - upstream := strings.TrimSpace(string(output)) gitLog.Printf("Upstream branch: %s", upstream) @@ -662,16 +688,13 @@ func checkWorkflowFileStatus(workflowPath string) (*WorkflowFileStatus, error) { cmd = exec.Command("git", "-C", gitRoot, "log", upstream+"..HEAD", "--oneline", "--", relPath) output, err = cmd.Output() if err != nil { - gitLog.Printf("Failed to check unpushed commits: %v", err) - return status, nil // Ignore error, return current status + return err } - if strings.TrimSpace(string(output)) != "" { status.HasUnpushedCommits = true gitLog.Print("File has unpushed commits") } - - return status, nil + return nil } // stageAllChanges stages all modified files using git add -A diff --git a/pkg/cli/health_command.go b/pkg/cli/health_command.go index 9e04c1ab201..1f564cf338b 100644 --- a/pkg/cli/health_command.go +++ b/pkg/cli/health_command.go @@ -99,29 +99,9 @@ func RunHealth(config HealthConfig) error { return fmt.Errorf("invalid days value: %d. Must be 7, 30, or 90", config.Days) } - // workflowAPIName is used for gh run list API calls. Using the lock file name - // (e.g. "smoke-copilot.lock.yml") is more reliable than the display name because - // the GitHub CLI matches by filename directly, avoiding "workflow not found" errors - // that can occur when the workflow's display name doesn't match the registry. - var workflowAPIName string - - // Resolve workflow name from workflow ID to GitHub Actions display name - if config.WorkflowName != "" { - resolvedName, err := workflow.FindWorkflowName(config.WorkflowName) - if err != nil { - return fmt.Errorf("workflow '%s' not found: %w", config.WorkflowName, err) - } - - lockFileName, lockErr := workflow.GetWorkflowLockFileName(config.WorkflowName) - if lockErr == nil { - workflowAPIName = lockFileName - } else { - // Fall back to resolved display name if lock file lookup fails - workflowAPIName = resolvedName - } - - healthLog.Printf("Resolved workflow name: %s -> %s (API name: %s)", config.WorkflowName, resolvedName, workflowAPIName) - config.WorkflowName = resolvedName + workflowAPIName, err := runHealthResolveWorkflowAPIName(&config) + if err != nil { + return err } // Calculate start date @@ -134,31 +114,11 @@ func RunHealth(config HealthConfig) error { // Fetch workflow runs from GitHub runs, err := fetchWorkflowRuns(workflowAPIName, startDate, config.RepoOverride, config.Verbose) if err != nil { - if gitutil.IsRateLimitError(err.Error()) { - // Rate limiting is a transient infrastructure condition, not a code error. - // Warn and exit cleanly so CI jobs are not marked as failed. - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Skipping health check: GitHub API rate limit exceeded")) - if config.JSONOutput && config.WorkflowName != "" { - // Emit an empty-run JSON structure so callers can still parse the output. - return displayDetailedHealth(nil, config) - } - return nil - } - return fmt.Errorf("failed to fetch workflow runs: %w", err) + return runHealthHandleFetchError(err, config) } if len(runs) == 0 { - if config.WorkflowName != "" { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No runs found for workflow '%s' in the last %d days", config.WorkflowName, config.Days))) - // When JSON output is requested for a specific workflow, still output a valid - // zero-run JSON structure so callers can parse the result programmatically. - if config.JSONOutput { - return displayDetailedHealth(runs, config) - } - } else { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No workflow runs found in the last %d days", config.Days))) - } - return nil + return runHealthHandleNoRuns(runs, config) } if config.WorkflowName != "" { @@ -170,6 +130,59 @@ func RunHealth(config HealthConfig) error { return displayHealthSummary(runs, config) } +func runHealthResolveWorkflowAPIName(config *HealthConfig) (string, error) { + // workflowAPIName is used for gh run list API calls. Using the lock file name + // (e.g. "smoke-copilot.lock.yml") is more reliable than the display name because + // the GitHub CLI matches by filename directly, avoiding "workflow not found" errors + // that can occur when the workflow's display name doesn't match the registry. + if config.WorkflowName == "" { + return "", nil + } + + // Resolve workflow name from workflow ID to GitHub Actions display name + resolvedName, err := workflow.FindWorkflowName(config.WorkflowName) + if err != nil { + return "", fmt.Errorf("workflow '%s' not found: %w", config.WorkflowName, err) + } + + workflowAPIName := resolvedName + if lockFileName, lockErr := workflow.GetWorkflowLockFileName(config.WorkflowName); lockErr == nil { + workflowAPIName = lockFileName + } + + healthLog.Printf("Resolved workflow name: %s -> %s (API name: %s)", config.WorkflowName, resolvedName, workflowAPIName) + config.WorkflowName = resolvedName + return workflowAPIName, nil +} + +func runHealthHandleFetchError(err error, config HealthConfig) error { + if gitutil.IsRateLimitError(err.Error()) { + // Rate limiting is a transient infrastructure condition, not a code error. + // Warn and exit cleanly so CI jobs are not marked as failed. + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Skipping health check: GitHub API rate limit exceeded")) + if config.JSONOutput && config.WorkflowName != "" { + // Emit an empty-run JSON structure so callers can still parse the output. + return displayDetailedHealth(nil, config) + } + return nil + } + return fmt.Errorf("failed to fetch workflow runs: %w", err) +} + +func runHealthHandleNoRuns(runs []WorkflowRun, config HealthConfig) error { + if config.WorkflowName != "" { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No runs found for workflow '%s' in the last %d days", config.WorkflowName, config.Days))) + // When JSON output is requested for a specific workflow, still output a valid + // zero-run JSON structure so callers can parse the result programmatically. + if config.JSONOutput { + return displayDetailedHealth(runs, config) + } + return nil + } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No workflow runs found in the last %d days", config.Days))) + return nil +} + // fetchWorkflowRuns fetches workflow runs from GitHub for the specified time period func fetchWorkflowRuns(workflowName, startDate, repoOverride string, verbose bool) ([]WorkflowRun, error) { healthLog.Printf("Fetching workflow runs: workflow=%s, startDate=%s", workflowName, startDate) diff --git a/pkg/cli/health_metrics.go b/pkg/cli/health_metrics.go index c3c15d8125c..720629bf78c 100644 --- a/pkg/cli/health_metrics.go +++ b/pkg/cli/health_metrics.go @@ -75,60 +75,70 @@ func CalculateWorkflowHealth(workflowName string, runs []WorkflowRun, threshold } } - // Accumulate success/failure counts and numerical metrics. - successCount := 0 - failureCount := 0 - var durationStats, tokenStats stats.StatVar - var totalTokens int - - for _, run := range runs { - if run.Conclusion == "success" { - successCount++ - } else if isFailureConclusion(run.Conclusion) { - failureCount++ - } - totalTokens += run.TokenUsage - durationStats.Add(float64(run.Duration)) - tokenStats.Add(float64(run.TokenUsage)) - } - - totalRuns := len(runs) - successRate := safePercent(successCount, totalRuns) - - avgDuration := time.Duration(durationStats.Mean()) - avgTokens := int(tokenStats.Mean()) + metrics := calculateWorkflowHealthMetrics(runs) // Calculate trend trend := calculateTrend(runs) // Format display values - displayRate := fmt.Sprintf("%.0f%% (%d/%d)", successRate, successCount, totalRuns) - displayDur := timeutil.FormatDuration(avgDuration) - displayTokens := console.FormatTokens(avgTokens) + displayRate := fmt.Sprintf("%.0f%% (%d/%d)", metrics.successRate, metrics.successCount, metrics.totalRuns) + displayDur := timeutil.FormatDuration(metrics.avgDuration) + displayTokens := console.FormatTokens(metrics.avgTokens) - belowThreshold := successRate < threshold + belowThreshold := metrics.successRate < threshold health := WorkflowHealth{ WorkflowName: workflowName, - TotalRuns: totalRuns, - SuccessCount: successCount, - FailureCount: failureCount, - SuccessRate: successRate, + TotalRuns: metrics.totalRuns, + SuccessCount: metrics.successCount, + FailureCount: metrics.failureCount, + SuccessRate: metrics.successRate, DisplayRate: displayRate, Trend: trend.String(), - AvgDuration: avgDuration, + AvgDuration: metrics.avgDuration, DisplayDur: displayDur, - TotalTokens: totalTokens, - AvgTokens: avgTokens, + TotalTokens: metrics.totalTokens, + AvgTokens: metrics.avgTokens, DisplayTokens: displayTokens, BelowThresh: belowThreshold, } - healthMetricsLog.Printf("Health calculated: workflow=%s, successRate=%.2f%%, trend=%s", workflowName, successRate, trend.String()) + healthMetricsLog.Printf("Health calculated: workflow=%s, successRate=%.2f%%, trend=%s", workflowName, metrics.successRate, trend.String()) return health } +type calculateWorkflowHealthMetricsResult struct { + successCount int + failureCount int + totalRuns int + totalTokens int + successRate float64 + avgDuration time.Duration + avgTokens int +} + +func calculateWorkflowHealthMetrics(runs []WorkflowRun) calculateWorkflowHealthMetricsResult { + // Accumulate success/failure counts and numerical metrics. + var result calculateWorkflowHealthMetricsResult + var durationStats, tokenStats stats.StatVar + for _, run := range runs { + if run.Conclusion == "success" { + result.successCount++ + } else if isFailureConclusion(run.Conclusion) { + result.failureCount++ + } + result.totalTokens += run.TokenUsage + durationStats.Add(float64(run.Duration)) + tokenStats.Add(float64(run.TokenUsage)) + } + result.totalRuns = len(runs) + result.successRate = safePercent(result.successCount, result.totalRuns) + result.avgDuration = time.Duration(durationStats.Mean()) + result.avgTokens = int(tokenStats.Mean()) + return result +} + // calculateTrend determines the trend direction based on recent vs older runs func calculateTrend(runs []WorkflowRun) TrendDirection { if len(runs) < 4 { diff --git a/pkg/cli/import_url_fetcher.go b/pkg/cli/import_url_fetcher.go index 6a09e6744a0..85163759a39 100644 --- a/pkg/cli/import_url_fetcher.go +++ b/pkg/cli/import_url_fetcher.go @@ -70,6 +70,39 @@ func FetchImportURL(ctx context.Context, rawURL string, opts FetchOptions) (*Fet importURLFetcherLog.Printf("HEAD result: content_type=%q ok=%v", ct, headOK) // Always perform the GET to retrieve the body. + resp, err := fetchImportURLGet(ctx, client, rawURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + importURLFetcherLog.Printf("GET response: status=%d content-type=%q", resp.StatusCode, resp.Header.Get("Content-Type")) + + if err := fetchImportURLCheckStatus(resp); err != nil { + return nil, err + } + + // Prefer Content-Type obtained via HEAD; fall back to GET response headers. + if !headOK || ct == "" { + ct = canonicalContentType(resp.Header.Get("Content-Type")) + } + + // Guard against oversized responses. + body, err := fetchImportURLReadBody(resp) + if err != nil { + return nil, err + } + + importURLFetcherLog.Printf("Fetched import URL: content_type=%s, bytes=%d", ct, len(body)) + + return &FetchedResource{ + URL: rawURL, + ContentType: ct, + Body: body, + }, nil +} + +func fetchImportURLGet(ctx context.Context, client *http.Client, rawURL string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) if err != nil { return nil, fmt.Errorf("failed to build GET request: %w", err) @@ -77,37 +110,33 @@ func FetchImportURL(ctx context.Context, rawURL string, opts FetchOptions) (*Fet attachImportAuthHeader(req, rawURL) logRequestVerbose(req) - importURLFetcherLog.Printf("Sending GET request to %s", req.URL.Host) resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to fetch URL: %w", sanitizeHTTPError(err)) } - defer resp.Body.Close() - - importURLFetcherLog.Printf("GET response: status=%d content-type=%q", resp.StatusCode, resp.Header.Get("Content-Type")) + return resp, nil +} +func fetchImportURLCheckStatus(resp *http.Response) error { switch resp.StatusCode { case http.StatusUnauthorized, http.StatusForbidden: - return nil, errors.New(console.FormatErrorMessage( + return errors.New(console.FormatErrorMessage( fmt.Sprintf("access denied (HTTP %d). Check that the URL is accessible or set an auth token.", resp.StatusCode), )) case http.StatusNotFound: - return nil, errors.New(console.FormatErrorMessage("URL not found (HTTP 404)")) + return errors.New(console.FormatErrorMessage("URL not found (HTTP 404)")) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { logResponseBodyVerbose(resp) - return nil, errors.New(console.FormatErrorMessage( + return errors.New(console.FormatErrorMessage( fmt.Sprintf("unexpected HTTP %d response from server", resp.StatusCode), )) } + return nil +} - // Prefer Content-Type obtained via HEAD; fall back to GET response headers. - if !headOK || ct == "" { - ct = canonicalContentType(resp.Header.Get("Content-Type")) - } - - // Guard against oversized responses. +func fetchImportURLReadBody(resp *http.Response) ([]byte, error) { limited := io.LimitReader(resp.Body, int64(importURLMaxBytes)+1) body, err := io.ReadAll(limited) if err != nil { @@ -118,14 +147,7 @@ func FetchImportURL(ctx context.Context, rawURL string, opts FetchOptions) (*Fet fmt.Sprintf("response body exceeds size limit (%s)", console.FormatFileSize(importURLMaxBytes)), )) } - - importURLFetcherLog.Printf("Fetched import URL: content_type=%s, bytes=%d", ct, len(body)) - - return &FetchedResource{ - URL: rawURL, - ContentType: ct, - Body: body, - }, nil + return body, nil } // tryHead issues a HEAD request and returns the canonicalized Content-Type and whether diff --git a/pkg/cli/importpath.go b/pkg/cli/importpath.go index 3f3dd0f60d3..0a46ecb701b 100644 --- a/pkg/cli/importpath.go +++ b/pkg/cli/importpath.go @@ -102,44 +102,15 @@ func resolveImportPath(importPath, baseDir string, opts importPathResolverOpts) // 3. Absolute paths (starting with "/"). if withoutLeadingSlash, hasLeadingSlash := strings.CutPrefix(importPath, "/"); hasLeadingSlash { - if opts.RepoRelativeAbsolute { - // Strip "/" and return as a repo-relative string (no disk lookup). - return withoutLeadingSlash - } - if !opts.UseParserFallback { - // Resolve against the git root on disk. - gitRoot := opts.GitRoot - if gitRoot == "" { - var err error - gitRoot, err = gitutil.FindGitRoot() - if err != nil { - return "" - } - } - return filepath.Join(gitRoot, withoutLeadingSlash) + if resolved, handled := resolveImportPathAbsolute(withoutLeadingSlash, opts); handled { + return resolved } // UseParserFallback=true: fall through to parser resolution below. } // 4. Relative paths (and "/" prefix paths in parser-fallback mode). if opts.UseParserFallback { - // Only attempt stat for non-absolute paths (absolute paths go directly to parser, - // matching the original dependency_graph.go behaviour). - if !strings.HasPrefix(importPath, "/") { - absPath := filepath.Join(baseDir, importPath) - if fileutil.FileExists(absPath) { - importPathLog.Printf("resolveImportPath: resolved %q via stat to %q", importPath, absPath) - return absPath - } - } - importCache := parser.NewImportCache(opts.ParserGitRoot) - fullPath, err := parser.ResolveIncludePath(importPath, baseDir, importCache) - if err != nil { - importPathLog.Printf("resolveImportPath: parser fallback failed for %q: %v", importPath, err) - return "" - } - importPathLog.Printf("resolveImportPath: parser fallback resolved %q to %q", importPath, fullPath) - return fullPath + return resolveImportPathParserFallback(importPath, baseDir, opts) } // Direct resolution with optional path normalisation. @@ -149,3 +120,44 @@ func resolveImportPath(importPath, baseDir string, opts importPathResolverOpts) } return result } + +func resolveImportPathAbsolute(withoutLeadingSlash string, opts importPathResolverOpts) (string, bool) { + if opts.RepoRelativeAbsolute { + // Strip "/" and return as a repo-relative string (no disk lookup). + return withoutLeadingSlash, true + } + if opts.UseParserFallback { + return "", false + } + + // Resolve against the git root on disk. + gitRoot := opts.GitRoot + if gitRoot == "" { + var err error + gitRoot, err = gitutil.FindGitRoot() + if err != nil { + return "", true + } + } + return filepath.Join(gitRoot, withoutLeadingSlash), true +} + +func resolveImportPathParserFallback(importPath, baseDir string, opts importPathResolverOpts) string { + // Only attempt stat for non-absolute paths (absolute paths go directly to parser, + // matching the original dependency_graph.go behaviour). + if !strings.HasPrefix(importPath, "/") { + absPath := filepath.Join(baseDir, importPath) + if fileutil.FileExists(absPath) { + importPathLog.Printf("resolveImportPath: resolved %q via stat to %q", importPath, absPath) + return absPath + } + } + importCache := parser.NewImportCache(opts.ParserGitRoot) + fullPath, err := parser.ResolveIncludePath(importPath, baseDir, importCache) + if err != nil { + importPathLog.Printf("resolveImportPath: parser fallback failed for %q: %v", importPath, err) + return "" + } + importPathLog.Printf("resolveImportPath: parser fallback resolved %q to %q", importPath, fullPath) + return fullPath +} diff --git a/pkg/cli/imports.go b/pkg/cli/imports.go index 63d55aea51c..24e428d61aa 100644 --- a/pkg/cli/imports.go +++ b/pkg/cli/imports.go @@ -56,68 +56,25 @@ func processImportsWithWorkflowSpec(content string, workflow *WorkflowSpec, comm return content, nil // No imports field, return original content } - // processImportPaths converts a list of raw import paths to workflowspec format. - // Paths that already use the workflowspec format (contain "@") are left unchanged. - // When localWorkflowDir is set, relative paths whose files exist locally are also - // preserved as-is so that consumers who have copied shared files into their own repo - // are not forced onto cross-repo references after every `gh aw update`. - processImportPaths := func(imports []string) []string { - processed := make([]string, 0, len(imports)) - for _, importPath := range imports { - if isWorkflowSpecFormat(importPath) { - importsLog.Printf("Import already in workflowspec format: %s", importPath) - processed = append(processed, importPath) - continue - } - // Preserve relative paths whose files exist in the local workflow directory. - // Absolute paths (starting with "/") are not checked — they are always resolved - // relative to the repo root and cannot be reliably tested here. - if localWorkflowDir != "" && !strings.HasPrefix(importPath, "/") { - if isLocalFileForUpdate(localWorkflowDir, importPath) { - importsLog.Printf("Import path exists locally, preserving relative path: %s", importPath) - processed = append(processed, importPath) - continue - } - } - resolvedPath := resolveImportPath(importPath, filepath.Dir(workflow.WorkflowPath), importPathImportsOpts) - importsLog.Printf("Resolved import path: %s -> %s (workflow: %s)", importPath, resolvedPath, workflow.WorkflowPath) - workflowSpec := buildWorkflowSpecRef(workflow.RepoSlug, resolvedPath, commitSHA, workflow.Version) - importsLog.Printf("Converted import: %s -> %s", importPath, workflowSpec) - processed = append(processed, workflowSpec) - } - return processed - } - - // collectStringImports extracts string paths from a []any slice. - collectStringImports := func(items []any) []string { - var paths []string - for _, item := range items { - if str, ok := item.(string); ok { - paths = append(paths, str) - } - } - return paths - } - switch v := importsField.(type) { case []any: - imports := collectStringImports(v) + imports := processImportsWithWorkflowSpecCollectStrings(v) importsLog.Printf("Found %d imports (array form) to process", len(imports)) - result.Frontmatter["imports"] = processImportPaths(imports) + result.Frontmatter["imports"] = processImportsWithWorkflowSpecPaths(imports, workflow, commitSHA, localWorkflowDir) case []string: importsLog.Printf("Found %d imports ([]string form) to process", len(v)) - result.Frontmatter["imports"] = processImportPaths(v) + result.Frontmatter["imports"] = processImportsWithWorkflowSpecPaths(v, workflow, commitSHA, localWorkflowDir) case map[string]any: // Object form: process the 'aw' subfield if present if awAny, hasAW := v["aw"]; hasAW { switch aw := awAny.(type) { case []any: - awImports := collectStringImports(aw) + awImports := processImportsWithWorkflowSpecCollectStrings(aw) importsLog.Printf("Found %d imports (object form, aw subfield) to process", len(awImports)) - v["aw"] = processImportPaths(awImports) + v["aw"] = processImportsWithWorkflowSpecPaths(awImports, workflow, commitSHA, localWorkflowDir) case []string: importsLog.Printf("Found %d imports (object form, aw []string) to process", len(aw)) - v["aw"] = processImportPaths(aw) + v["aw"] = processImportsWithWorkflowSpecPaths(aw, workflow, commitSHA, localWorkflowDir) } } default: @@ -129,6 +86,43 @@ func processImportsWithWorkflowSpec(content string, workflow *WorkflowSpec, comm return reconstructWorkflowFileFromMap(result.Frontmatter, result.Markdown) } +func processImportsWithWorkflowSpecPaths(imports []string, workflow *WorkflowSpec, commitSHA, localWorkflowDir string) []string { + // Converts raw import paths to workflowspec format. Paths that already use + // workflowspec format or exist under localWorkflowDir are preserved. + processed := make([]string, 0, len(imports)) + for _, importPath := range imports { + if isWorkflowSpecFormat(importPath) { + importsLog.Printf("Import already in workflowspec format: %s", importPath) + processed = append(processed, importPath) + continue + } + if localWorkflowDir != "" && !strings.HasPrefix(importPath, "/") { + if isLocalFileForUpdate(localWorkflowDir, importPath) { + importsLog.Printf("Import path exists locally, preserving relative path: %s", importPath) + processed = append(processed, importPath) + continue + } + } + resolvedPath := resolveImportPath(importPath, filepath.Dir(workflow.WorkflowPath), importPathImportsOpts) + importsLog.Printf("Resolved import path: %s -> %s (workflow: %s)", importPath, resolvedPath, workflow.WorkflowPath) + workflowSpec := buildWorkflowSpecRef(workflow.RepoSlug, resolvedPath, commitSHA, workflow.Version) + importsLog.Printf("Converted import: %s -> %s", importPath, workflowSpec) + processed = append(processed, workflowSpec) + } + return processed +} + +func processImportsWithWorkflowSpecCollectStrings(items []any) []string { + // Extracts string paths from a []any slice. + var paths []string + for _, item := range items { + if str, ok := item.(string); ok { + paths = append(paths, str) + } + } + return paths +} + // reconstructWorkflowFileFromMap reconstructs a workflow file from frontmatter map and markdown // using proper field ordering and YAML helpers func reconstructWorkflowFileFromMap(frontmatter map[string]any, markdown string) (string, error) { @@ -156,146 +150,160 @@ func processIncludesWithWorkflowSpec(content string, workflow *WorkflowSpec, com fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Processing @include directives to replace with workflowspec")) } - // Track visited includes to prevent cycles - visited := make(map[string]struct { - }) - - // Use a queue to process files iteratively instead of recursion + visited := make(map[string]struct{}) queue := []string{} + result, err := processIncludesWithWorkflowSpecContent(content, workflow, commitSHA, localWorkflowDir, visited, &queue, verbose) + if err != nil { + return "", err + } + processIncludesWithWorkflowSpecQueue(queue, visited, packagePath, verbose) + return result, nil +} - // Process the main content first +func processIncludesWithWorkflowSpecContent( + content string, + workflow *WorkflowSpec, + commitSHA, localWorkflowDir string, + visited map[string]struct{}, + queue *[]string, + verbose bool, +) (string, error) { scanner := bufio.NewScanner(strings.NewReader(content)) var result strings.Builder - for scanner.Scan() { line := scanner.Text() - - // Parse import directive using the helper function that handles both syntaxes directive := parser.ParseImportDirective(line) - if directive != nil { - isOptional := directive.IsOptional - includePath := directive.Path - - // Skip if it's already a workflowspec (owner/repo/path@sha format) - if isWorkflowSpecFormat(includePath) { - result.WriteString(line + "\n") - continue - } - - // Handle section references (file.md#Section) - filePath, sectionName := splitImportPath(includePath) - - // Skip if filePath is empty (e.g., section-only reference like "#Section") - if filePath == "" { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Skipping include with empty file path: "+line)) - } - result.WriteString(line + "\n") - continue - } - - // Preserve relative {{#import}} paths whose files exist in the local workflow directory. - if localWorkflowDir != "" && !strings.HasPrefix(filePath, "/") { - if isLocalFileForUpdate(localWorkflowDir, filePath) { - importsLog.Printf("Include path exists locally, preserving: %s", filePath) - result.WriteString(line + "\n") - // Add file to queue for processing nested includes (first visit only) - if !setutil.Contains(visited, filePath) { - visited[filePath] = struct { - }{} - queue = append(queue, filePath) - } - continue - } - } - - // Resolve the file path relative to the workflow file's directory - resolvedPath := resolveImportPath(filePath, filepath.Dir(workflow.WorkflowPath), importPathImportsOpts) - - // Build workflowspec for this include - workflowSpec := buildWorkflowSpecRef(workflow.RepoSlug, resolvedPath, commitSHA, workflow.Version) - - // Add section if present - if sectionName != "" { - workflowSpec += "#" + sectionName - } - - // Write the updated @include directive (even for duplicate occurrences) - writeImportDirective(&result, workflowSpec, isOptional) - - // Only enqueue for nested-include processing on the first visit to prevent cycles - if !setutil.Contains(visited, filePath) { - visited[filePath] = struct { - }{} - queue = append(queue, filePath) - } - } else { + if directive == nil { // Regular line, pass through result.WriteString(line + "\n") + continue } + processIncludesWithWorkflowSpecDirective(processIncludesWithWorkflowSpecDirectiveParams{ + Result: &result, + Line: line, + Directive: directive, + Workflow: workflow, + CommitSHA: commitSHA, + LocalWorkflowDir: localWorkflowDir, + Visited: visited, + Queue: queue, + Verbose: verbose, + }) } - if err := scanner.Err(); err != nil { return "", err } + return result.String(), nil +} + +type processIncludesWithWorkflowSpecDirectiveParams struct { + Result *strings.Builder + Line string + Directive *parser.ImportDirectiveMatch + Workflow *WorkflowSpec + CommitSHA string + LocalWorkflowDir string + Visited map[string]struct{} + Queue *[]string + Verbose bool +} +func processIncludesWithWorkflowSpecDirective(p processIncludesWithWorkflowSpecDirectiveParams) { + includePath := p.Directive.Path + if isWorkflowSpecFormat(includePath) { + p.Result.WriteString(p.Line + "\n") + return + } + filePath, sectionName := splitImportPath(includePath) + if filePath == "" { + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Skipping include with empty file path: "+p.Line)) + } + p.Result.WriteString(p.Line + "\n") + return + } + if processIncludesWithWorkflowSpecPreserveLocal(p.Result, p.Line, filePath, p.LocalWorkflowDir, p.Visited, p.Queue) { + return + } + resolvedPath := resolveImportPath(filePath, filepath.Dir(p.Workflow.WorkflowPath), importPathImportsOpts) + workflowSpec := buildWorkflowSpecRef(p.Workflow.RepoSlug, resolvedPath, p.CommitSHA, p.Workflow.Version) + if sectionName != "" { + workflowSpec += "#" + sectionName + } + writeImportDirective(p.Result, workflowSpec, p.Directive.IsOptional) + processIncludesWithWorkflowSpecEnqueue(filePath, p.Visited, p.Queue) +} + +func processIncludesWithWorkflowSpecPreserveLocal(result *strings.Builder, line, filePath, localWorkflowDir string, visited map[string]struct{}, queue *[]string) bool { + if localWorkflowDir == "" || strings.HasPrefix(filePath, "/") || !isLocalFileForUpdate(localWorkflowDir, filePath) { + return false + } + importsLog.Printf("Include path exists locally, preserving: %s", filePath) + result.WriteString(line + "\n") + processIncludesWithWorkflowSpecEnqueue(filePath, visited, queue) + return true +} + +func processIncludesWithWorkflowSpecEnqueue(filePath string, visited map[string]struct{}, queue *[]string) { + if !setutil.Contains(visited, filePath) { + visited[filePath] = struct{}{} + *queue = append(*queue, filePath) + } +} + +func processIncludesWithWorkflowSpecQueue(queue []string, visited map[string]struct{}, packagePath string, verbose bool) { // Process queue of files to check for nested includes for len(queue) > 0 { - // Dequeue the first file filePath := queue[0] queue = queue[1:] + markdownContent, ok := processIncludesWithWorkflowSpecReadQueued(filePath, packagePath, verbose) + if !ok { + continue + } + queue = processIncludesWithWorkflowSpecNested(markdownContent, queue, visited, verbose) + } +} - fullSourcePath := filepath.Join(packagePath, filePath) - if _, err := os.Stat(fullSourcePath); err != nil { - continue // File doesn't exist, skip +func processIncludesWithWorkflowSpecReadQueued(filePath, packagePath string, verbose bool) (string, bool) { + fullSourcePath := filepath.Join(packagePath, filePath) + if _, err := os.Stat(fullSourcePath); err != nil { + return "", false + } + includedContent, err := os.ReadFile(fullSourcePath) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not read include file %s: %v", fullSourcePath, err))) + } + return "", false + } + markdownContent, err := parser.ExtractMarkdownContent(string(includedContent)) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not extract markdown from %s: %v", fullSourcePath, err))) } + return "", false + } + return markdownContent, true +} - includedContent, err := os.ReadFile(fullSourcePath) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not read include file %s: %v", fullSourcePath, err))) - } +func processIncludesWithWorkflowSpecNested(markdownContent string, queue []string, visited map[string]struct{}, verbose bool) []string { + nestedScanner := bufio.NewScanner(strings.NewReader(markdownContent)) + for nestedScanner.Scan() { + directive := parser.ParseImportDirective(nestedScanner.Text()) + if directive == nil { continue } - - // Extract markdown content from the included file - markdownContent, err := parser.ExtractMarkdownContent(string(includedContent)) - if err != nil { + nestedFilePath, _ := splitImportPath(directive.Path) + if setutil.Contains(visited, nestedFilePath) { if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not extract markdown from %s: %v", fullSourcePath, err))) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Cycle detected for include: %s, skipping", nestedFilePath))) } continue } - - // Scan for nested includes - nestedScanner := bufio.NewScanner(strings.NewReader(markdownContent)) - for nestedScanner.Scan() { - line := nestedScanner.Text() - - directive := parser.ParseImportDirective(line) - if directive != nil { - includePath := directive.Path - - // Handle section references - nestedFilePath, _ := splitImportPath(includePath) - - // Check for cycle detection - if setutil.Contains(visited, nestedFilePath) { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Cycle detected for include: %s, skipping", nestedFilePath))) - } - continue - } - - // Mark as visited and add to queue - visited[nestedFilePath] = struct { - }{} - queue = append(queue, nestedFilePath) - } - } + visited[nestedFilePath] = struct{}{} + queue = append(queue, nestedFilePath) } - - return result.String(), nil + return queue } // processIncludesInContent processes @include directives in workflow content for update command @@ -322,59 +330,53 @@ func processIncludesInContent(content string, workflow *WorkflowSpec, commitSHA // Parse import directive directive := parser.ParseImportDirective(line) - if directive != nil { - isOptional := directive.IsOptional - includePath := directive.Path - - // Skip if it's already a workflowspec (owner/repo/path@sha format) - if isWorkflowSpecFormat(includePath) { - result.WriteString(line + "\n") - continue - } - - // Handle section references (file.md#Section) - filePath, sectionName := splitImportPath(includePath) - - // Skip if filePath is empty (e.g., section-only reference like "#Section") - if filePath == "" { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Skipping include with empty file path: "+line)) - } - result.WriteString(line + "\n") - continue - } - - // Preserve relative {{#import}} paths whose files exist in the local workflow directory. - if localWorkflowDir != "" && !strings.HasPrefix(filePath, "/") { - if isLocalFileForUpdate(localWorkflowDir, filePath) { - importsLog.Printf("Include path exists locally, preserving: %s", filePath) - result.WriteString(line + "\n") - continue - } - } - - // Resolve the file path relative to the workflow file's directory - resolvedPath := resolveImportPath(filePath, filepath.Dir(workflow.WorkflowPath), importPathImportsOpts) - - // Build workflowspec for this include - workflowSpec := buildWorkflowSpecRef(workflow.RepoSlug, resolvedPath, commitSHA, workflow.Version) - - // Add section if present - if sectionName != "" { - workflowSpec += "#" + sectionName - } - - // Write the updated import directive - writeImportDirective(&result, workflowSpec, isOptional) - } else { + if directive == nil { // Regular line, pass through result.WriteString(line + "\n") + continue } + processIncludesInContentDirective(&result, line, directive, workflow, commitSHA, localWorkflowDir, verbose) } return result.String(), scanner.Err() } +func processIncludesInContentDirective( + result *strings.Builder, + line string, + directive *parser.ImportDirectiveMatch, + workflow *WorkflowSpec, + commitSHA, localWorkflowDir string, + verbose bool, +) { + includePath := directive.Path + if isWorkflowSpecFormat(includePath) { + result.WriteString(line + "\n") + return + } + filePath, sectionName := splitImportPath(includePath) + if filePath == "" { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Skipping include with empty file path: "+line)) + } + result.WriteString(line + "\n") + return + } + if localWorkflowDir != "" && !strings.HasPrefix(filePath, "/") { + if isLocalFileForUpdate(localWorkflowDir, filePath) { + importsLog.Printf("Include path exists locally, preserving: %s", filePath) + result.WriteString(line + "\n") + return + } + } + resolvedPath := resolveImportPath(filePath, filepath.Dir(workflow.WorkflowPath), importPathImportsOpts) + workflowSpec := buildWorkflowSpecRef(workflow.RepoSlug, resolvedPath, commitSHA, workflow.Version) + if sectionName != "" { + workflowSpec += "#" + sectionName + } + writeImportDirective(result, workflowSpec, directive.IsOptional) +} + // isLocalFileForUpdate returns true when importPath resolves to an existing file // within localWorkflowDir. The resolved absolute path must stay inside localWorkflowDir // to guard against path traversal (e.g. "../../etc/passwd" in import paths). diff --git a/pkg/cli/includes.go b/pkg/cli/includes.go index 8dbb2a6e772..ed8026e3cf0 100644 --- a/pkg/cli/includes.go +++ b/pkg/cli/includes.go @@ -36,85 +36,90 @@ func FetchIncludeFromSource(ctx context.Context, includePath string, baseSpec *W // Extract section reference (e.g., "#section-name") from the path upfront // This ensures consistent behavior regardless of which code path is taken - cleanPath := includePath - var section string - if idx := strings.Index(includePath, "#"); idx != -1 { - cleanPath = includePath[:idx] - section = includePath[idx:] - } + cleanPath, section := fetchIncludeFromSourceSplitSection(includePath) // Check if this is a workflowspec format (owner/repo/path[@ref]) if isWorkflowSpecFormat(cleanPath) { - // Split on @ to get path and ref - parts := strings.SplitN(cleanPath, "@", 2) - pathPart := parts[0] - var ref string - if len(parts) == 2 { - ref = parts[1] - } else { - ref = "main" - } + return fetchIncludeFromSourceWorkflowSpec(ctx, includePath, cleanPath, section) + } - // Parse path: owner/repo/path/to/file.md - slashParts := strings.Split(pathPart, "/") - if len(slashParts) < 3 { - return nil, section, errors.New("invalid workflowspec: must be owner/repo/path[@ref]") + // For relative paths, resolve against the base spec + if baseSpec != nil && baseSpec.RepoSlug != "" { + if content, ok, err := fetchIncludeFromSourceRelative(ctx, cleanPath, baseSpec); ok || err != nil { + return content, section, err } + } - owner := slashParts[0] - repo := slashParts[1] - filePath := strings.Join(slashParts[2:], "/") - - // Download the file - content, err := parser.DownloadFileFromGitHub(ctx, owner, repo, filePath, ref) - if err != nil { - return nil, section, fmt.Errorf("failed to fetch include from %s: %w", includePath, err) - } + return nil, section, fmt.Errorf("cannot resolve include path: %s (no base spec provided)", includePath) +} - return content, section, nil +func fetchIncludeFromSourceSplitSection(includePath string) (string, string) { + cleanPath := includePath + var section string + if idx := strings.Index(includePath, "#"); idx != -1 { + cleanPath = includePath[:idx] + section = includePath[idx:] } + return cleanPath, section +} - // For relative paths, resolve against the base spec - if baseSpec != nil && baseSpec.RepoSlug != "" { - parts := strings.SplitN(baseSpec.RepoSlug, "/", 2) - if len(parts) == 2 { - owner := parts[0] - repo := parts[1] - ref := baseSpec.Version - if ref == "" { - ref = "main" - } +func fetchIncludeFromSourceWorkflowSpec(ctx context.Context, includePath, cleanPath, section string) ([]byte, string, error) { + // Split on @ to get path and ref + parts := strings.SplitN(cleanPath, "@", 2) + pathPart := parts[0] + ref := "main" + if len(parts) == 2 { + ref = parts[1] + } - // Remove @ ref suffix if present in the clean path (for relative paths with explicit refs) - filePath := cleanPath - if idx := strings.Index(filePath, "@"); idx != -1 { - filePath = filePath[:idx] - } + // Parse path: owner/repo/path/to/file.md + slashParts := strings.Split(pathPart, "/") + if len(slashParts) < 3 { + return nil, section, errors.New("invalid workflowspec: must be owner/repo/path[@ref]") + } - // If it's a relative path starting with shared/, it's relative to .github/ - var fullPath string - if strings.HasPrefix(filePath, "shared/") { - fullPath = constants.GithubDir + filePath - } else { - // Otherwise, resolve relative to the workflow path directory - baseDir := getParentDir(baseSpec.WorkflowPath) - if baseDir != "" { - fullPath = baseDir + "/" + filePath - } else { - fullPath = filePath - } - } + owner := slashParts[0] + repo := slashParts[1] + filePath := strings.Join(slashParts[2:], "/") + content, err := parser.DownloadFileFromGitHub(ctx, owner, repo, filePath, ref) + if err != nil { + return nil, section, fmt.Errorf("failed to fetch include from %s: %w", includePath, err) + } + return content, section, nil +} - content, err := parser.DownloadFileFromGitHub(ctx, owner, repo, fullPath, ref) - if err != nil { - return nil, section, fmt.Errorf("failed to fetch include %s from %s/%s: %w", filePath, owner, repo, err) - } +func fetchIncludeFromSourceRelative(ctx context.Context, cleanPath string, baseSpec *WorkflowSpec) ([]byte, bool, error) { + parts := strings.SplitN(baseSpec.RepoSlug, "/", 2) + if len(parts) != 2 { + return nil, false, nil + } + owner, repo := parts[0], parts[1] + ref := baseSpec.Version + if ref == "" { + ref = "main" + } - return content, section, nil - } + filePath := cleanPath + if idx := strings.Index(filePath, "@"); idx != -1 { + filePath = filePath[:idx] } + fullPath := fetchIncludeFromSourceRelativeFullPath(filePath, baseSpec.WorkflowPath) + content, err := parser.DownloadFileFromGitHub(ctx, owner, repo, fullPath, ref) + if err != nil { + return nil, true, fmt.Errorf("failed to fetch include %s from %s/%s: %w", filePath, owner, repo, err) + } + return content, true, nil +} - return nil, section, fmt.Errorf("cannot resolve include path: %s (no base spec provided)", includePath) +func fetchIncludeFromSourceRelativeFullPath(filePath, workflowPath string) string { + if strings.HasPrefix(filePath, "shared/") { + return constants.GithubDir + filePath + } + baseDir := getParentDir(workflowPath) + if baseDir != "" { + return baseDir + "/" + filePath + } + return filePath } // fetchAndSaveRemoteFrontmatterImports fetches and saves files referenced in the frontmatter @@ -208,35 +213,7 @@ func fetchFrontmatterImportsRecursive(ctx context.Context, content, currentBaseD return } - importsField, exists := result.Frontmatter["imports"] - if !exists { - return - } - - var importPaths []string - switch v := importsField.(type) { - case []any: - for _, item := range v { - switch importItem := item.(type) { - case string: - importPaths = append(importPaths, importItem) - case map[string]any: - // Handle uses: and path: forms (mirrors GitHub Actions reusable workflow syntax) - if usesVal, ok := importItem["uses"]; ok { - if p, ok := usesVal.(string); ok { - importPaths = append(importPaths, p) - } - } else if pathVal, ok := importItem["path"]; ok { - if p, ok := pathVal.(string); ok { - importPaths = append(importPaths, p) - } - } - } - } - case []string: - importPaths = v - } - + importPaths := fetchFrontmatterImportsRecursivePaths(result.Frontmatter) if len(importPaths) == 0 { return } @@ -250,69 +227,8 @@ func fetchFrontmatterImportsRecursive(ctx context.Context, content, currentBaseD } for _, importPath := range importPaths { - // Skip workflowspec-format imports (already pinned to a remote ref) - if isWorkflowSpecFormat(importPath) { - continue - } - - // Strip any section reference (file.md#Section → file.md) - filePath := importPath - if before, _, hasSec := strings.Cut(importPath, "#"); hasSec { - filePath = before - } - if filePath == "" { - continue - } - - // Resolve the remote file path to an absolute repo path. - // Use path (not filepath) because this is always a forward-slash URL/API path. - var remoteFilePath string - if rest, ok := strings.CutPrefix(filePath, "/"); ok { - // Absolute path from repo root (e.g. "/scripts/helper.md") - remoteFilePath = rest - } else if strings.HasPrefix(filePath, "./") || strings.HasPrefix(filePath, "../") { - // Explicitly-relative path (e.g. "./serena.md"): resolve relative to the - // current importing file's directory so that sibling-file references work - // correctly regardless of nesting depth. - if currentBaseDir != "" { - remoteFilePath = path.Join(currentBaseDir, filePath) - } else { - remoteFilePath = filePath - } - } else { - // Non-explicit relative path: resolution strategy mirrors the compiler's - // determineNestedBaseDir logic — it depends on whether the path contains a - // "/" separator. - // - // Paths WITHOUT "/" (e.g. "helper.md"): the importing file treats them as - // siblings in its own directory, so resolve relative to currentBaseDir. - // Example: "control-precompute.md" from ".github/workflows/shared/control.md" - // → ".github/workflows/shared/control-precompute.md" - // - // Paths WITH "/" (e.g. "shared/foo.md"): workflows in this repository write - // these paths relative to the workflow root (.github/workflows), not relative to - // the importing file's directory, so resolve against originalBaseDir. - // Example: "shared/reporting.md" from ".github/workflows/shared/daily-audit-base.md" - // → ".github/workflows/shared/reporting.md" - var baseDir string - if !strings.Contains(filePath, "/") { - baseDir = currentBaseDir - } else { - baseDir = opts.originalBaseDir - } - if baseDir != "" { - remoteFilePath = path.Join(baseDir, filePath) - } else { - remoteFilePath = filePath - } - } - remoteFilePath = path.Clean(remoteFilePath) - - // Reject paths that try to escape the repository root (e.g. "../../etc/passwd") - if remoteFilePath == ".." || strings.HasPrefix(remoteFilePath, "../") { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping import with unsafe path: %q", importPath))) - } + remoteFilePath, ok := fetchFrontmatterImportsRecursiveRemotePath(importPath, currentBaseDir, opts) + if !ok { continue } @@ -323,115 +239,193 @@ func fetchFrontmatterImportsRecursive(ctx context.Context, content, currentBaseD } opts.seen[remoteFilePath] = struct{}{} - // Derive the local path relative to targetDir by stripping the original base-dir - // prefix from the remote path. This ensures that imports in nested files resolve - // to the correct location regardless of how many levels deep the recursion goes. - // - // Example: originalBaseDir=".github/workflows" - // remoteFilePath=".github/workflows/shared/analysis.md" → localRelPath="shared/analysis.md" - // (nested) remoteFilePath=".github/workflows/other.md" → localRelPath="other.md" - var localRelPath string - if opts.originalBaseDir != "" && strings.HasPrefix(remoteFilePath, opts.originalBaseDir+"/") { - localRelPath = remoteFilePath[len(opts.originalBaseDir)+1:] - } else { - // Workflow at repo root, or import outside the original base dir: - // use the full remote path relative to targetDir. - localRelPath = remoteFilePath - } - localRelPath = filepath.Clean(filepath.FromSlash(localRelPath)) - // Strip any leading separator produced by Clean on root-relative paths. - localRelPath = strings.TrimLeft(localRelPath, string(filepath.Separator)) - // Reject empty or "." paths (would point to targetDir itself) as a safety guard. - // ".." cannot appear here because remoteFilePath was already rejected above if it - // started with "..", and path.Clean cannot introduce new ".." components. - if localRelPath == "" || localRelPath == "." { + targetPath, ok := fetchFrontmatterImportsRecursiveTargetPath(remoteFilePath, importPath, absTargetDir, opts) + if !ok { continue } - targetPath := filepath.Join(opts.targetDir, localRelPath) - - // Belt-and-suspenders: verify the resolved path is inside targetDir - absTargetPath, absErr := filepath.Abs(targetPath) - if absErr != nil { + fileExists, skip := fetchFrontmatterImportsRecursiveHandleExisting(ctx, targetPath, remoteFilePath, opts) + if skip { continue } - if rel, relErr := filepath.Rel(absTargetDir, absTargetPath); relErr != nil || strings.HasPrefix(rel, "..") { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Refusing to write import outside target directory: %q", importPath))) - } + + importContent, ok := fetchFrontmatterImportsRecursiveDownloadAndWrite(ctx, targetPath, remoteFilePath, opts) + if !ok { continue } + fetchFrontmatterImportsRecursiveTrack(targetPath, fileExists, opts.tracker) + importedBaseDir := path.Dir(remoteFilePath) + fetchFrontmatterImportsRecursive(ctx, string(importContent), importedBaseDir, opts) + } +} - // Check existence before downloading: if the file already exists and force=false, - // skip the download entirely (no unnecessary network round-trip). However, still - // recurse into the existing file's imports so that any transitive dependencies - // it references are fetched even when the parent file was already present. - fileExists := false - if fileutil.FileExists(targetPath) { - fileExists = true - if !opts.force { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Import file already exists, skipping: "+targetPath)) +func fetchFrontmatterImportsRecursivePaths(frontmatter map[string]any) []string { + importsField, exists := frontmatter["imports"] + if !exists { + return nil + } + switch v := importsField.(type) { + case []any: + return fetchFrontmatterImportsRecursiveAnyPaths(v) + case []string: + return v + default: + return nil + } +} + +func fetchFrontmatterImportsRecursiveAnyPaths(items []any) []string { + var importPaths []string + for _, item := range items { + switch importItem := item.(type) { + case string: + importPaths = append(importPaths, importItem) + case map[string]any: + // Handle uses: and path: forms (mirrors GitHub Actions reusable workflow syntax) + if usesVal, ok := importItem["uses"]; ok { + if p, ok := usesVal.(string); ok { + importPaths = append(importPaths, p) } - // Read the existing file so we can recurse into its own imports. - // If the read fails, log it and skip — the compiler will report - // any missing transitive dependencies. - if existingContent, readErr := os.ReadFile(targetPath); readErr == nil { - importedBaseDir := path.Dir(remoteFilePath) - fetchFrontmatterImportsRecursive(ctx, string(existingContent), importedBaseDir, opts) - } else { - remoteWorkflowLog.Printf("Failed to read existing import %s for recursion: %v", targetPath, readErr) + } else if pathVal, ok := importItem["path"]; ok { + if p, ok := pathVal.(string); ok { + importPaths = append(importPaths, p) } - continue } } + } + return importPaths +} - // Download from the source repository - downloadFn := opts.downloadFn - if downloadFn == nil { - downloadFn = downloadRemoteImportFile - } - importContent, err := downloadFn(ctx, opts.owner, opts.repo, remoteFilePath, opts.ref) - if err != nil { - remoteWorkflowLog.Printf("Failed to download import %s from %s/%s@%s: %v", remoteFilePath, opts.owner, opts.repo, opts.ref, err) - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch import %s: %v", remoteFilePath, err))) - } - continue +func fetchFrontmatterImportsRecursiveRemotePath(importPath, currentBaseDir string, opts frontmatterImportsOpts) (string, bool) { + if isWorkflowSpecFormat(importPath) { + return "", false + } + filePath := importPath + if before, _, hasSec := strings.Cut(importPath, "#"); hasSec { + filePath = before + } + if filePath == "" { + return "", false + } + remoteFilePath := fetchFrontmatterImportsRecursiveResolveRemote(filePath, currentBaseDir, opts.originalBaseDir) + remoteFilePath = path.Clean(remoteFilePath) + if remoteFilePath == ".." || strings.HasPrefix(remoteFilePath, "../") { + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping import with unsafe path: %q", importPath))) } + return "", false + } + return remoteFilePath, true +} - // Create the parent directory if needed - if err := os.MkdirAll(filepath.Dir(targetPath), constants.DirPermPublic); err != nil { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for import %s: %v", remoteFilePath, err))) - } - continue +func fetchFrontmatterImportsRecursiveResolveRemote(filePath, currentBaseDir, originalBaseDir string) string { + // Use path (not filepath) because this is always a forward-slash URL/API path. + if rest, ok := strings.CutPrefix(filePath, "/"); ok { + return rest + } + if strings.HasPrefix(filePath, "./") || strings.HasPrefix(filePath, "../") { + if currentBaseDir != "" { + return path.Join(currentBaseDir, filePath) } + return filePath + } + baseDir := originalBaseDir + if !strings.Contains(filePath, "/") { + baseDir = currentBaseDir + } + if baseDir != "" { + return path.Join(baseDir, filePath) + } + return filePath +} - // Write the file - if err := os.WriteFile(targetPath, importContent, constants.FilePermSensitive); err != nil { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write import %s: %v", remoteFilePath, err))) - } - continue +func fetchFrontmatterImportsRecursiveTargetPath(remoteFilePath, importPath, absTargetDir string, opts frontmatterImportsOpts) (string, bool) { + localRelPath := remoteFilePath + if opts.originalBaseDir != "" && strings.HasPrefix(remoteFilePath, opts.originalBaseDir+"/") { + localRelPath = remoteFilePath[len(opts.originalBaseDir)+1:] + } + localRelPath = filepath.Clean(filepath.FromSlash(localRelPath)) + localRelPath = strings.TrimLeft(localRelPath, string(filepath.Separator)) + if localRelPath == "" || localRelPath == "." { + return "", false + } + targetPath := filepath.Join(opts.targetDir, localRelPath) + absTargetPath, absErr := filepath.Abs(targetPath) + if absErr != nil { + return "", false + } + if rel, relErr := filepath.Rel(absTargetDir, absTargetPath); relErr != nil || strings.HasPrefix(rel, "..") { + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Refusing to write import outside target directory: %q", importPath))) } + return "", false + } + return targetPath, true +} + +func fetchFrontmatterImportsRecursiveHandleExisting(ctx context.Context, targetPath, remoteFilePath string, opts frontmatterImportsOpts) (bool, bool) { + if !fileutil.FileExists(targetPath) { + return false, false + } + if opts.force { + return true, false + } + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Import file already exists, skipping: "+targetPath)) + } + if existingContent, readErr := os.ReadFile(targetPath); readErr == nil { + fetchFrontmatterImportsRecursive(ctx, string(existingContent), path.Dir(remoteFilePath), opts) + } else { + remoteWorkflowLog.Printf("Failed to read existing import %s for recursion: %v", targetPath, readErr) + } + return true, true +} +func fetchFrontmatterImportsRecursiveDownloadAndWrite(ctx context.Context, targetPath, remoteFilePath string, opts frontmatterImportsOpts) ([]byte, bool) { + downloadFn := opts.downloadFn + if downloadFn == nil { + downloadFn = downloadRemoteImportFile + } + importContent, err := downloadFn(ctx, opts.owner, opts.repo, remoteFilePath, opts.ref) + if err != nil { + remoteWorkflowLog.Printf("Failed to download import %s from %s/%s@%s: %v", remoteFilePath, opts.owner, opts.repo, opts.ref, err) if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched import: "+targetPath)) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch import %s: %v", remoteFilePath, err))) } + return nil, false + } + if err := fetchFrontmatterImportsRecursiveWriteFile(targetPath, remoteFilePath, importContent, opts.verbose); err != nil { + return nil, false + } + return importContent, true +} - // Track the file for git staging and potential rollback - if opts.tracker != nil { - if fileExists { - opts.tracker.TrackModified(targetPath) - } else { - opts.tracker.TrackCreated(targetPath) - } +func fetchFrontmatterImportsRecursiveWriteFile(targetPath, remoteFilePath string, importContent []byte, verbose bool) error { + if err := os.MkdirAll(filepath.Dir(targetPath), constants.DirPermPublic); err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for import %s: %v", remoteFilePath, err))) + } + return err + } + if err := os.WriteFile(targetPath, importContent, constants.FilePermSensitive); err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write import %s: %v", remoteFilePath, err))) } + return err + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched import: "+targetPath)) + } + return nil +} - // Recurse into the imported file's imports. Use the imported file's directory as - // currentBaseDir so that relative paths inside it resolve correctly. - importedBaseDir := path.Dir(remoteFilePath) - fetchFrontmatterImportsRecursive(ctx, string(importContent), importedBaseDir, opts) +func fetchFrontmatterImportsRecursiveTrack(targetPath string, fileExists bool, tracker *FileTracker) { + if tracker == nil { + return + } + if fileExists { + tracker.TrackModified(targetPath) + } else { + tracker.TrackCreated(targetPath) } } @@ -450,94 +444,112 @@ func fetchAndSaveRemoteIncludes(ctx context.Context, content string, spec *Workf if matches == nil { continue } - - isOptional := matches[1] == "?" - includePath := strings.TrimSpace(matches[2]) - - // Remove section reference for file fetching - filePath := includePath - if before, _, ok := strings.Cut(includePath, "#"); ok { - filePath = before - } - - // Skip if already processed - if setutil.Contains(seen, filePath) { - continue - } - seen[filePath] = struct { - }{} - - // Fetch the include file - includeContent, _, err := FetchIncludeFromSource(ctx, includePath, spec, verbose) - if err != nil { - if isOptional { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Optional include not found: "+includePath)) - } - continue - } - return fmt.Errorf("failed to fetch include %s: %w", includePath, err) + if err := fetchAndSaveRemoteIncludesDirective(ctx, matches, spec, targetDir, verbose, force, tracker, seen); err != nil { + return err } + } - // Determine target path for the include file - var targetPath string - if strings.HasPrefix(filePath, "shared/") { - // shared/ files go to .github/shared/ - targetPath = filepath.Join(filepath.Dir(targetDir), filePath) - } else if isWorkflowSpecFormat(filePath) { - // Workflowspec includes: extract just the filename and put in shared/ - parts := strings.Split(filePath, "/") - filename := parts[len(parts)-1] - targetPath = filepath.Join(filepath.Dir(targetDir), "shared", filename) - } else { - // Relative includes go alongside the workflow - targetPath = filepath.Join(targetDir, filePath) - } + return nil +} - // Create target directory if needed - if err := os.MkdirAll(filepath.Dir(targetPath), constants.DirPermPublic); err != nil { - return fmt.Errorf("failed to create directory for %s: %w", targetPath, err) - } +func fetchAndSaveRemoteIncludesDirective( + ctx context.Context, + matches []string, + spec *WorkflowSpec, + targetDir string, + verbose bool, + force bool, + tracker *FileTracker, + seen map[string]struct{}, +) error { + isOptional := matches[1] == "?" + includePath := strings.TrimSpace(matches[2]) + filePath := fetchAndSaveRemoteIncludesFilePath(includePath) + if setutil.Contains(seen, filePath) { + return nil + } + seen[filePath] = struct{}{} - // Check if file already exists - fileExists := false - if fileutil.FileExists(targetPath) { - fileExists = true - if !force { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Include file already exists, skipping: "+targetPath)) - } - continue + includeContent, _, err := FetchIncludeFromSource(ctx, includePath, spec, verbose) + if err != nil { + if isOptional { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Optional include not found: "+includePath)) } + return nil } + return fmt.Errorf("failed to fetch include %s: %w", includePath, err) + } - // Write the include file - if err := os.WriteFile(targetPath, includeContent, constants.FilePermSensitive); err != nil { - return fmt.Errorf("failed to write include file %s: %w", targetPath, err) - } - + targetPath := fetchAndSaveRemoteIncludesTargetPath(filePath, targetDir) + fileExists, skip, err := fetchAndSaveRemoteIncludesWrite(targetPath, includeContent, verbose, force) + if err != nil || skip { + return err + } + fetchAndSaveRemoteIncludesTrack(targetPath, fileExists, tracker) + if err := fetchAndSaveRemoteIncludes(ctx, string(includeContent), spec, targetDir, verbose, force, tracker); err != nil { if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched include: "+targetPath)) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch nested includes from %s: %v", filePath, err))) } + } + return nil +} - // Track the file - if tracker != nil { - if fileExists { - tracker.TrackModified(targetPath) - } else { - tracker.TrackCreated(targetPath) - } - } +func fetchAndSaveRemoteIncludesFilePath(includePath string) string { + filePath := includePath + if before, _, ok := strings.Cut(includePath, "#"); ok { + filePath = before + } + return filePath +} - // Recursively fetch includes from the fetched file - if err := fetchAndSaveRemoteIncludes(ctx, string(includeContent), spec, targetDir, verbose, force, tracker); err != nil { +func fetchAndSaveRemoteIncludesTargetPath(filePath, targetDir string) string { + if strings.HasPrefix(filePath, "shared/") { + // shared/ files go to .github/shared/ + return filepath.Join(filepath.Dir(targetDir), filePath) + } + if isWorkflowSpecFormat(filePath) { + // Workflowspec includes: extract just the filename and put in shared/ + parts := strings.Split(filePath, "/") + filename := parts[len(parts)-1] + return filepath.Join(filepath.Dir(targetDir), "shared", filename) + } + // Relative includes go alongside the workflow + return filepath.Join(targetDir, filePath) +} + +func fetchAndSaveRemoteIncludesWrite(targetPath string, includeContent []byte, verbose bool, force bool) (bool, bool, error) { + if err := os.MkdirAll(filepath.Dir(targetPath), constants.DirPermPublic); err != nil { + return false, false, fmt.Errorf("failed to create directory for %s: %w", targetPath, err) + } + fileExists := false + if fileutil.FileExists(targetPath) { + fileExists = true + if !force { if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch nested includes from %s: %v", filePath, err))) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Include file already exists, skipping: "+targetPath)) } + return fileExists, true, nil } } + if err := os.WriteFile(targetPath, includeContent, constants.FilePermSensitive); err != nil { + return false, false, fmt.Errorf("failed to write include file %s: %w", targetPath, err) + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched include: "+targetPath)) + } + return fileExists, false, nil +} - return nil +func fetchAndSaveRemoteIncludesTrack(targetPath string, fileExists bool, tracker *FileTracker) { + if tracker == nil { + return + } + if fileExists { + tracker.TrackModified(targetPath) + } else { + tracker.TrackCreated(targetPath) + } } // fetchAllRemoteDependencies fetches all remote dependencies for a workflow: diff --git a/pkg/cli/init.go b/pkg/cli/init.go index 08dab407caf..38a5300150d 100644 --- a/pkg/cli/init.go +++ b/pkg/cli/init.go @@ -46,136 +46,192 @@ func InitRepository(opts InitOptions) error { } copilotArtifactsEnabled := opts.Engine == "" || opts.Engine == "copilot" + if err := initRepositoryPreflight(opts); err != nil { + return err + } + initLog.Print("Verified git repository") + + if err := initRepositoryConfigureBase(opts); err != nil { + return err + } + + if err := initRepositoryConfigureCopilot(opts, copilotArtifactsEnabled); err != nil { + return err + } + + // Delete existing setup agentic workflows agent if it exists + initLog.Print("Cleaning up setup agentic workflows agent") + if err := deleteSetupAgenticWorkflowsAgent(opts.Verbose); err != nil { + initLog.Printf("Failed to delete setup agentic workflows agent: %v", err) + return fmt.Errorf("failed to delete setup agentic workflows agent: %w", err) + } + + if err := initRepositoryConfigureMCP(ctx, opts, copilotArtifactsEnabled); err != nil { + return err + } + + if err := initRepositoryConfigureCodespaces(opts); err != nil { + return err + } + + if err := initRepositoryConfigureEditor(opts); err != nil { + return err + } + + // Generate/update maintenance workflow if any workflows use expires field + initLog.Print("Checking for workflows with expires field to generate maintenance workflow") + if err := ensureMaintenanceWorkflow(ctx, opts.Verbose); err != nil { + initLog.Printf("Failed to generate maintenance workflow: %v", err) + // Don't fail init if maintenance workflow generation has issues + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to generate maintenance workflow: %v", err))) + } + + initLog.Print("Repository initialization completed successfully") + + if err := initRepositoryCreatePR(opts); err != nil { + return err + } + + initRepositoryDisplaySuccess(opts) + return nil +} + +func initRepositoryPreflight(opts InitOptions) error { // Show welcome banner for interactive mode console.ShowWelcomeBanner("This tool will initialize your repository for GitHub Agentic Workflows.") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Setting up repository...")) fmt.Fprintln(os.Stderr, "") - // If --create-pull-request is enabled, run pre-flight checks before doing any work if opts.CreatePR { if err := PreflightCheckForCreatePR(opts.Verbose); err != nil { return err } } - - // Ensure we're in a git repository if !isGitRepo() { initLog.Print("Not in a git repository, initialization failed") return errors.New("not in a git repository") } - initLog.Print("Verified git repository") + return nil +} - // Auto-detect GHES deployment and configure aw.json ghes: true when needed. +func initRepositoryConfigureBase(opts InitOptions) error { if _, err := ensureGHESRepoConfig(opts.Verbose); err != nil { initLog.Printf("Failed to configure GHES repo config: %v", err) - // Non-fatal: continue with the rest of init fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to configure GHES repo config: %v", err))) } - if opts.NoGitattributes { initLog.Print("Skipping .gitattributes configuration") - } else { - // Configure .gitattributes - initLog.Print("Configuring .gitattributes") - if updated, err := ensureGitAttributes(); err != nil { - initLog.Printf("Failed to configure .gitattributes: %v", err) - return fmt.Errorf("failed to configure .gitattributes: %w", err) - } else if updated && opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Configured .gitattributes")) - } + return nil + } + initLog.Print("Configuring .gitattributes") + if updated, err := ensureGitAttributes(); err != nil { + initLog.Printf("Failed to configure .gitattributes: %v", err) + return fmt.Errorf("failed to configure .gitattributes: %w", err) + } else if updated && opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Configured .gitattributes")) } + return nil +} - // Write dispatcher skill for Copilot engine only - if copilotArtifactsEnabled { - if opts.Skill { - initLog.Print("Writing agentic workflows dispatcher skill") - if err := ensureAgenticWorkflowsDispatcher(opts.Verbose, false); err != nil { - initLog.Printf("Failed to write dispatcher skill: %v", err) - return fmt.Errorf("failed to write dispatcher skill: %w", err) - } - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Created dispatcher skill")) - } - } else { - initLog.Print("Skipping agentic workflows dispatcher skill") - } - if opts.Agent { - initLog.Print("Writing agentic workflows custom agent") - if err := ensureAgenticWorkflowsAgent(opts.Verbose); err != nil { - initLog.Printf("Failed to write agentic workflows custom agent: %v", err) - return fmt.Errorf("failed to write agentic workflows custom agent: %w", err) - } - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Created agentic workflows custom agent")) - } - } else { - initLog.Print("Skipping agentic workflows custom agent") - } - if err := deleteLegacyAgentFiles(opts.Verbose); err != nil { - initLog.Printf("Failed to delete legacy agent files: %v", err) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete legacy agent files: %v", err))) - } - if err := deleteAgenticWorkflowDesignerSkillDir(opts.Verbose); err != nil { - initLog.Printf("Failed to delete legacy agentic-workflow-designer skill directory: %v", err) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete legacy agentic-workflow-designer skill directory: %v", err))) - } - } else { +func initRepositoryConfigureCopilot(opts InitOptions, copilotArtifactsEnabled bool) error { + if !copilotArtifactsEnabled { initLog.Printf("Skipping Copilot dispatcher skill for engine: %s", opts.Engine) + return nil } - - // Delete existing setup agentic workflows agent if it exists - initLog.Print("Cleaning up setup agentic workflows agent") - if err := deleteSetupAgenticWorkflowsAgent(opts.Verbose); err != nil { - initLog.Printf("Failed to delete setup agentic workflows agent: %v", err) - return fmt.Errorf("failed to delete setup agentic workflows agent: %w", err) + if err := initRepositoryConfigureSkill(opts); err != nil { + return err } + if err := initRepositoryConfigureAgent(opts); err != nil { + return err + } + initRepositoryDeleteLegacyAgents(opts.Verbose) + return nil +} - // Configure MCP if requested - if opts.MCP && copilotArtifactsEnabled { - initLog.Print("Configuring GitHub Copilot Agent MCP integration") - - // Detect action mode for setup steps generation - actionMode := workflow.DetectActionMode(GetVersion()) - initLog.Printf("Using action mode for copilot-setup-steps.yml: %s", actionMode) +func initRepositoryConfigureSkill(opts InitOptions) error { + if !opts.Skill { + initLog.Print("Skipping agentic workflows dispatcher skill") + return nil + } + initLog.Print("Writing agentic workflows dispatcher skill") + if err := ensureAgenticWorkflowsDispatcher(opts.Verbose, false); err != nil { + initLog.Printf("Failed to write dispatcher skill: %v", err) + return fmt.Errorf("failed to write dispatcher skill: %w", err) + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Created dispatcher skill")) + } + return nil +} - // Create copilot-setup-steps.yml - if err := ensureCopilotSetupSteps(ctx, opts.Verbose, actionMode, GetVersion()); err != nil { - initLog.Printf("Failed to create copilot-setup-steps.yml: %v", err) - return fmt.Errorf("failed to create copilot-setup-steps.yml: %w", err) - } - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Created .github/workflows/copilot-setup-steps.yml")) - } +func initRepositoryConfigureAgent(opts InitOptions) error { + if !opts.Agent { + initLog.Print("Skipping agentic workflows custom agent") + return nil + } + initLog.Print("Writing agentic workflows custom agent") + if err := ensureAgenticWorkflowsAgent(opts.Verbose); err != nil { + initLog.Printf("Failed to write agentic workflows custom agent: %v", err) + return fmt.Errorf("failed to write agentic workflows custom agent: %w", err) + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Created agentic workflows custom agent")) + } + return nil +} - // Create .github/mcp.json - if err := ensureMCPConfig(opts.Verbose); err != nil { - initLog.Printf("Failed to create MCP config: %v", err) - return fmt.Errorf("failed to create MCP config: %w", err) - } - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Configured .github/mcp.json")) - } +func initRepositoryDeleteLegacyAgents(verbose bool) { + if err := deleteLegacyAgentFiles(verbose); err != nil { + initLog.Printf("Failed to delete legacy agent files: %v", err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete legacy agent files: %v", err))) + } + if err := deleteAgenticWorkflowDesignerSkillDir(verbose); err != nil { + initLog.Printf("Failed to delete legacy agentic-workflow-designer skill directory: %v", err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to delete legacy agentic-workflow-designer skill directory: %v", err))) } +} - // Configure Codespaces if requested - if opts.CodespaceEnabled { - initLog.Printf("Configuring GitHub Codespaces devcontainer with additional repos: %v", opts.CodespaceRepos) +func initRepositoryConfigureMCP(ctx context.Context, opts InitOptions, copilotArtifactsEnabled bool) error { + if !opts.MCP || !copilotArtifactsEnabled { + return nil + } + initLog.Print("Configuring GitHub Copilot Agent MCP integration") + actionMode := workflow.DetectActionMode(GetVersion()) + initLog.Printf("Using action mode for copilot-setup-steps.yml: %s", actionMode) + if err := ensureCopilotSetupSteps(ctx, opts.Verbose, actionMode, GetVersion()); err != nil { + initLog.Printf("Failed to create copilot-setup-steps.yml: %v", err) + return fmt.Errorf("failed to create copilot-setup-steps.yml: %w", err) + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Created .github/workflows/copilot-setup-steps.yml")) + } + if err := ensureMCPConfig(opts.Verbose); err != nil { + initLog.Printf("Failed to create MCP config: %v", err) + return fmt.Errorf("failed to create MCP config: %w", err) + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Configured .github/mcp.json")) + } + return nil +} - // Create or update .devcontainer/devcontainer.json - if err := ensureDevcontainerConfig(opts.Verbose, opts.CodespaceRepos); err != nil { - initLog.Printf("Failed to configure devcontainer: %v", err) - return fmt.Errorf("failed to configure devcontainer: %w", err) - } - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Configured .devcontainer/devcontainer.json")) - } +func initRepositoryConfigureCodespaces(opts InitOptions) error { + if !opts.CodespaceEnabled { + return nil } + initLog.Printf("Configuring GitHub Codespaces devcontainer with additional repos: %v", opts.CodespaceRepos) + if err := ensureDevcontainerConfig(opts.Verbose, opts.CodespaceRepos); err != nil { + initLog.Printf("Failed to configure devcontainer: %v", err) + return fmt.Errorf("failed to configure devcontainer: %w", err) + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Configured .devcontainer/devcontainer.json")) + } + return nil +} - // Configure VSCode settings +func initRepositoryConfigureEditor(opts InitOptions) error { initLog.Print("Configuring VSCode settings") - - // Update .vscode/settings.json if err := ensureVSCodeSettings(opts.Verbose); err != nil { initLog.Printf("Failed to update VSCode settings: %v", err) return fmt.Errorf("failed to update VSCode settings: %w", err) @@ -183,45 +239,37 @@ func InitRepository(opts InitOptions) error { if opts.Verbose { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Updated .vscode/settings.json")) } - - // Install shell completions if requested if opts.Completions { - initLog.Print("Installing shell completions") - fmt.Fprintln(os.Stderr, "") - - if err := InstallShellCompletion(opts.Verbose, opts.RootCmd); err != nil { - initLog.Printf("Shell completion installation failed: %v", err) - // Don't fail init if completion installation has issues - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Shell completion installation encountered an issue: %v", err))) - } - fmt.Fprintln(os.Stderr, "") + initRepositoryInstallCompletions(opts) } + return nil +} - // Generate/update maintenance workflow if any workflows use expires field - initLog.Print("Checking for workflows with expires field to generate maintenance workflow") - if err := ensureMaintenanceWorkflow(ctx, opts.Verbose); err != nil { - initLog.Printf("Failed to generate maintenance workflow: %v", err) - // Don't fail init if maintenance workflow generation has issues - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to generate maintenance workflow: %v", err))) +func initRepositoryInstallCompletions(opts InitOptions) { + initLog.Print("Installing shell completions") + fmt.Fprintln(os.Stderr, "") + if err := InstallShellCompletion(opts.Verbose, opts.RootCmd); err != nil { + initLog.Printf("Shell completion installation failed: %v", err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Shell completion installation encountered an issue: %v", err))) } + fmt.Fprintln(os.Stderr, "") +} - initLog.Print("Repository initialization completed successfully") - - // If --create-pull-request is enabled, create branch, commit, push, and create PR - if opts.CreatePR { - initLog.Print("Create PR enabled - preparing to create branch, commit, push, and create PR") - fmt.Fprintln(os.Stderr, "") - - prBody := "This PR initializes the repository for agentic workflows by:\n" + - "- Configuring .gitattributes\n" + - "- Creating GitHub Copilot custom instructions\n" + - "- Setting up workflow prompts and skills" - if _, err := CreatePRWithChanges("init-agentic-workflows", "chore: initialize agentic workflows", "Initialize agentic workflows", prBody, opts.Verbose); err != nil { - return err - } +func initRepositoryCreatePR(opts InitOptions) error { + if !opts.CreatePR { + return nil } + initLog.Print("Create PR enabled - preparing to create branch, commit, push, and create PR") + fmt.Fprintln(os.Stderr, "") + prBody := "This PR initializes the repository for agentic workflows by:\n" + + "- Configuring .gitattributes\n" + + "- Creating GitHub Copilot custom instructions\n" + + "- Setting up workflow prompts and skills" + _, err := CreatePRWithChanges("init-agentic-workflows", "chore: initialize agentic workflows", "Initialize agentic workflows", prBody, opts.Verbose) + return err +} - // Display success message with next steps +func initRepositoryDisplaySuccess(opts InitOptions) { fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Repository initialized for agentic workflows!")) fmt.Fprintln(os.Stderr, "") @@ -233,8 +281,6 @@ func InitRepository(opts InitOptions) error { fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Or add an example workflow, see https://github.com/githubnext/agentics")) fmt.Fprintln(os.Stderr, "") - - return nil } // ensureMaintenanceWorkflow checks existing workflows for expires field and generates/updates @@ -256,20 +302,38 @@ func ensureMaintenanceWorkflow(ctx context.Context, verbose bool) error { return nil } + workflowDataList, compiler, err := ensureMaintenanceWorkflowParseWorkflows(workflowsDir) + if err != nil { + return err + } + + // Always call GenerateMaintenanceWorkflow even with empty list + // This allows it to delete existing maintenance workflow if no workflows have expires + initLog.Printf("Generating maintenance workflow for %d workflows", len(workflowDataList)) + + if err := ensureMaintenanceWorkflowGenerate(ctx, gitRoot, workflowsDir, workflowDataList, compiler); err != nil { + return fmt.Errorf("failed to generate maintenance workflow: %w", err) + } + + if verbose && len(workflowDataList) > 0 { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Generated/updated maintenance workflow")) + } + + return nil +} + +func ensureMaintenanceWorkflowParseWorkflows(workflowsDir string) ([]*workflow.WorkflowData, *workflow.Compiler, error) { // Find all workflow markdown files files, err := filepath.Glob(filepath.Join(workflowsDir, "*.md")) if err != nil { - return fmt.Errorf("failed to find workflow files: %w", err) + return nil, nil, fmt.Errorf("failed to find workflow files: %w", err) } - - // Filter out README.md files files = filterWorkflowFiles(files) // Create a compiler to parse workflows (version and action mode auto-detected) compiler := workflow.NewCompiler() initLog.Printf("Action mode detected for maintenance workflow: %s", compiler.GetActionMode()) - // Parse all workflows to collect WorkflowData var workflowDataList []*workflow.WorkflowData for _, file := range files { initLog.Printf("Parsing workflow: %s", file) @@ -279,22 +343,25 @@ func ensureMaintenanceWorkflow(ctx context.Context, verbose bool) error { initLog.Printf("Skipping workflow %s due to parse error: %v", file, err) continue } - workflowDataList = append(workflowDataList, workflowData) } + return workflowDataList, compiler, nil +} - // Always call GenerateMaintenanceWorkflow even with empty list - // This allows it to delete existing maintenance workflow if no workflows have expires - initLog.Printf("Generating maintenance workflow for %d workflows", len(workflowDataList)) - +func ensureMaintenanceWorkflowGenerate( + ctx context.Context, + gitRoot string, + workflowsDir string, + workflowDataList []*workflow.WorkflowData, + compiler *workflow.Compiler, +) error { // Load repo-level configuration (optional; errors are non-fatal during init). repoConfig, err := workflow.LoadRepoConfig(gitRoot) if err != nil { initLog.Printf("Failed to load repo config, using defaults: %v", err) repoConfig = nil } - - if err := workflow.GenerateMaintenanceWorkflow(ctx, workflow.GenerateMaintenanceWorkflowOptions{ + return workflow.GenerateMaintenanceWorkflow(ctx, workflow.GenerateMaintenanceWorkflowOptions{ WorkflowDataList: workflowDataList, WorkflowDir: workflowsDir, Version: GetVersion(), @@ -302,15 +369,7 @@ func ensureMaintenanceWorkflow(ctx context.Context, verbose bool) error { ActionTag: compiler.GetActionTag(), RepoConfig: repoConfig, RepoSlug: compiler.GetRepositorySlug(), - }); err != nil { - return fmt.Errorf("failed to generate maintenance workflow: %w", err) - } - - if verbose && len(workflowDataList) > 0 { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Generated/updated maintenance workflow")) - } - - return nil + }) } // isGHESHost returns true when the given host is a GitHub Enterprise Server instance, @@ -393,14 +452,9 @@ func ensureGHESRepoConfig(verbose bool) (bool, error) { configPath := filepath.Join(gitRoot, workflow.RepoConfigFileName) - // Read existing content or start with an empty document. - var doc map[string]any - if data, readErr := os.ReadFile(configPath); readErr == nil { - if jsonErr := json.Unmarshal(data, &doc); jsonErr != nil { - return false, fmt.Errorf("failed to parse %s: %w", workflow.RepoConfigFileName, jsonErr) - } - } else if !errors.Is(readErr, os.ErrNotExist) { - return false, fmt.Errorf("failed to read %s: %w", workflow.RepoConfigFileName, readErr) + doc, err := ensureGHESRepoConfigRead(configPath) + if err != nil { + return false, err } if doc == nil { @@ -412,14 +466,11 @@ func ensureGHESRepoConfig(verbose bool) (bool, error) { initLog.Print("aw.json already has ghes: true, nothing to update") return false, nil } - doc["ghes"] = true - - data, err := json.MarshalIndent(doc, "", " ") + data, err := ensureGHESRepoConfigMarshal(doc) if err != nil { - return false, fmt.Errorf("failed to serialise %s: %w", workflow.RepoConfigFileName, err) + return false, err } - data = append(data, '\n') if mkdirErr := fileutil.EnsureParentDir(configPath, constants.DirPermPublic); mkdirErr != nil { return false, fmt.Errorf("failed to create directory for %s: %w", workflow.RepoConfigFileName, mkdirErr) @@ -430,15 +481,38 @@ func ensureGHESRepoConfig(verbose bool) (bool, error) { } initLog.Printf("Wrote ghes: true to %s", configPath) + ensureGHESRepoConfigDisplay(verbose, ghesHost) + return true, nil +} +func ensureGHESRepoConfigRead(configPath string) (map[string]any, error) { + var doc map[string]any + if data, readErr := os.ReadFile(configPath); readErr == nil { + if jsonErr := json.Unmarshal(data, &doc); jsonErr != nil { + return nil, fmt.Errorf("failed to parse %s: %w", workflow.RepoConfigFileName, jsonErr) + } + } else if !errors.Is(readErr, os.ErrNotExist) { + return nil, fmt.Errorf("failed to read %s: %w", workflow.RepoConfigFileName, readErr) + } + return doc, nil +} + +func ensureGHESRepoConfigMarshal(doc map[string]any) ([]byte, error) { + data, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to serialise %s: %w", workflow.RepoConfigFileName, err) + } + return append(data, '\n'), nil +} + +func ensureGHESRepoConfigDisplay(verbose bool, ghesHost string) { if verbose { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage( fmt.Sprintf("Configured %s with ghes: true (GHES deployment detected: %s)", workflow.RepoConfigFileName, ghesHost), )) - } else { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage( - fmt.Sprintf("GHES deployment detected (%s): set ghes: true in %s for artifact compatibility", ghesHost, workflow.RepoConfigFileName), - )) + return } - return true, nil + fmt.Fprintln(os.Stderr, console.FormatInfoMessage( + fmt.Sprintf("GHES deployment detected (%s): set ghes: true in %s for artifact compatibility", ghesHost, workflow.RepoConfigFileName), + )) } diff --git a/pkg/cli/init_command.go b/pkg/cli/init_command.go index 73879f99ae5..4e5880a4026 100644 --- a/pkg/cli/init_command.go +++ b/pkg/cli/init_command.go @@ -16,9 +16,33 @@ var initCommandLog = logger.New("cli:init_command") // NewInitCommand creates the init command func NewInitCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "init", - Short: "Initialize the repository for agentic workflows", - Long: `Initialize the repository for agentic workflows by configuring .gitattributes and creating the dispatcher skill file. + Use: "init", + Short: "Initialize the repository for agentic workflows", + Long: newInitCommandLong(), + Example: newInitCommandExample(), + RunE: func(cmd *cobra.Command, args []string) error { + return newInitCommandRun(cmd, args) + }, + } + + addEngineFlag(cmd) + cmd.Flags().Bool("no-mcp", false, "Skip configuring gh-aw MCP server integration for GitHub Copilot Agent") + cmd.Flags().Bool("no-skill", false, "Skip creating the agentic-workflows dispatcher skill") + cmd.Flags().Bool("no-agent", false, "Skip creating the Agentic Workflows custom agent") + cmd.Flags().String("codespaces", "", "Create devcontainer.json for GitHub Codespaces with agentic workflow support. Specify comma-separated repository names in the same organization (e.g., repo1,repo2), or use with an empty value for the current repo only") + cmd.Flags().Bool("completions", false, "Install shell completion for the detected shell (bash, zsh, fish, or PowerShell)") + cmd.Flags().Bool("create-pull-request", false, "Create a pull request with the initialization changes") + cmd.Flags().Bool("pr", false, "Alias for --create-pull-request") + _ = cmd.Flags().MarkHidden("pr") // Hide the short alias from help output + cmd.Flags().Bool("mcp", false, "Configure GitHub Copilot Agent MCP server integration (deprecated, MCP is enabled by default)") + // Hide the deprecated --mcp flag from help (kept for backward compatibility) + _ = cmd.Flags().MarkHidden("mcp") + + return cmd +} + +func newInitCommandLong() string { + return `Initialize the repository for agentic workflows by configuring .gitattributes and creating the dispatcher skill file. This command performs non-interactive repository setup and does not prompt for engine selection or secret configuration. @@ -60,8 +84,11 @@ After running this command, you can: - Use GitHub Copilot Chat or coding agent tools with the agentic-workflows skill to get started with workflow tasks - The dispatcher skill will route your request to the appropriate specialized prompt - Add workflows from the catalog with: ` + string(constants.CLIExtensionPrefix) + ` add -- Create new workflows from scratch with: ` + string(constants.CLIExtensionPrefix) + ` new `, - Example: ` ` + string(constants.CLIExtensionPrefix) + ` init # Initialize repository with defaults +- Create new workflows from scratch with: ` + string(constants.CLIExtensionPrefix) + ` new ` +} + +func newInitCommandExample() string { + return ` ` + string(constants.CLIExtensionPrefix) + ` init # Initialize repository with defaults ` + string(constants.CLIExtensionPrefix) + ` init -v # Initialize with verbose output ` + string(constants.CLIExtensionPrefix) + ` init --engine claude # Skip Copilot-specific artifacts ` + string(constants.CLIExtensionPrefix) + ` init --no-mcp # Skip MCP configuration @@ -70,86 +97,81 @@ After running this command, you can: ` + string(constants.CLIExtensionPrefix) + ` init --codespaces "" # Configure Codespaces for current repo only ` + string(constants.CLIExtensionPrefix) + ` init --codespaces repo1,repo2 # Codespaces with additional repos ` + string(constants.CLIExtensionPrefix) + ` init --completions # Install shell completions - ` + string(constants.CLIExtensionPrefix) + ` init --create-pull-request # Initialize and create a pull request`, - RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Flags().GetBool("verbose") - mcpFlag, _ := cmd.Flags().GetBool("mcp") - noMcp, _ := cmd.Flags().GetBool("no-mcp") - noSkill, _ := cmd.Flags().GetBool("no-skill") - noAgent, _ := cmd.Flags().GetBool("no-agent") - engineOverride, _ := cmd.Flags().GetString("engine") - codespaceReposStr, _ := cmd.Flags().GetString("codespaces") - codespaceEnabled := cmd.Flags().Changed("codespaces") - completions, _ := cmd.Flags().GetBool("completions") - createPRFlag, _ := cmd.Flags().GetBool("create-pull-request") - prFlagAlias, _ := cmd.Flags().GetBool("pr") - createPR := createPRFlag || prFlagAlias // Support both --create-pull-request and --pr - - if engineOverride != "" { - registry := workflow.GetGlobalEngineRegistry() - if !registry.IsValidEngine(engineOverride) { - supportedEngines := registry.GetSupportedEngines() - sort.Strings(supportedEngines) - return fmt.Errorf("invalid engine value '%s'. Must be one of: %s", engineOverride, strings.Join(supportedEngines, ", ")) - } - } - - // Determine MCP state: default true, unless --no-mcp is specified - // --mcp flag is kept for backward compatibility (hidden from help) - mcp := !noMcp - if cmd.Flags().Changed("mcp") { - // If --mcp is explicitly set, use it (backward compatibility) - mcp = mcpFlag - } - - // Trim the codespace repos string (explicit value required; use --codespaces "" for current repo only) - codespaceReposStr = strings.TrimSpace(codespaceReposStr) - - // Parse codespace repos from comma-separated string - var codespaceRepos []string - if codespaceReposStr != "" { - codespaceRepos = strings.Split(codespaceReposStr, ",") - // Trim spaces from each repo name - for i, repo := range codespaceRepos { - codespaceRepos[i] = strings.TrimSpace(repo) - } - } - - initCommandLog.Printf("Executing init command: verbose=%v, skill=%v, agent=%v, mcp=%v, codespaces=%v, codespaceEnabled=%v, completions=%v, createPR=%v", verbose, !noSkill, !noAgent, mcp, codespaceRepos, codespaceEnabled, completions, createPR) - opts := InitOptions{ - Ctx: cmd.Context(), - Verbose: verbose, - Engine: engineOverride, - Skill: !noSkill, - Agent: !noAgent, - MCP: mcp, - CodespaceRepos: codespaceRepos, - CodespaceEnabled: codespaceEnabled, - Completions: completions, - CreatePR: createPR, - RootCmd: cmd.Root(), - } - if err := InitRepository(opts); err != nil { - initCommandLog.Printf("Init command failed: %v", err) - return err - } - initCommandLog.Print("Init command completed successfully") - return nil - }, + ` + string(constants.CLIExtensionPrefix) + ` init --create-pull-request # Initialize and create a pull request` +} + +func newInitCommandRun(cmd *cobra.Command, args []string) error { + opts, err := newInitCommandOptions(cmd) + if err != nil { + return err + } + if err := InitRepository(opts); err != nil { + initCommandLog.Printf("Init command failed: %v", err) + return err } + initCommandLog.Print("Init command completed successfully") + return nil +} - addEngineFlag(cmd) - cmd.Flags().Bool("no-mcp", false, "Skip configuring gh-aw MCP server integration for GitHub Copilot Agent") - cmd.Flags().Bool("no-skill", false, "Skip creating the agentic-workflows dispatcher skill") - cmd.Flags().Bool("no-agent", false, "Skip creating the Agentic Workflows custom agent") - cmd.Flags().String("codespaces", "", "Create devcontainer.json for GitHub Codespaces with agentic workflow support. Specify comma-separated repository names in the same organization (e.g., repo1,repo2), or use with an empty value for the current repo only") - cmd.Flags().Bool("completions", false, "Install shell completion for the detected shell (bash, zsh, fish, or PowerShell)") - cmd.Flags().Bool("create-pull-request", false, "Create a pull request with the initialization changes") - cmd.Flags().Bool("pr", false, "Alias for --create-pull-request") - _ = cmd.Flags().MarkHidden("pr") // Hide the short alias from help output - cmd.Flags().Bool("mcp", false, "Configure GitHub Copilot Agent MCP server integration (deprecated, MCP is enabled by default)") - // Hide the deprecated --mcp flag from help (kept for backward compatibility) - _ = cmd.Flags().MarkHidden("mcp") +func newInitCommandOptions(cmd *cobra.Command) (InitOptions, error) { + verbose, _ := cmd.Flags().GetBool("verbose") + mcpFlag, _ := cmd.Flags().GetBool("mcp") + noMcp, _ := cmd.Flags().GetBool("no-mcp") + noSkill, _ := cmd.Flags().GetBool("no-skill") + noAgent, _ := cmd.Flags().GetBool("no-agent") + engineOverride, _ := cmd.Flags().GetString("engine") + completions, _ := cmd.Flags().GetBool("completions") + createPRFlag, _ := cmd.Flags().GetBool("create-pull-request") + prFlagAlias, _ := cmd.Flags().GetBool("pr") + + if err := newInitCommandValidateEngine(engineOverride); err != nil { + return InitOptions{}, err + } + mcp := !noMcp + if cmd.Flags().Changed("mcp") { + mcp = mcpFlag + } + codespaceRepos, codespaceEnabled := newInitCommandCodespaces(cmd) + createPR := createPRFlag || prFlagAlias + initCommandLog.Printf("Executing init command: verbose=%v, skill=%v, agent=%v, mcp=%v, codespaces=%v, codespaceEnabled=%v, completions=%v, createPR=%v", verbose, !noSkill, !noAgent, mcp, codespaceRepos, codespaceEnabled, completions, createPR) + return InitOptions{ + Ctx: cmd.Context(), + Verbose: verbose, + Engine: engineOverride, + Skill: !noSkill, + Agent: !noAgent, + MCP: mcp, + CodespaceRepos: codespaceRepos, + CodespaceEnabled: codespaceEnabled, + Completions: completions, + CreatePR: createPR, + RootCmd: cmd.Root(), + }, nil +} - return cmd +func newInitCommandValidateEngine(engineOverride string) error { + if engineOverride == "" { + return nil + } + registry := workflow.GetGlobalEngineRegistry() + if registry.IsValidEngine(engineOverride) { + return nil + } + supportedEngines := registry.GetSupportedEngines() + sort.Strings(supportedEngines) + return fmt.Errorf("invalid engine value '%s'. Must be one of: %s", engineOverride, strings.Join(supportedEngines, ", ")) +} + +func newInitCommandCodespaces(cmd *cobra.Command) ([]string, bool) { + codespaceReposStr, _ := cmd.Flags().GetString("codespaces") + codespaceEnabled := cmd.Flags().Changed("codespaces") + codespaceReposStr = strings.TrimSpace(codespaceReposStr) + if codespaceReposStr == "" { + return nil, codespaceEnabled + } + codespaceRepos := strings.Split(codespaceReposStr, ",") + for i, repo := range codespaceRepos { + codespaceRepos[i] = strings.TrimSpace(repo) + } + return codespaceRepos, codespaceEnabled } diff --git a/pkg/cli/interactive.go b/pkg/cli/interactive.go index ed0c8a38d9d..520656314a7 100644 --- a/pkg/cli/interactive.go +++ b/pkg/cli/interactive.go @@ -152,8 +152,41 @@ func (b *InteractiveWorkflowBuilder) promptForConfiguration() error { return b.promptForConfigurationFrom(os.Stdin) } - // Prepare trigger options - triggerOptions := []huh.Option[string]{ + triggerOptions := promptForConfigurationTriggerOptions() + engineOptions := promptForConfigurationEngineOptions() + toolOptions := promptForConfigurationToolOptions() + + // Prepare safe output options programmatically from safe_outputs_tools.json + outputOptions := buildSafeOutputOptions() + + // Pre-detect network access based on repo contents + detectedNetworks := detectNetworkFromRepo() + interactiveLog.Printf("Pre-detected networks from repo: %v", detectedNetworks) + + networkOptions := promptForConfigurationNetworkOptions(detectedNetworks) + b.NetworkAccess = promptForConfigurationDefaultNetwork(detectedNetworks) + + // Variables to hold multi-select results + var selectedTools []string + var selectedOutputs []string + + form := promptForConfigurationForm(b, triggerOptions, engineOptions, toolOptions, outputOptions, networkOptions, &selectedTools, &selectedOutputs) + + if err := form.RunWithContext(b.ctx); err != nil { + return err + } + + // Store the multi-select results + b.Tools = selectedTools + b.SafeOutputs = selectedOutputs + + interactiveLog.Printf("User configuration selected: trigger=%s, engine=%s, tools=%v, safe_outputs=%v", b.Trigger, b.Engine, selectedTools, selectedOutputs) + + return nil +} + +func promptForConfigurationTriggerOptions() []huh.Option[string] { + return []huh.Option[string]{ huh.NewOption("Manual trigger (workflow_dispatch)", "workflow_dispatch"), huh.NewOption("Issue opened or reopened", "issues"), huh.NewOption("Pull request opened or synchronized", "pull_request"), @@ -163,17 +196,19 @@ func (b *InteractiveWorkflowBuilder) promptForConfiguration() error { huh.NewOption("Schedule (weekly on Monday, scattered execution time)", "schedule_weekly"), huh.NewOption("Command trigger (/bot-name)", "command"), } +} - // Prepare engine options - engineOptions := []huh.Option[string]{ +func promptForConfigurationEngineOptions() []huh.Option[string] { + return []huh.Option[string]{ huh.NewOption("copilot - GitHub Copilot CLI", "copilot"), huh.NewOption("claude - Anthropic Claude Code coding agent", "claude"), huh.NewOption("codex - OpenAI Codex engine", "codex"), huh.NewOption("gemini - Google Gemini CLI", "gemini"), } +} - // Prepare tool options - toolOptions := []huh.Option[string]{ +func promptForConfigurationToolOptions() []huh.Option[string] { + return []huh.Option[string]{ huh.NewOption("github - GitHub API tools (issues, PRs, comments, repos)", "github"), huh.NewOption("edit - File editing tools", "edit"), huh.NewOption("bash - Shell command tools", "bash"), @@ -181,106 +216,65 @@ func (b *InteractiveWorkflowBuilder) promptForConfiguration() error { huh.NewOption("web-search - Web search tools", "web-search"), huh.NewOption("playwright - Browser automation tools", "playwright"), } +} - // Prepare safe output options programmatically from safe_outputs_tools.json - outputOptions := buildSafeOutputOptions() - - // Pre-detect network access based on repo contents - detectedNetworks := detectNetworkFromRepo() - interactiveLog.Printf("Pre-detected networks from repo: %v", detectedNetworks) - - // Prepare network options +func promptForConfigurationNetworkOptions(detectedNetworks []string) []huh.Option[string] { networkOptions := []huh.Option[string]{ huh.NewOption("defaults - Basic infrastructure only", "defaults"), huh.NewOption("ecosystem - Common development ecosystems (Python, Node.js, Go, etc.)", "ecosystem"), } if len(detectedNetworks) > 0 { - // Build a custom option that reflects what was auto-detected label := "detected - Auto-detected ecosystems: " + strings.Join(detectedNetworks, ", ") - networkOptions = append([]huh.Option[string]{huh.NewOption(label, strings.Join(append([]string{"defaults"}, detectedNetworks...), ","))}, networkOptions...) + value := strings.Join(append([]string{"defaults"}, detectedNetworks...), ",") + networkOptions = append([]huh.Option[string]{huh.NewOption(label, value)}, networkOptions...) } + return networkOptions +} - // Set default network access - b.NetworkAccess = "defaults" +func promptForConfigurationDefaultNetwork(detectedNetworks []string) string { if len(detectedNetworks) > 0 { - b.NetworkAccess = strings.Join(append([]string{"defaults"}, detectedNetworks...), ",") + return strings.Join(append([]string{"defaults"}, detectedNetworks...), ",") } + return "defaults" +} - // Variables to hold multi-select results - var selectedTools []string - var selectedOutputs []string - - // Create form with organized groups - form := console.NewForm( - // Group 1: Basic Configuration - huh.NewGroup( - huh.NewSelect[string](). - Title("When should this workflow run?"). - Description("Choose the GitHub event that triggers this workflow"). - Options(triggerOptions...). - Height(8). - Value(&b.Trigger), - huh.NewSelect[string](). - Title("Which AI engine should process this workflow?"). - Description("The AI engine interprets instructions and executes tasks using available tools"). - Options(engineOptions...). - Value(&b.Engine), - ). - Title("Basic Configuration"). - Description("Let's start with the fundamentals of your workflow"), - - // Group 2: Capabilities - huh.NewGroup( - huh.NewMultiSelect[string](). - Title("Which tools should the AI have access to?"). - Description("Tools enable the AI to interact with code, APIs, and external systems"). - Options(toolOptions...). - Height(8). - Value(&selectedTools), - huh.NewMultiSelect[string](). - Title("What outputs should the AI be able to create?"). - Description("Safe outputs allow the AI to create GitHub resources after human approval"). - Options(outputOptions...). - Height(10). - Value(&selectedOutputs), - ). - Title("Capabilities"). - Description("Select the tools and outputs your workflow needs"), - - // Group 3: Network & Security - huh.NewGroup( - huh.NewSelect[string](). - Title("What network access does the workflow need?"). - Description("Network access controls which external domains the workflow can reach"). - Options(networkOptions...). - Value(&b.NetworkAccess), - ). - Title("Network & Security"). - Description("Configure network access and security settings"), - - // Group 4: Instructions - huh.NewGroup( - huh.NewText(). - Title("Describe what this workflow should do:"). - Description("Provide clear, detailed instructions for the AI to follow when executing this workflow"). - Value(&b.Intent). - Validate(ValidateWorkflowIntent), - ). - Title("Instructions"). - Description("Describe what you want this workflow to accomplish"), +func promptForConfigurationForm( + b *InteractiveWorkflowBuilder, + triggerOptions, engineOptions, toolOptions, outputOptions, networkOptions []huh.Option[string], + selectedTools, selectedOutputs *[]string, +) *huh.Form { + return console.NewForm( + promptForConfigurationBasicGroup(b, triggerOptions, engineOptions), + promptForConfigurationCapabilitiesGroup(toolOptions, outputOptions, selectedTools, selectedOutputs), + promptForConfigurationNetworkGroup(b, networkOptions), + promptForConfigurationInstructionsGroup(b), ) +} - if err := form.RunWithContext(b.ctx); err != nil { - return err - } +func promptForConfigurationBasicGroup(b *InteractiveWorkflowBuilder, triggerOptions, engineOptions []huh.Option[string]) *huh.Group { + return huh.NewGroup( + huh.NewSelect[string]().Title("When should this workflow run?").Description("Choose the GitHub event that triggers this workflow").Options(triggerOptions...).Height(8).Value(&b.Trigger), + huh.NewSelect[string]().Title("Which AI engine should process this workflow?").Description("The AI engine interprets instructions and executes tasks using available tools").Options(engineOptions...).Value(&b.Engine), + ).Title("Basic Configuration").Description("Let's start with the fundamentals of your workflow") +} - // Store the multi-select results - b.Tools = selectedTools - b.SafeOutputs = selectedOutputs +func promptForConfigurationCapabilitiesGroup(toolOptions, outputOptions []huh.Option[string], selectedTools, selectedOutputs *[]string) *huh.Group { + return huh.NewGroup( + huh.NewMultiSelect[string]().Title("Which tools should the AI have access to?").Description("Tools enable the AI to interact with code, APIs, and external systems").Options(toolOptions...).Height(8).Value(selectedTools), + huh.NewMultiSelect[string]().Title("What outputs should the AI be able to create?").Description("Safe outputs allow the AI to create GitHub resources after human approval").Options(outputOptions...).Height(10).Value(selectedOutputs), + ).Title("Capabilities").Description("Select the tools and outputs your workflow needs") +} - interactiveLog.Printf("User configuration selected: trigger=%s, engine=%s, tools=%v, safe_outputs=%v", b.Trigger, b.Engine, selectedTools, selectedOutputs) +func promptForConfigurationNetworkGroup(b *InteractiveWorkflowBuilder, networkOptions []huh.Option[string]) *huh.Group { + return huh.NewGroup( + huh.NewSelect[string]().Title("What network access does the workflow need?").Description("Network access controls which external domains the workflow can reach").Options(networkOptions...).Value(&b.NetworkAccess), + ).Title("Network & Security").Description("Configure network access and security settings") +} - return nil +func promptForConfigurationInstructionsGroup(b *InteractiveWorkflowBuilder) *huh.Group { + return huh.NewGroup( + huh.NewText().Title("Describe what this workflow should do:").Description("Provide clear, detailed instructions for the AI to follow when executing this workflow").Value(&b.Intent).Validate(ValidateWorkflowIntent), + ).Title("Instructions").Description("Describe what you want this workflow to accomplish") } // promptForConfigurationFrom is the non-TTY fallback for promptForConfiguration. @@ -291,8 +285,39 @@ func (b *InteractiveWorkflowBuilder) promptForConfigurationFrom(r io.Reader) err interactiveLog.Print("Non-TTY detected, using text prompts for configuration") scanner := b.ensureNonTTYScanner(r) - // --- Trigger (single-select) --- - triggerOptions := []struct{ label, value string }{ + if err := promptForConfigurationFromSelections(b, scanner); err != nil { + return err + } + + // --- Intent / instructions (free text) --- + if err := promptForConfigurationFromIntent(b, scanner); err != nil { + return err + } + + interactiveLog.Printf("Non-TTY configuration selected: trigger=%s, engine=%s, tools=%v, safe_outputs=%v", b.Trigger, b.Engine, b.Tools, b.SafeOutputs) + return nil +} + +type promptForConfigurationFromItem = struct{ label, value string } + +func promptForConfigurationFromSelections(b *InteractiveWorkflowBuilder, scanner *bufio.Scanner) error { + if err := promptForConfigurationFromTrigger(b, scanner); err != nil { + return err + } + if err := promptForConfigurationFromEngine(b, scanner); err != nil { + return err + } + if err := promptForConfigurationFromTools(b, scanner); err != nil { + return err + } + if err := promptForConfigurationFromSafeOutputs(b, scanner); err != nil { + return err + } + return promptForConfigurationFromNetwork(b, scanner) +} + +func promptForConfigurationFromTrigger(b *InteractiveWorkflowBuilder, scanner *bufio.Scanner) error { + triggerOptions := []promptForConfigurationFromItem{ {"Manual trigger (workflow_dispatch)", "workflow_dispatch"}, {"Issue opened or reopened", "issues"}, {"Pull request opened or synchronized", "pull_request"}, @@ -307,9 +332,11 @@ func (b *InteractiveWorkflowBuilder) promptForConfigurationFrom(r io.Reader) err return fmt.Errorf("failed to select trigger: %w", err) } b.Trigger = trigger + return nil +} - // --- Engine (single-select) --- - engineOptions := []struct{ label, value string }{ +func promptForConfigurationFromEngine(b *InteractiveWorkflowBuilder, scanner *bufio.Scanner) error { + engineOptions := []promptForConfigurationFromItem{ {"copilot - GitHub Copilot CLI", "copilot"}, {"claude - Anthropic Claude Code coding agent", "claude"}, {"codex - OpenAI Codex engine", "codex"}, @@ -320,9 +347,11 @@ func (b *InteractiveWorkflowBuilder) promptForConfigurationFrom(r io.Reader) err return fmt.Errorf("failed to select engine: %w", err) } b.Engine = engine + return nil +} - // --- Tools (multi-select) --- - toolOptions := []struct{ label, value string }{ +func promptForConfigurationFromTools(b *InteractiveWorkflowBuilder, scanner *bufio.Scanner) error { + toolOptions := []promptForConfigurationFromItem{ {"github - GitHub API tools (issues, PRs, comments, repos)", "github"}, {"edit - File editing tools", "edit"}, {"bash - Shell command tools", "bash"}, @@ -335,37 +364,42 @@ func (b *InteractiveWorkflowBuilder) promptForConfigurationFrom(r io.Reader) err return fmt.Errorf("failed to select tools: %w", err) } b.Tools = tools + return nil +} - // --- Safe outputs (multi-select) --- +func promptForConfigurationFromSafeOutputs(b *InteractiveWorkflowBuilder, scanner *bufio.Scanner) error { outputOptions := buildSafeOutputOptions() - safeOutputItems := make([]struct{ label, value string }, len(outputOptions)) + safeOutputItems := make([]promptForConfigurationFromItem, len(outputOptions)) for i, opt := range outputOptions { - safeOutputItems[i] = struct{ label, value string }{opt.Key, opt.Value} + safeOutputItems[i] = promptForConfigurationFromItem{opt.Key, opt.Value} } safeOutputs, err := promptNonInteractiveMultiSelect(scanner, "What outputs should the AI be able to create? (comma-separated values or numbers, or leave blank for none)", safeOutputItems) if err != nil { return fmt.Errorf("failed to select safe outputs: %w", err) } b.SafeOutputs = safeOutputs + return nil +} - // --- Network access (single-select) --- - detectedNetworks := detectNetworkFromRepo() - networkItems := []struct{ label, value string }{ +func promptForConfigurationFromNetwork(b *InteractiveWorkflowBuilder, scanner *bufio.Scanner) error { + networkItems := []promptForConfigurationFromItem{ {"defaults - Basic infrastructure only", "defaults"}, {"ecosystem - Common development ecosystems (Python, Node.js, Go, etc.)", "ecosystem"}, } - if len(detectedNetworks) > 0 { + if detectedNetworks := detectNetworkFromRepo(); len(detectedNetworks) > 0 { label := "detected - Auto-detected ecosystems: " + strings.Join(detectedNetworks, ", ") value := strings.Join(append([]string{"defaults"}, detectedNetworks...), ",") - networkItems = append([]struct{ label, value string }{{label, value}}, networkItems...) + networkItems = append([]promptForConfigurationFromItem{{label, value}}, networkItems...) } network, err := promptNonInteractiveSelect(scanner, "What network access does the workflow need?", networkItems) if err != nil { return fmt.Errorf("failed to select network access: %w", err) } b.NetworkAccess = network + return nil +} - // --- Intent / instructions (free text) --- +func promptForConfigurationFromIntent(b *InteractiveWorkflowBuilder, scanner *bufio.Scanner) error { fmt.Fprintf(os.Stderr, "\nDescribe what this workflow should do (enter text, then press Enter):\n> ") if scanner.Scan() { b.Intent = strings.TrimSpace(scanner.Text()) @@ -375,14 +409,12 @@ func (b *InteractiveWorkflowBuilder) promptForConfigurationFrom(r io.Reader) err if err := ValidateWorkflowIntent(b.Intent); err != nil { return fmt.Errorf("invalid workflow intent: %w", err) } - - interactiveLog.Printf("Non-TTY configuration selected: trigger=%s, engine=%s, tools=%v, safe_outputs=%v", b.Trigger, b.Engine, b.Tools, b.SafeOutputs) return nil } // promptNonInteractiveSelect prints a numbered list and reads a single selection. // The user may enter a number (1-based index) or the option value directly. -func promptNonInteractiveSelect(scanner *bufio.Scanner, title string, options []struct{ label, value string }) (string, error) { +func promptNonInteractiveSelect(scanner *bufio.Scanner, title string, options []promptForConfigurationFromItem) (string, error) { fmt.Fprintf(os.Stderr, "\n%s\n", title) for i, opt := range options { fmt.Fprintf(os.Stderr, " %d) %s\n", i+1, opt.label) @@ -416,7 +448,7 @@ func promptNonInteractiveSelect(scanner *bufio.Scanner, title string, options [] // promptNonInteractiveMultiSelect prints a numbered list and reads comma-separated selections. // Each token may be a 1-based index or an option value. An empty input selects nothing. -func promptNonInteractiveMultiSelect(scanner *bufio.Scanner, title string, options []struct{ label, value string }) ([]string, error) { +func promptNonInteractiveMultiSelect(scanner *bufio.Scanner, title string, options []promptForConfigurationFromItem) ([]string, error) { fmt.Fprintf(os.Stderr, "\n%s\n", title) for i, opt := range options { fmt.Fprintf(os.Stderr, " %d) %s\n", i+1, opt.label) @@ -568,7 +600,12 @@ func (b *InteractiveWorkflowBuilder) generateWorkflowContent() string { fmt.Fprintf(&content, "%s\n\n", b.Intent) } - // Add TODO sections for customization + b.generateWorkflowContentTodo(&content) + + return content.String() +} + +func (b *InteractiveWorkflowBuilder) generateWorkflowContentTodo(content *strings.Builder) { content.WriteString("\n") - - return content.String() } // Helper methods for generating configuration sections diff --git a/pkg/cli/jsonworkflow_to_markdown.go b/pkg/cli/jsonworkflow_to_markdown.go index f496eadebdf..b73c3f07a0b 100644 --- a/pkg/cli/jsonworkflow_to_markdown.go +++ b/pkg/cli/jsonworkflow_to_markdown.go @@ -166,132 +166,149 @@ func ConvertJSONWorkflowToMarkdown(a *JSONWorkflow, opts ConvertOptions) (*Gener jsonWorkflowLog.Printf("Converting JSON workflow: id=%q name=%q filename=%q", a.ID, a.Name, filename) // ── Build frontmatter ──────────────────────────────────────────────────────── + fm, frontmatterWarnings := convertJSONWorkflowToMarkdownFrontmatter(a) + warnings = append(warnings, frontmatterWarnings...) + + // ── Build body ─────────────────────────────────────────────────────────────── + markdown := fm + "\n" + convertJSONWorkflowToMarkdownBody(a) + + jsonWorkflowLog.Printf("Generated workflow %q: %d byte(s), %d warning(s)", filename, len(markdown), len(warnings)) + + return &GeneratedWorkflow{ + Filename: filename, + Markdown: markdown, + Warnings: warnings, + }, nil +} + +func convertJSONWorkflowToMarkdownFrontmatter(a *JSONWorkflow) (string, []string) { + var warnings []string var fm strings.Builder fm.WriteString("---\n") + convertJSONWorkflowToMarkdownBasics(&fm, a) + warnings = append(warnings, convertJSONWorkflowToMarkdownOn(&fm, a)...) + warnings = append(warnings, convertJSONWorkflowToMarkdownTools(&fm, a)...) + warnings = append(warnings, convertJSONWorkflowToMarkdownPermissions(&fm, a)...) + convertJSONWorkflowToMarkdownTags(&fm, a) + warnings = append(warnings, convertJSONWorkflowToMarkdownExtra(&fm, a)...) + fm.WriteString("---\n") + return fm.String(), warnings +} +func convertJSONWorkflowToMarkdownBasics(fm *strings.Builder, a *JSONWorkflow) { if a.Description != "" { fm.WriteString("description: ") fm.WriteString(yamlQuoteString(a.Description)) fm.WriteString("\n") } - if a.Engine != "" { fm.WriteString("engine: ") fm.WriteString(a.Engine) fm.WriteString("\n") } +} - // "on:" is resolved from On (explicit, takes precedence) or converted from Triggers. - onVal, triggerWarnings := resolveOnValue(a) - warnings = append(warnings, triggerWarnings...) - if onVal != nil { - if s, ok := onVal.(string); ok { - // Scalar shorthand e.g. "on: hourly" - fm.WriteString("on: ") - fm.WriteString(s) - fm.WriteString("\n") - } else { - onYAML, err := marshalFrontmatterValue(onVal) - if err == nil { - fm.WriteString("on:\n") - for line := range strings.SplitSeq(onYAML, "\n") { - if line == "" { - continue - } - fm.WriteString(" ") - fm.WriteString(line) - fm.WriteString("\n") - } - } else { - warnings = append(warnings, fmt.Sprintf("could not serialize 'on' field: %v", err)) - } - } +func convertJSONWorkflowToMarkdownOn(fm *strings.Builder, a *JSONWorkflow) []string { + onVal, warnings := resolveOnValue(a) + if onVal == nil { + return warnings + } + if s, ok := onVal.(string); ok { + fm.WriteString("on: ") + fm.WriteString(s) + fm.WriteString("\n") + return warnings + } + if onYAML, err := marshalFrontmatterValue(onVal); err == nil { + convertJSONWorkflowToMarkdownIndentedYAML(fm, "on", onYAML) + } else { + warnings = append(warnings, fmt.Sprintf("could not serialize 'on' field: %v", err)) + } + return warnings +} + +func convertJSONWorkflowToMarkdownTools(fm *strings.Builder, a *JSONWorkflow) []string { + if len(a.Tools) == 0 { + return nil + } + toolsConfig, warnings := convertToolsToConfig(a.Tools) + if len(toolsConfig) == 0 { + return warnings + } + if toolsYAML, err := marshalFrontmatterValue(toolsConfig); err == nil { + convertJSONWorkflowToMarkdownIndentedYAML(fm, "tools", toolsYAML) + } else { + warnings = append(warnings, fmt.Sprintf("could not serialize 'tools' field: %v", err)) } + return warnings +} - if len(a.Tools) > 0 { - toolsConfig, toolWarnings := convertToolsToConfig(a.Tools) - warnings = append(warnings, toolWarnings...) - if len(toolsConfig) > 0 { - toolsYAML, err := marshalFrontmatterValue(toolsConfig) - if err == nil { - fm.WriteString("tools:\n") - for line := range strings.SplitSeq(toolsYAML, "\n") { - if line == "" { - continue - } - fm.WriteString(" ") - fm.WriteString(line) - fm.WriteString("\n") - } - } else { - warnings = append(warnings, fmt.Sprintf("could not serialize 'tools' field: %v", err)) - } - } +func convertJSONWorkflowToMarkdownPermissions(fm *strings.Builder, a *JSONWorkflow) []string { + if len(a.Permissions) == 0 { + return nil + } + permYAML, err := marshalFrontmatterValue(a.Permissions) + if err != nil { + return []string{fmt.Sprintf("could not serialize 'permissions' field: %v", err)} } + convertJSONWorkflowToMarkdownIndentedYAML(fm, "permissions", permYAML) + return nil +} - if len(a.Permissions) > 0 { - permYAML, err := marshalFrontmatterValue(a.Permissions) - if err == nil { - fm.WriteString("permissions:\n") - for line := range strings.SplitSeq(permYAML, "\n") { - if line == "" { - continue - } - fm.WriteString(" ") - fm.WriteString(line) - fm.WriteString("\n") - } - } else { - warnings = append(warnings, fmt.Sprintf("could not serialize 'permissions' field: %v", err)) - } +func convertJSONWorkflowToMarkdownTags(fm *strings.Builder, a *JSONWorkflow) { + if len(a.Tags) == 0 { + return + } + fm.WriteString("tags:\n") + for _, tag := range a.Tags { + fm.WriteString(" - ") + fm.WriteString(yamlQuoteString(tag)) + fm.WriteString("\n") } +} - if len(a.Tags) > 0 { - fm.WriteString("tags:\n") - for _, tag := range a.Tags { - fm.WriteString(" - ") - fm.WriteString(yamlQuoteString(tag)) +func convertJSONWorkflowToMarkdownExtra(fm *strings.Builder, a *JSONWorkflow) []string { + if len(a.Extra) == 0 { + return nil + } + extraYAML, err := marshalFrontmatterValue(a.Extra) + if err != nil { + return []string{fmt.Sprintf("could not serialize unsupported fields: %v", err)} + } + fm.WriteString("# Unsupported fields preserved from source JSON:\n") + for line := range strings.SplitSeq(extraYAML, "\n") { + if line != "" { + fm.WriteString("# ") + fm.WriteString(line) fm.WriteString("\n") } } + var warnings []string + for _, k := range sliceutil.SortedKeys(a.Extra) { + warnings = append(warnings, fmt.Sprintf("field %q has no gh-aw frontmatter equivalent and was preserved as a comment", k)) + } + return warnings +} - // Emit unknown fields as YAML comments so the file remains valid YAML while - // preserving the original data for the operator to inspect. - if len(a.Extra) > 0 { - extraYAML, err := marshalFrontmatterValue(a.Extra) - if err == nil { - fm.WriteString("# Unsupported fields preserved from source JSON:\n") - for line := range strings.SplitSeq(extraYAML, "\n") { - if line == "" { - continue - } - fm.WriteString("# ") - fm.WriteString(line) - fm.WriteString("\n") - } - // Sort keys for deterministic warning output. - extraKeys := sliceutil.SortedKeys(a.Extra) - for _, k := range extraKeys { - warnings = append(warnings, fmt.Sprintf("field %q has no gh-aw frontmatter equivalent and was preserved as a comment", k)) - } - } else { - warnings = append(warnings, fmt.Sprintf("could not serialize unsupported fields: %v", err)) +func convertJSONWorkflowToMarkdownIndentedYAML(fm *strings.Builder, key, yamlValue string) { + fm.WriteString(key + ":\n") + for line := range strings.SplitSeq(yamlValue, "\n") { + if line == "" { + continue } + fm.WriteString(" ") + fm.WriteString(line) + fm.WriteString("\n") } +} - fm.WriteString("---\n") - - // ── Build body ─────────────────────────────────────────────────────────────── +func convertJSONWorkflowToMarkdownBody(a *JSONWorkflow) string { var body strings.Builder - - // Heading from name (or ID as fallback). - heading := a.Name - if heading != "" { + if a.Name != "" { body.WriteString("# ") - body.WriteString(heading) + body.WriteString(a.Name) body.WriteString("\n\n") } - if a.Instructions != "" { body.WriteString(strings.TrimRight(a.Instructions, "\n")) body.WriteString("\n") @@ -300,16 +317,7 @@ func ConvertJSONWorkflowToMarkdown(a *JSONWorkflow, opts ConvertOptions) (*Gener body.WriteString(strings.TrimRight(a.Prompt, "\n")) body.WriteString("\n") } - - markdown := fm.String() + "\n" + body.String() - - jsonWorkflowLog.Printf("Generated workflow %q: %d byte(s), %d warning(s)", filename, len(markdown), len(warnings)) - - return &GeneratedWorkflow{ - Filename: filename, - Markdown: markdown, - Warnings: warnings, - }, nil + return body.String() } // filenameFromJSONWorkflow derives a kebab-cased filename slug from the workflow's name @@ -403,43 +411,9 @@ func convertTriggersToOn(t *JSONWorkflowTriggers) (any, []string) { // parts accumulates the trigger map entries. parts := map[string]any{} - if t.Interval != nil && len(t.Interval.Types) > 0 { - intervalType := t.Interval.Types[0] - if len(t.Interval.Types) > 1 { - warnings = append(warnings, fmt.Sprintf( - "triggers.interval has multiple types %v; only the first (%q) will be used", - t.Interval.Types, intervalType)) - } - switch intervalType { - case "hourly", "daily", "weekly": - parts["_interval"] = intervalType - default: - warnings = append(warnings, fmt.Sprintf("triggers.interval type %q is not supported; skipped", intervalType)) - } - } - - if t.Issues != nil && len(t.Issues.Types) > 0 { - issueEntry := map[string]any{"types": t.Issues.Types} - if t.Issues.Query != "" { - warnings = append(warnings, `triggers.issues.query has no gh-aw equivalent; add a skip-if-match block manually if needed`) - } - parts["issues"] = issueEntry - } - - if t.WorkflowRun != nil { - if len(t.WorkflowRun.Workflows) == 0 || len(t.WorkflowRun.Types) == 0 { - warnings = append(warnings, `triggers.workflow_run requires non-empty workflows and types; skipped`) - } else { - wfEntry := map[string]any{ - "workflows": t.WorkflowRun.Workflows, - "types": t.WorkflowRun.Types, - } - if len(t.WorkflowRun.Conclusions) > 0 { - warnings = append(warnings, `triggers.workflow_run.conclusions has no gh-aw equivalent; review the generated "on:" block`) - } - parts["workflow_run"] = wfEntry - } - } + warnings = append(warnings, convertTriggersToOnInterval(t, parts)...) + warnings = append(warnings, convertTriggersToOnIssues(t, parts)...) + warnings = append(warnings, convertTriggersToOnWorkflowRun(t, parts)...) if len(parts) == 0 { return nil, warnings @@ -471,6 +445,57 @@ func convertTriggersToOn(t *JSONWorkflowTriggers) (any, []string) { return parts, warnings } +func convertTriggersToOnInterval(t *JSONWorkflowTriggers, parts map[string]any) []string { + if t.Interval == nil || len(t.Interval.Types) == 0 { + return nil + } + var warnings []string + intervalType := t.Interval.Types[0] + if len(t.Interval.Types) > 1 { + warnings = append(warnings, fmt.Sprintf( + "triggers.interval has multiple types %v; only the first (%q) will be used", + t.Interval.Types, intervalType)) + } + switch intervalType { + case "hourly", "daily", "weekly": + parts["_interval"] = intervalType + default: + warnings = append(warnings, fmt.Sprintf("triggers.interval type %q is not supported; skipped", intervalType)) + } + return warnings +} + +func convertTriggersToOnIssues(t *JSONWorkflowTriggers, parts map[string]any) []string { + if t.Issues == nil || len(t.Issues.Types) == 0 { + return nil + } + issueEntry := map[string]any{"types": t.Issues.Types} + if t.Issues.Query != "" { + parts["issues"] = issueEntry + return []string{`triggers.issues.query has no gh-aw equivalent; add a skip-if-match block manually if needed`} + } + parts["issues"] = issueEntry + return nil +} + +func convertTriggersToOnWorkflowRun(t *JSONWorkflowTriggers, parts map[string]any) []string { + if t.WorkflowRun == nil { + return nil + } + if len(t.WorkflowRun.Workflows) == 0 || len(t.WorkflowRun.Types) == 0 { + return []string{`triggers.workflow_run requires non-empty workflows and types; skipped`} + } + wfEntry := map[string]any{ + "workflows": t.WorkflowRun.Workflows, + "types": t.WorkflowRun.Types, + } + parts["workflow_run"] = wfEntry + if len(t.WorkflowRun.Conclusions) > 0 { + return []string{`triggers.workflow_run.conclusions has no gh-aw equivalent; review the generated "on:" block`} + } + return nil +} + // intervalToFuzzySchedule returns a gh-aw fuzzy schedule expression for a // JSON interval type. These expressions are understood by gh-aw's schedule // parser (pkg/parser/schedule_parser.go) and compiled to randomised cron diff --git a/pkg/cli/list_workflows_command.go b/pkg/cli/list_workflows_command.go index a33a6f214ec..37ea2d925f7 100644 --- a/pkg/cli/list_workflows_command.go +++ b/pkg/cli/list_workflows_command.go @@ -85,28 +85,7 @@ It accepts workflow IDs (basename without .md) or full filenames.`, func RunListWorkflows(ctx context.Context, repo, path, pattern string, verbose bool, jsonOutput bool, labelFilter string) error { listWorkflowsLog.Printf("Listing workflows: repo=%s, path=%s, pattern=%s, jsonOutput=%v, labelFilter=%s", repo, path, pattern, jsonOutput, labelFilter) - var mdFiles []string - var err error - var isRemote bool - - if repo != "" { - // List workflows from remote repository - isRemote = true - if verbose && !jsonOutput { - fmt.Fprintf(os.Stderr, "Listing workflow files from %s\n", repo) - } - mdFiles, err = getRemoteWorkflowFiles(ctx, repo, path, verbose, jsonOutput) - } else { - // List workflows from local repository - if verbose && !jsonOutput { - fmt.Fprintf(os.Stderr, "Listing workflow files\n") - if pattern != "" { - fmt.Fprintf(os.Stderr, "Filtering by pattern: %s\n", pattern) - } - } - mdFiles, err = getMarkdownWorkflowFiles(path) - } - + mdFiles, isRemote, err := runListWorkflowsFiles(ctx, repo, path, pattern, verbose, jsonOutput) if err != nil { listWorkflowsLog.Printf("Failed to get markdown workflow files: %v", err) fmt.Fprintln(os.Stderr, console.FormatErrorMessage(err.Error())) @@ -131,89 +110,108 @@ func RunListWorkflows(ctx context.Context, repo, path, pattern string, verbose b } // Build workflow list - var workflows []WorkflowListItem + workflows := runListWorkflowsItems(mdFiles, pattern, labelFilter, isRemote) - // Shared import cache across all iterations to avoid re-creating it for every workflow - importCache := parser.NewImportCache("") + // Output results + return runListWorkflowsOutput(workflows, jsonOutput) +} +func runListWorkflowsFiles(ctx context.Context, repo, path, pattern string, verbose bool, jsonOutput bool) ([]string, bool, error) { + if repo != "" { + if verbose && !jsonOutput { + fmt.Fprintf(os.Stderr, "Listing workflow files from %s\n", repo) + } + mdFiles, err := getRemoteWorkflowFiles(ctx, repo, path, verbose, jsonOutput) + return mdFiles, true, err + } + if verbose && !jsonOutput { + fmt.Fprintf(os.Stderr, "Listing workflow files\n") + if pattern != "" { + fmt.Fprintf(os.Stderr, "Filtering by pattern: %s\n", pattern) + } + } + mdFiles, err := getMarkdownWorkflowFiles(path) + return mdFiles, false, err +} + +func runListWorkflowsItems(mdFiles []string, pattern, labelFilter string, isRemote bool) []WorkflowListItem { + var workflows []WorkflowListItem + importCache := parser.NewImportCache("") for _, file := range mdFiles { name := extractWorkflowNameFromPath(file) - - // Skip if pattern specified and doesn't match if pattern != "" && !strings.Contains(strings.ToLower(name), strings.ToLower(pattern)) { continue } - - // For remote repos, we can't check compilation status or read local files if isRemote { - // For remote repos, skip fetching individual file metadata to avoid slowness - // Just show file name with minimal info - workflows = append(workflows, WorkflowListItem{ - Workflow: name, - EngineID: "N/A", // Skip fetching to avoid slow API/git calls - Compiled: "N/A", // Cannot determine for remote repos - Labels: nil, - On: nil, - }) - } else { - // Extract engine ID from workflow file - agent := extractEngineIDFromFile(file) + workflows = append(workflows, runListWorkflowsRemoteItem(name)) + continue + } + item, labels := runListWorkflowsLocalItem(file, name, importCache) + if labelFilter != "" && !runListWorkflowsHasLabel(labels, labelFilter) { + continue + } + workflows = append(workflows, item) + } + return workflows +} - // Check if compiled (.lock.yml file is in .github/workflows) - lockFile := stringutil.MarkdownToLockFile(file) - compiled := "N/A" +func runListWorkflowsRemoteItem(name string) WorkflowListItem { + return WorkflowListItem{ + Workflow: name, + EngineID: "N/A", // Skip fetching to avoid slow API/git calls + Compiled: "N/A", // Cannot determine for remote repos + Labels: nil, + On: nil, + } +} - if _, err := os.Stat(lockFile); err == nil { - compiled = isCompiledUpToDateWithCache(file, lockFile, importCache) - } +func runListWorkflowsLocalItem(file, name string, importCache *parser.ImportCache) (WorkflowListItem, []string) { + agent := extractEngineIDFromFile(file) + lockFile := stringutil.MarkdownToLockFile(file) + compiled := "N/A" + if _, err := os.Stat(lockFile); err == nil { + compiled = isCompiledUpToDateWithCache(file, lockFile, importCache) + } + onField, labels := runListWorkflowsFrontmatter(file) + return WorkflowListItem{ + Workflow: name, + EngineID: agent, + Compiled: compiled, + Labels: labels, + On: onField, + }, labels +} - // Extract "on" field and labels from frontmatter - var onField any - var labels []string - if content, err := os.ReadFile(file); err == nil { - if result, err := parser.ExtractFrontmatterFromContent(string(content)); err == nil { - if result.Frontmatter != nil { - onField = result.Frontmatter["on"] - // Extract labels field if present - if labelsField, ok := result.Frontmatter["labels"]; ok { - if labelsArray, ok := labelsField.([]any); ok { - for _, label := range labelsArray { - if labelStr, ok := label.(string); ok { - labels = append(labels, labelStr) - } - } - } +func runListWorkflowsFrontmatter(file string) (any, []string) { + var onField any + var labels []string + if content, err := os.ReadFile(file); err == nil { + if result, err := parser.ExtractFrontmatterFromContent(string(content)); err == nil && result.Frontmatter != nil { + onField = result.Frontmatter["on"] + if labelsField, ok := result.Frontmatter["labels"]; ok { + if labelsArray, ok := labelsField.([]any); ok { + for _, label := range labelsArray { + if labelStr, ok := label.(string); ok { + labels = append(labels, labelStr) } } } } + } + } + return onField, labels +} - // Skip if label filter specified and workflow doesn't have the label - if labelFilter != "" { - hasLabel := false - for _, label := range labels { - if strings.EqualFold(label, labelFilter) { - hasLabel = true - break - } - } - if !hasLabel { - continue - } - } - - // Build workflow list item - workflows = append(workflows, WorkflowListItem{ - Workflow: name, - EngineID: agent, - Compiled: compiled, - Labels: labels, - On: onField, - }) +func runListWorkflowsHasLabel(labels []string, labelFilter string) bool { + for _, label := range labels { + if strings.EqualFold(label, labelFilter) { + return true } } + return false +} - // Output results +func runListWorkflowsOutput(workflows []WorkflowListItem, jsonOutput bool) error { if jsonOutput { jsonBytes, err := json.MarshalIndent(workflows, "", " ") if err != nil { @@ -222,18 +220,12 @@ func RunListWorkflows(ctx context.Context, repo, path, pattern string, verbose b fmt.Fprintln(os.Stdout, string(jsonBytes)) return nil } - - // Print workflow count message for text output - workflowCount := len(workflows) - if workflowCount == 1 { + if len(workflows) == 1 { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Found 1 workflow")) } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Found %d workflows", workflowCount))) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Found %d workflows", len(workflows)))) } - - // Render the table using struct-based rendering fmt.Fprint(os.Stderr, console.RenderStruct(workflows)) - return nil } diff --git a/pkg/cli/log_aggregation.go b/pkg/cli/log_aggregation.go index b029cebf3d0..aeb130628a8 100644 --- a/pkg/cli/log_aggregation.go +++ b/pkg/cli/log_aggregation.go @@ -69,21 +69,36 @@ func aggregateLogFiles[T MutableLogAnalysis]( fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Analyzing %d log files from %s", len(files), logsDir))) } - // Initialize aggregated analysis - aggregated := newAnalysis() + aggregated, allAllowedDomains, allBlockedDomains := aggregateLogFilesParse(files, verbose, parser, newAnalysis) + + // Convert maps to sorted slices + allowedDomains := sliceutil.SortedKeys(allAllowedDomains) + + blockedDomains := sliceutil.SortedKeys(allBlockedDomains) + + // Set the sorted domain lists + aggregated.SetAllowedDomains(allowedDomains) + aggregated.SetBlockedDomains(blockedDomains) - // Track unique domains across all files - allAllowedDomains := make(map[string]struct { - }) - allBlockedDomains := make(map[string]struct { - }) + logAggregationLog.Printf("Aggregation complete: processed %d files, found %d allowed and %d blocked domains", + len(files), len(allowedDomains), len(blockedDomains)) + + return aggregated, nil +} - // Parse each file and aggregate results +func aggregateLogFilesParse[T MutableLogAnalysis]( + files []string, + verbose bool, + parser LogParser[T], + newAnalysis func() T, +) (T, map[string]struct{}, map[string]struct{}) { + aggregated := newAnalysis() + allAllowedDomains := make(map[string]struct{}) + allBlockedDomains := make(map[string]struct{}) for _, file := range files { if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Parsing "+filepath.Base(file))) } - analysis, err := parser(file, verbose) if err != nil { if verbose { @@ -91,32 +106,17 @@ func aggregateLogFiles[T MutableLogAnalysis]( } continue } - - // Aggregate metrics aggregated.AddMetrics(analysis) - - // Collect unique domains - for _, domain := range analysis.GetAllowedDomains() { - allAllowedDomains[domain] = struct { - }{} - } - for _, domain := range analysis.GetBlockedDomains() { - allBlockedDomains[domain] = struct { - }{} - } + aggregateLogFilesDomains(analysis, allAllowedDomains, allBlockedDomains) } + return aggregated, allAllowedDomains, allBlockedDomains +} - // Convert maps to sorted slices - allowedDomains := sliceutil.SortedKeys(allAllowedDomains) - - blockedDomains := sliceutil.SortedKeys(allBlockedDomains) - - // Set the sorted domain lists - aggregated.SetAllowedDomains(allowedDomains) - aggregated.SetBlockedDomains(blockedDomains) - - logAggregationLog.Printf("Aggregation complete: processed %d files, found %d allowed and %d blocked domains", - len(files), len(allowedDomains), len(blockedDomains)) - - return aggregated, nil +func aggregateLogFilesDomains[T MutableLogAnalysis](analysis T, allAllowedDomains, allBlockedDomains map[string]struct{}) { + for _, domain := range analysis.GetAllowedDomains() { + allAllowedDomains[domain] = struct{}{} + } + for _, domain := range analysis.GetBlockedDomains() { + allBlockedDomains[domain] = struct{}{} + } } diff --git a/pkg/cli/logs_cache.go b/pkg/cli/logs_cache.go index 65eb46c7e79..9aa2b76fa4e 100644 --- a/pkg/cli/logs_cache.go +++ b/pkg/cli/logs_cache.go @@ -120,40 +120,13 @@ func cleanupOldRunFolders(outputDir string, cutoff time.Time, verbose bool) (int } runDir := filepath.Join(outputDir, entry.Name()) - - // Determine the run date: prefer the GitHub run creation timestamp from - // run_summary.json so the cutoff is relative to when the workflow actually - // ran, not when we downloaded or processed it. - var runDate time.Time - summaryPath := filepath.Join(runDir, runSummaryFileName) - if data, readErr := os.ReadFile(summaryPath); readErr == nil { - var summary RunSummary - if jsonErr := json.Unmarshal(data, &summary); jsonErr == nil && !summary.Run.CreatedAt.IsZero() { - runDate = summary.Run.CreatedAt - } - } - - // Fall back to directory modification time when the summary is unavailable. - if runDate.IsZero() { - info, statErr := entry.Info() - if statErr != nil { - logsCacheLog.Printf("Failed to stat run directory %s: %v", entry.Name(), statErr) - continue - } - runDate = info.ModTime() + runDate, ok := cleanupOldRunFoldersDate(entry, runDir) + if !ok { + continue } if runDate.Before(cutoff) { - logsCacheLog.Printf("Removing old run folder: %s (run date: %s, cutoff: %s)", entry.Name(), runDate.Format(time.RFC3339), cutoff.Format(time.RFC3339)) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Removing old run folder: %s (run date: %s)", entry.Name(), runDate.Format("2006-01-02")))) - } - if removeErr := os.RemoveAll(runDir); removeErr != nil { - logsCacheLog.Printf("Failed to remove run folder %s: %v", entry.Name(), removeErr) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove old run folder %s: %v", entry.Name(), removeErr))) - continue - } - removed++ + removed += cleanupOldRunFoldersRemove(entry.Name(), runDir, runDate, cutoff, verbose) } } @@ -161,6 +134,39 @@ func cleanupOldRunFolders(outputDir string, cutoff time.Time, verbose bool) (int return removed, nil } +func cleanupOldRunFoldersDate(entry os.DirEntry, runDir string) (time.Time, bool) { + var runDate time.Time + summaryPath := filepath.Join(runDir, runSummaryFileName) + if data, readErr := os.ReadFile(summaryPath); readErr == nil { + var summary RunSummary + if jsonErr := json.Unmarshal(data, &summary); jsonErr == nil && !summary.Run.CreatedAt.IsZero() { + runDate = summary.Run.CreatedAt + } + } + if !runDate.IsZero() { + return runDate, true + } + info, statErr := entry.Info() + if statErr != nil { + logsCacheLog.Printf("Failed to stat run directory %s: %v", entry.Name(), statErr) + return time.Time{}, false + } + return info.ModTime(), true +} + +func cleanupOldRunFoldersRemove(name, runDir string, runDate, cutoff time.Time, verbose bool) int { + logsCacheLog.Printf("Removing old run folder: %s (run date: %s, cutoff: %s)", name, runDate.Format(time.RFC3339), cutoff.Format(time.RFC3339)) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Removing old run folder: %s (run date: %s)", name, runDate.Format("2006-01-02")))) + } + if removeErr := os.RemoveAll(runDir); removeErr != nil { + logsCacheLog.Printf("Failed to remove run folder %s: %v", name, removeErr) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove old run folder %s: %v", name, removeErr))) + return 0 + } + return 1 +} + // saveRunSummary saves a run summary to disk func saveRunSummary(outputDir string, summary *RunSummary, verbose bool) error { logsCacheLog.Printf("Saving run summary to cache: dir=%s, run_id=%d", outputDir, summary.RunID) diff --git a/pkg/cli/logs_command.go b/pkg/cli/logs_command.go index 6b6143d228c..21fd511c550 100644 --- a/pkg/cli/logs_command.go +++ b/pkg/cli/logs_command.go @@ -31,9 +31,28 @@ func NewLogsCommand() *cobra.Command { validArtifactSets := strings.Join(ValidArtifactSetNames(), ", ") logsCmd := &cobra.Command{ - Use: "logs [workflow]", - Short: "Download and analyze agentic workflow logs and artifacts", - Long: fmt.Sprintf(`Download and analyze agentic workflow logs and artifacts from GitHub Actions. + Use: "logs [workflow]", + Short: "Download and analyze agentic workflow logs and artifacts", + Long: newLogsCommandLong(validArtifactSets), + Example: newLogsCommandExample(), + RunE: func(cmd *cobra.Command, args []string) error { + return newLogsCommandRun(cmd, args) + }, + } + + // Add flags to logs command + newLogsCommandAddFlags(logsCmd, validArtifactSets) + + // Register completions for logs command + logsCmd.ValidArgsFunction = CompleteWorkflowNames + RegisterEngineFlagCompletion(logsCmd) + RegisterDirFlagCompletion(logsCmd, "output") + + return logsCmd +} + +func newLogsCommandLong(validArtifactSets string) string { + return fmt.Sprintf(`Download and analyze agentic workflow logs and artifacts from GitHub Actions. This command fetches workflow runs, downloads their artifacts, and extracts them into organized folders named by run ID. It also provides an overview table with aggregate @@ -54,8 +73,17 @@ Downloaded artifacts include (when using --artifacts all): - aw-{branch}.patch: Git patch of changes for each branch (one file per PR/push) - workflow-logs/: GitHub Actions workflow run logs (job logs organized in subdirectory) - summary.json: Complete metrics and run data for all downloaded runs -`, validArtifactSets) + "\n\n" + WorkflowIDExplanation, - Example: ` # Basic usage +`, validArtifactSets) + "\n\n" + WorkflowIDExplanation +} + +func newLogsCommandExample() string { + return newLogsCommandExampleBasicAndDates() + + newLogsCommandExampleFiltersAndArtifacts() + + newLogsCommandExampleOutputAndCache() +} + +func newLogsCommandExampleBasicAndDates() string { + return ` # Basic usage ` + string(constants.CLIExtensionPrefix) + ` logs # Download logs for all workflows ` + string(constants.CLIExtensionPrefix) + ` logs weekly-research # Download logs for specific workflow ` + string(constants.CLIExtensionPrefix) + ` logs weekly-research.md # Download logs (alternative format) @@ -68,7 +96,11 @@ Downloaded artifacts include (when using --artifacts all): ` + string(constants.CLIExtensionPrefix) + ` logs --start-date -1w -c 5 # Download up to 5 runs from last week ` + string(constants.CLIExtensionPrefix) + ` logs --end-date -1d # Download up to 10 runs before yesterday ` + string(constants.CLIExtensionPrefix) + ` logs --start-date -1mo # Download up to 10 runs from last month +` +} +func newLogsCommandExampleFiltersAndArtifacts() string { + return ` # Content filtering ` + string(constants.CLIExtensionPrefix) + ` logs --engine claude # Filter logs by claude engine ` + string(constants.CLIExtensionPrefix) + ` logs --engine codex # Filter logs by codex engine @@ -96,7 +128,11 @@ Downloaded artifacts include (when using --artifacts all): ` + string(constants.CLIExtensionPrefix) + ` logs --artifacts agent # Download only agent logs ` + string(constants.CLIExtensionPrefix) + ` logs --artifacts agent,firewall # Download agent and firewall artifacts ` + string(constants.CLIExtensionPrefix) + ` logs --artifacts mcp # Download only MCP gateway logs +` +} +func newLogsCommandExampleOutputAndCache() string { + return ` # Output options (default output is compact format optimized for agents) ` + string(constants.CLIExtensionPrefix) + ` logs -o ./my-logs # Custom output directory ` + string(constants.CLIExtensionPrefix) + ` logs --tool-graph # Generate Mermaid tool sequence graph @@ -119,253 +155,10 @@ Downloaded artifacts include (when using --artifacts all): ` + string(constants.CLIExtensionPrefix) + ` logs --cache-before -1w # Evict local cache older than 1 week before downloading runs ` + string(constants.CLIExtensionPrefix) + ` logs --cache-before -30d # Evict local cache older than 30 days before downloading runs ` + string(constants.CLIExtensionPrefix) + ` logs --cache-before -1mo # Evict local cache older than 1 month before downloading runs - ` + string(constants.CLIExtensionPrefix) + ` logs --cache-before 2024-01-01 # Evict local cache older than 2024-01-01 before downloading runs`, - RunE: func(cmd *cobra.Command, args []string) error { - logsCommandLog.Printf("Starting logs command: args=%d", len(args)) - - stdin, _ := cmd.Flags().GetBool("stdin") - - // When --stdin is provided, read run IDs/URLs from stdin and bypass GitHub API discovery. - if stdin { - if len(args) > 0 { - return errors.New(console.FormatErrorWithSuggestions( - "positional arguments are not allowed with --stdin", - []string{"Remove the workflow name argument, or omit --stdin to use the normal discovery mode"}, - )) - } - logsCommandLog.Printf("Reading run IDs from stdin") - runURLs, err := readRunIDsFromStdin(os.Stdin) - if err != nil { - return fmt.Errorf("failed to read run IDs from stdin: %w", err) - } - - outputDir, _ := cmd.Flags().GetString("output") - engine, _ := cmd.Flags().GetString("engine") - repoOverride, _ := cmd.Flags().GetString("repo") - verbose, _ := cmd.Flags().GetBool("verbose") - toolGraph, _ := cmd.Flags().GetBool("tool-graph") - noStaged, _ := cmd.Flags().GetBool("exclude-staged") - firewallOnly, _ := cmd.Flags().GetBool("firewall") - noFirewall, _ := cmd.Flags().GetBool("no-firewall") - parse, _ := cmd.Flags().GetBool("parse") - jsonOutput, _ := cmd.Flags().GetBool("json") - timeout, _ := cmd.Flags().GetInt("timeout") - summaryFile, _ := cmd.Flags().GetString("summary-file") - safeOutputType, _ := cmd.Flags().GetString("safe-output") - filteredIntegrity, _ := cmd.Flags().GetBool("filtered-integrity") - evalsOnly, _ := cmd.Flags().GetBool("evals") - train, _ := cmd.Flags().GetBool("train") - format, _ := cmd.Flags().GetString("format") - reportFile, _ := cmd.Flags().GetString("report-file") - artifacts, _ := cmd.Flags().GetStringSlice("artifacts") - - if engine != "" { - logsCommandLog.Printf("Validating engine parameter: %s", engine) - registry := workflow.GetGlobalEngineRegistry() - if !registry.IsValidEngine(engine) { - supportedEngines := registry.GetSupportedEngines() - return fmt.Errorf("invalid engine value '%s'. Must be one of: %s", engine, strings.Join(supportedEngines, ", ")) - } - } - - if err := validateReportFileFlags(reportFile, format, jsonOutput); err != nil { - return err - } - - if len(artifacts) > 0 { - artifacts = applyEvalsArtifact(artifacts, evalsOnly) - } - - return DownloadWorkflowLogsFromStdin(cmd.Context(), StdinLogsOptions{ - RunURLs: runURLs, - OutputDir: outputDir, - Engine: engine, - RepoOverride: repoOverride, - Verbose: verbose, - ToolGraph: toolGraph, - NoStaged: noStaged, - FirewallOnly: firewallOnly, - NoFirewall: noFirewall, - Parse: parse, - JSONOutput: jsonOutput, - Timeout: timeout, - SummaryFile: summaryFile, - SafeOutputType: safeOutputType, - FilteredIntegrity: filteredIntegrity, - EvalsOnly: evalsOnly, - Train: train, - Format: format, - ReportFile: reportFile, - ArtifactSets: artifacts, - }) - } - - var workflowName string - if len(args) > 0 && args[0] != "" { - logsCommandLog.Printf("Resolving workflow name from argument: %s", args[0]) - - repoOverrideEarly, _ := cmd.Flags().GetString("repo") - if repoOverrideEarly != "" { - // When --repo is specified, only use local lock-file resolution when - // the target repo is the current repository. Local lock files are - // authoritative for the current repo and allow us to map the workflow - // ID (e.g. "audit-workflows") to its GitHub Actions display name - // (e.g. "Agentic Workflow Audit Agent"), which gh run list requires. - // - // For cross-repo queries, skip local resolution to avoid mapping a - // local display name onto a different repository's workflow topology. - // - // Note: the argument must be a workflow ID (e.g. "test-claude"), - // not a display name (e.g. "Test Claude"). Display-name lookup - // requires local lock files, which are unavailable for remote repos. - if repoIsLocal(repoOverrideEarly) { - if resolved, resolveErr := workflow.FindWorkflowName(args[0]); resolveErr == nil { - workflowName = resolved - logsCommandLog.Printf("Resolved workflow name via local lock files: %s -> %s", args[0], workflowName) - } else { - workflowName = normalizeWorkflowID(args[0]) - logsCommandLog.Printf("Local resolution failed, using normalized workflow name: %s", workflowName) - } - } else { - workflowName = normalizeWorkflowID(args[0]) - logsCommandLog.Printf("Using normalized workflow name for remote repo: %s", workflowName) - } - } else { - // Use flexible workflow name matching (workflow ID or display name) - resolvedName, err := workflow.FindWorkflowName(args[0]) - if err != nil { - // Workflow not found - provide suggestions - suggestions := []string{ - fmt.Sprintf("Run '%s status' to see all available workflows", string(constants.CLIExtensionPrefix)), - "Check for typos in the workflow name", - "Use the workflow ID (e.g., 'test-claude') or GitHub Actions workflow name (e.g., 'Test Claude')", - } - - // Add fuzzy match suggestions - similarNames := suggestWorkflowNames(args[0]) - if len(similarNames) > 0 { - suggestions = append([]string{fmt.Sprintf("Did you mean: %s?", strings.Join(similarNames, ", "))}, suggestions...) - } - - return errors.New(console.FormatErrorWithSuggestions( - fmt.Sprintf("workflow '%s' not found", args[0]), - suggestions, - )) - } - workflowName = resolvedName - } - } - - count, _ := cmd.Flags().GetInt("count") - // --last is an alias for --count (for compatibility with users of `audit report --last`) - if last, _ := cmd.Flags().GetInt("last"); last > 0 { - count = last - } - startDate, _ := cmd.Flags().GetString("start-date") - endDate, _ := cmd.Flags().GetString("end-date") - outputDir, _ := cmd.Flags().GetString("output") - engine, _ := cmd.Flags().GetString("engine") - ref, _ := cmd.Flags().GetString("ref") - beforeRunID, _ := cmd.Flags().GetInt64("before-run-id") - afterRunID, _ := cmd.Flags().GetInt64("after-run-id") - verbose, _ := cmd.Flags().GetBool("verbose") - toolGraph, _ := cmd.Flags().GetBool("tool-graph") - noStaged, _ := cmd.Flags().GetBool("exclude-staged") - firewallOnly, _ := cmd.Flags().GetBool("firewall") - noFirewall, _ := cmd.Flags().GetBool("no-firewall") - parse, _ := cmd.Flags().GetBool("parse") - jsonOutput, _ := cmd.Flags().GetBool("json") - timeout, _ := cmd.Flags().GetInt("timeout") - repoOverride, _ := cmd.Flags().GetString("repo") - summaryFile, _ := cmd.Flags().GetString("summary-file") - safeOutputType, _ := cmd.Flags().GetString("safe-output") - filteredIntegrity, _ := cmd.Flags().GetBool("filtered-integrity") - evalsOnly, _ := cmd.Flags().GetBool("evals") - train, _ := cmd.Flags().GetBool("train") - format, _ := cmd.Flags().GetString("format") - reportFile, _ := cmd.Flags().GetString("report-file") - artifacts, _ := cmd.Flags().GetStringSlice("artifacts") - cacheBefore, _ := cmd.Flags().GetString("cache-before") - if !cmd.Flags().Changed("cache-before") { - if cmd.Flags().Changed("after") { - cacheBefore, _ = cmd.Flags().GetString("after") - } - } - - // Resolve relative dates to absolute dates for GitHub CLI - now := time.Now() - if startDate != "" { - logsCommandLog.Printf("Resolving start date: %s", startDate) - resolvedStartDate, err := workflow.ResolveRelativeDate(startDate, now) - if err != nil { - return fmt.Errorf("invalid start-date format '%s': %w", startDate, err) - } - startDate = resolvedStartDate - logsCommandLog.Printf("Resolved start date to: %s", startDate) - } - if endDate != "" { - logsCommandLog.Printf("Resolving end date: %s", endDate) - resolvedEndDate, err := workflow.ResolveRelativeDate(endDate, now) - if err != nil { - return fmt.Errorf("invalid end-date format '%s': %w", endDate, err) - } - endDate = resolvedEndDate - logsCommandLog.Printf("Resolved end date to: %s", endDate) - } - - // Validate engine parameter using the engine registry - if engine != "" { - logsCommandLog.Printf("Validating engine parameter: %s", engine) - registry := workflow.GetGlobalEngineRegistry() - if !registry.IsValidEngine(engine) { - supportedEngines := registry.GetSupportedEngines() - return fmt.Errorf("invalid engine value '%s'. Must be one of: %s", engine, strings.Join(supportedEngines, ", ")) - } - } - - if err := validateReportFileFlags(reportFile, format, jsonOutput); err != nil { - return err - } - - logsCommandLog.Printf("Executing logs download: workflow=%s, count=%d, engine=%s, train=%v, cache_before=%s", workflowName, count, engine, train, cacheBefore) - - if len(artifacts) > 0 { - artifacts = applyEvalsArtifact(artifacts, evalsOnly) - } - - return DownloadWorkflowLogs(cmd.Context(), LogsDownloadOptions{ - WorkflowName: workflowName, - Count: count, - StartDate: startDate, - EndDate: endDate, - OutputDir: outputDir, - Engine: engine, - Ref: ref, - BeforeRunID: beforeRunID, - AfterRunID: afterRunID, - RepoOverride: repoOverride, - Verbose: verbose, - ToolGraph: toolGraph, - NoStaged: noStaged, - FirewallOnly: firewallOnly, - NoFirewall: noFirewall, - Parse: parse, - JSONOutput: jsonOutput, - TimeoutMinutes: timeout, - SummaryFile: summaryFile, - SafeOutputType: safeOutputType, - FilteredIntegrity: filteredIntegrity, - EvalsOnly: evalsOnly, - Train: train, - Format: format, - ReportFile: reportFile, - ArtifactSets: artifacts, - After: cacheBefore, - }) - }, - } + ` + string(constants.CLIExtensionPrefix) + ` logs --cache-before 2024-01-01 # Evict local cache older than 2024-01-01 before downloading runs` +} - // Add flags to logs command +func newLogsCommandAddFlags(logsCmd *cobra.Command, validArtifactSets string) { logsCmd.Flags().IntP("count", "c", 10, "Maximum number of matching workflow runs to return (after applying filters)") logsCmd.Flags().String("start-date", "", "Filter runs created after this date (YYYY-MM-DD or delta like -1d, -1w, -1mo)") logsCmd.Flags().String("end-date", "", "Filter runs created before this date (YYYY-MM-DD or delta like -1d, -1w, -1mo)") @@ -397,13 +190,264 @@ Downloaded artifacts include (when using --artifacts all): _ = logsCmd.Flags().MarkDeprecated("after", "use --cache-before") logsCmd.Flags().Bool("stdin", false, "Read workflow run IDs or URLs from stdin (one per line) instead of discovering runs via the GitHub API") logsCmd.MarkFlagsMutuallyExclusive("firewall", "no-firewall") +} - // Register completions for logs command - logsCmd.ValidArgsFunction = CompleteWorkflowNames - RegisterEngineFlagCompletion(logsCmd) - RegisterDirFlagCompletion(logsCmd, "output") +type newLogsCommandFlags struct { + count int + startDate string + endDate string + outputDir string + engine string + ref string + beforeRunID int64 + afterRunID int64 + repoOverride string + verbose bool + toolGraph bool + noStaged bool + firewallOnly bool + noFirewall bool + parse bool + jsonOutput bool + timeout int + summaryFile string + safeOutputType string + filteredIntegrity bool + evalsOnly bool + train bool + format string + reportFile string + artifacts []string + cacheBefore string +} - return logsCmd +func newLogsCommandRun(cmd *cobra.Command, args []string) error { + logsCommandLog.Printf("Starting logs command: args=%d", len(args)) + stdin, _ := cmd.Flags().GetBool("stdin") + if stdin { + return newLogsCommandRunStdin(cmd, args) + } + workflowName, err := newLogsCommandResolveWorkflowName(cmd, args) + if err != nil { + return err + } + flags, err := newLogsCommandReadFlags(cmd) + if err != nil { + return err + } + logsCommandLog.Printf("Executing logs download: workflow=%s, count=%d, engine=%s, train=%v, cache_before=%s", workflowName, flags.count, flags.engine, flags.train, flags.cacheBefore) + return DownloadWorkflowLogs(cmd.Context(), newLogsCommandDownloadOptions(workflowName, flags)) +} + +func newLogsCommandRunStdin(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + return errors.New(console.FormatErrorWithSuggestions( + "positional arguments are not allowed with --stdin", + []string{"Remove the workflow name argument, or omit --stdin to use the normal discovery mode"}, + )) + } + logsCommandLog.Printf("Reading run IDs from stdin") + runURLs, err := readRunIDsFromStdin(os.Stdin) + if err != nil { + return fmt.Errorf("failed to read run IDs from stdin: %w", err) + } + flags, err := newLogsCommandReadCommonFlags(cmd) + if err != nil { + return err + } + return DownloadWorkflowLogsFromStdin(cmd.Context(), newLogsCommandStdinOptions(runURLs, flags)) +} + +func newLogsCommandReadFlags(cmd *cobra.Command) (newLogsCommandFlags, error) { + flags, err := newLogsCommandReadCommonFlags(cmd) + if err != nil { + return flags, err + } + flags.count, _ = cmd.Flags().GetInt("count") + if last, _ := cmd.Flags().GetInt("last"); last > 0 { + flags.count = last + } + flags.startDate, _ = cmd.Flags().GetString("start-date") + flags.endDate, _ = cmd.Flags().GetString("end-date") + flags.ref, _ = cmd.Flags().GetString("ref") + flags.beforeRunID, _ = cmd.Flags().GetInt64("before-run-id") + flags.afterRunID, _ = cmd.Flags().GetInt64("after-run-id") + flags.cacheBefore, _ = cmd.Flags().GetString("cache-before") + if !cmd.Flags().Changed("cache-before") && cmd.Flags().Changed("after") { + flags.cacheBefore, _ = cmd.Flags().GetString("after") + } + if err := newLogsCommandResolveDates(&flags); err != nil { + return flags, err + } + return flags, nil +} + +func newLogsCommandReadCommonFlags(cmd *cobra.Command) (newLogsCommandFlags, error) { + var flags newLogsCommandFlags + flags.outputDir, _ = cmd.Flags().GetString("output") + flags.engine, _ = cmd.Flags().GetString("engine") + flags.repoOverride, _ = cmd.Flags().GetString("repo") + flags.verbose, _ = cmd.Flags().GetBool("verbose") + flags.toolGraph, _ = cmd.Flags().GetBool("tool-graph") + flags.noStaged, _ = cmd.Flags().GetBool("exclude-staged") + flags.firewallOnly, _ = cmd.Flags().GetBool("firewall") + flags.noFirewall, _ = cmd.Flags().GetBool("no-firewall") + flags.parse, _ = cmd.Flags().GetBool("parse") + flags.jsonOutput, _ = cmd.Flags().GetBool("json") + flags.timeout, _ = cmd.Flags().GetInt("timeout") + flags.summaryFile, _ = cmd.Flags().GetString("summary-file") + flags.safeOutputType, _ = cmd.Flags().GetString("safe-output") + flags.filteredIntegrity, _ = cmd.Flags().GetBool("filtered-integrity") + flags.evalsOnly, _ = cmd.Flags().GetBool("evals") + flags.train, _ = cmd.Flags().GetBool("train") + flags.format, _ = cmd.Flags().GetString("format") + flags.reportFile, _ = cmd.Flags().GetString("report-file") + flags.artifacts, _ = cmd.Flags().GetStringSlice("artifacts") + if err := newLogsCommandValidateCommonFlags(&flags); err != nil { + return flags, err + } + return flags, nil +} + +func newLogsCommandValidateCommonFlags(flags *newLogsCommandFlags) error { + if flags.engine != "" { + logsCommandLog.Printf("Validating engine parameter: %s", flags.engine) + registry := workflow.GetGlobalEngineRegistry() + if !registry.IsValidEngine(flags.engine) { + supportedEngines := registry.GetSupportedEngines() + return fmt.Errorf("invalid engine value '%s'. Must be one of: %s", flags.engine, strings.Join(supportedEngines, ", ")) + } + } + if err := validateReportFileFlags(flags.reportFile, flags.format, flags.jsonOutput); err != nil { + return err + } + if len(flags.artifacts) > 0 { + flags.artifacts = applyEvalsArtifact(flags.artifacts, flags.evalsOnly) + } + return nil +} + +func newLogsCommandResolveDates(flags *newLogsCommandFlags) error { + now := time.Now() + if flags.startDate != "" { + logsCommandLog.Printf("Resolving start date: %s", flags.startDate) + resolvedStartDate, err := workflow.ResolveRelativeDate(flags.startDate, now) + if err != nil { + return fmt.Errorf("invalid start-date format '%s': %w", flags.startDate, err) + } + flags.startDate = resolvedStartDate + logsCommandLog.Printf("Resolved start date to: %s", flags.startDate) + } + if flags.endDate != "" { + logsCommandLog.Printf("Resolving end date: %s", flags.endDate) + resolvedEndDate, err := workflow.ResolveRelativeDate(flags.endDate, now) + if err != nil { + return fmt.Errorf("invalid end-date format '%s': %w", flags.endDate, err) + } + flags.endDate = resolvedEndDate + logsCommandLog.Printf("Resolved end date to: %s", flags.endDate) + } + return nil +} + +func newLogsCommandResolveWorkflowName(cmd *cobra.Command, args []string) (string, error) { + if len(args) == 0 || args[0] == "" { + return "", nil + } + logsCommandLog.Printf("Resolving workflow name from argument: %s", args[0]) + repoOverrideEarly, _ := cmd.Flags().GetString("repo") + if repoOverrideEarly != "" { + return newLogsCommandResolveRemoteWorkflowName(args[0], repoOverrideEarly), nil + } + resolvedName, err := workflow.FindWorkflowName(args[0]) + if err != nil { + return "", newLogsCommandWorkflowNotFound(args[0]) + } + return resolvedName, nil +} + +func newLogsCommandResolveRemoteWorkflowName(arg, repoOverride string) string { + if repoIsLocal(repoOverride) { + if resolved, resolveErr := workflow.FindWorkflowName(arg); resolveErr == nil { + logsCommandLog.Printf("Resolved workflow name via local lock files: %s -> %s", arg, resolved) + return resolved + } + normalized := normalizeWorkflowID(arg) + logsCommandLog.Printf("Local resolution failed, using normalized workflow name: %s", normalized) + return normalized + } + normalized := normalizeWorkflowID(arg) + logsCommandLog.Printf("Using normalized workflow name for remote repo: %s", normalized) + return normalized +} + +func newLogsCommandWorkflowNotFound(arg string) error { + suggestions := []string{ + fmt.Sprintf("Run '%s status' to see all available workflows", string(constants.CLIExtensionPrefix)), + "Check for typos in the workflow name", + "Use the workflow ID (e.g., 'test-claude') or GitHub Actions workflow name (e.g., 'Test Claude')", + } + if similarNames := suggestWorkflowNames(arg); len(similarNames) > 0 { + suggestions = append([]string{fmt.Sprintf("Did you mean: %s?", strings.Join(similarNames, ", "))}, suggestions...) + } + return errors.New(console.FormatErrorWithSuggestions(fmt.Sprintf("workflow '%s' not found", arg), suggestions)) +} + +func newLogsCommandStdinOptions(runURLs []string, flags newLogsCommandFlags) StdinLogsOptions { + return StdinLogsOptions{ + RunURLs: runURLs, + OutputDir: flags.outputDir, + Engine: flags.engine, + RepoOverride: flags.repoOverride, + Verbose: flags.verbose, + ToolGraph: flags.toolGraph, + NoStaged: flags.noStaged, + FirewallOnly: flags.firewallOnly, + NoFirewall: flags.noFirewall, + Parse: flags.parse, + JSONOutput: flags.jsonOutput, + Timeout: flags.timeout, + SummaryFile: flags.summaryFile, + SafeOutputType: flags.safeOutputType, + FilteredIntegrity: flags.filteredIntegrity, + EvalsOnly: flags.evalsOnly, + Train: flags.train, + Format: flags.format, + ReportFile: flags.reportFile, + ArtifactSets: flags.artifacts, + } +} + +func newLogsCommandDownloadOptions(workflowName string, flags newLogsCommandFlags) LogsDownloadOptions { + return LogsDownloadOptions{ + WorkflowName: workflowName, + Count: flags.count, + StartDate: flags.startDate, + EndDate: flags.endDate, + OutputDir: flags.outputDir, + Engine: flags.engine, + Ref: flags.ref, + BeforeRunID: flags.beforeRunID, + AfterRunID: flags.afterRunID, + RepoOverride: flags.repoOverride, + Verbose: flags.verbose, + ToolGraph: flags.toolGraph, + NoStaged: flags.noStaged, + FirewallOnly: flags.firewallOnly, + NoFirewall: flags.noFirewall, + Parse: flags.parse, + JSONOutput: flags.jsonOutput, + TimeoutMinutes: flags.timeout, + SummaryFile: flags.summaryFile, + SafeOutputType: flags.safeOutputType, + FilteredIntegrity: flags.filteredIntegrity, + EvalsOnly: flags.evalsOnly, + Train: flags.train, + Format: flags.format, + ReportFile: flags.reportFile, + ArtifactSets: flags.artifacts, + After: flags.cacheBefore, + } } // flattenSingleFileArtifacts applies the artifact unfold rule to downloaded artifacts diff --git a/pkg/cli/logs_download.go b/pkg/cli/logs_download.go index 85c2c7dac06..ed8f314e506 100644 --- a/pkg/cli/logs_download.go +++ b/pkg/cli/logs_download.go @@ -69,70 +69,67 @@ func flattenSingleFileArtifacts(outputDir string, verbose bool) error { continue } - artifactDir := filepath.Join(outputDir, entry.Name()) - - // Read contents of artifact directory - artifactEntries, err := os.ReadDir(artifactDir) - if err != nil { - logsDownloadLog.Printf("Failed to read artifact directory %s: %v", artifactDir, err) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read artifact directory %s: %v", artifactDir, err))) - } - continue - } - - logsDownloadLog.Printf("Artifact directory %s contains %d entries", entry.Name(), len(artifactEntries)) + flattenSingleFileArtifactsEntry(outputDir, entry.Name(), verbose) + } - // Apply unfold rule: Check if directory contains exactly one entry and it's a file - if len(artifactEntries) != 1 { - if verbose && len(artifactEntries) > 1 { - // Log what's in multi-file artifacts for debugging - var fileNames []string - for _, e := range artifactEntries { - fileNames = append(fileNames, e.Name()) - } - logsDownloadLog.Printf("Artifact directory %s has %d files, not flattening: %v", entry.Name(), len(artifactEntries), fileNames) - } - continue - } + return nil +} - singleEntry := artifactEntries[0] - if singleEntry.IsDir() { - logsDownloadLog.Printf("Artifact directory %s contains a subdirectory, not flattening", entry.Name()) - continue +func flattenSingleFileArtifactsEntry(outputDir, entryName string, verbose bool) { + artifactDir := filepath.Join(outputDir, entryName) + artifactEntries, err := os.ReadDir(artifactDir) + if err != nil { + logsDownloadLog.Printf("Failed to read artifact directory %s: %v", artifactDir, err) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read artifact directory %s: %v", artifactDir, err))) } + return + } + logsDownloadLog.Printf("Artifact directory %s contains %d entries", entryName, len(artifactEntries)) + if len(artifactEntries) != 1 { + flattenSingleFileArtifactsLogMulti(entryName, artifactEntries, verbose) + return + } + singleEntry := artifactEntries[0] + if singleEntry.IsDir() { + logsDownloadLog.Printf("Artifact directory %s contains a subdirectory, not flattening", entryName) + return + } + flattenSingleFileArtifactsMove(outputDir, artifactDir, entryName, singleEntry.Name(), verbose) +} - // Unfold: Move the single file to parent directory and remove the artifact folder - sourcePath := filepath.Join(artifactDir, singleEntry.Name()) - destPath := filepath.Join(outputDir, singleEntry.Name()) - - logsDownloadLog.Printf("Flattening: %s → %s", sourcePath, destPath) - - // Move the file to root (parent directory) - if err := os.Rename(sourcePath, destPath); err != nil { - logsDownloadLog.Printf("Failed to move file %s to %s: %v", sourcePath, destPath, err) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to move file %s to %s: %v", sourcePath, destPath, err))) - } - continue +func flattenSingleFileArtifactsLogMulti(entryName string, artifactEntries []os.DirEntry, verbose bool) { + if verbose && len(artifactEntries) > 1 { + var fileNames []string + for _, e := range artifactEntries { + fileNames = append(fileNames, e.Name()) } + logsDownloadLog.Printf("Artifact directory %s has %d files, not flattening: %v", entryName, len(artifactEntries), fileNames) + } +} - // Delete the now-empty artifact folder - if err := os.Remove(artifactDir); err != nil { - logsDownloadLog.Printf("Failed to remove empty directory %s: %v", artifactDir, err) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove empty directory %s: %v", artifactDir, err))) - } - continue +func flattenSingleFileArtifactsMove(outputDir, artifactDir, entryName, singleName string, verbose bool) { + sourcePath := filepath.Join(artifactDir, singleName) + destPath := filepath.Join(outputDir, singleName) + logsDownloadLog.Printf("Flattening: %s → %s", sourcePath, destPath) + if err := os.Rename(sourcePath, destPath); err != nil { + logsDownloadLog.Printf("Failed to move file %s to %s: %v", sourcePath, destPath, err) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to move file %s to %s: %v", sourcePath, destPath, err))) } - - logsDownloadLog.Printf("Successfully flattened: %s/%s → %s", entry.Name(), singleEntry.Name(), singleEntry.Name()) + return + } + if err := os.Remove(artifactDir); err != nil { + logsDownloadLog.Printf("Failed to remove empty directory %s: %v", artifactDir, err) if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Unfolded single-file artifact: %s → %s", filepath.Join(entry.Name(), singleEntry.Name()), singleEntry.Name()))) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove empty directory %s: %v", artifactDir, err))) } + return + } + logsDownloadLog.Printf("Successfully flattened: %s/%s → %s", entryName, singleName, singleName) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Unfolded single-file artifact: %s → %s", filepath.Join(entryName, singleName), singleName))) } - - return nil } // findArtifactDir looks for an artifact directory by its base name (suffix) in outputDir. @@ -223,6 +220,12 @@ func flattenArtifactTree(sourceDir, artifactDir, outputDir, label string, verbos return fmt.Errorf("failed to flatten %s: %w", label, err) } + flattenArtifactTreeCleanup(artifactDir, label, verbose) + + return nil +} + +func flattenArtifactTreeCleanup(artifactDir, label string, verbose bool) { // Remove the now-empty artifact directory structure. // Don't fail the entire operation if cleanup fails. if err := os.RemoveAll(artifactDir); err != nil { @@ -230,14 +233,12 @@ func flattenArtifactTree(sourceDir, artifactDir, outputDir, label string, verbos if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove %s directory: %v", label, err))) } - } else { - logsDownloadLog.Printf("Removed %s directory: %s", label, artifactDir) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Flattened %s and removed nested structure", label))) - } + return + } + logsDownloadLog.Printf("Removed %s directory: %s", label, artifactDir) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Flattened %s and removed nested structure", label))) } - - return nil } // flattenUnifiedArtifact flattens the unified agent artifact directory structure. @@ -333,19 +334,7 @@ func downloadWorkflowRunLogs(ctx context.Context, runID int64, outputDir string, fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Downloading workflow run logs for run %d...", runID))) } - // Use gh api to download the logs zip file - // The endpoint returns a 302 redirect to the actual zip file - var endpoint string - if owner != "" && repo != "" { - endpoint = fmt.Sprintf("repos/%s/%s/actions/runs/%d/logs", owner, repo, runID) - } else { - endpoint = fmt.Sprintf("repos/{owner}/{repo}/actions/runs/%d/logs", runID) - } - - args := []string{"api", endpoint} - if hostname != "" && hostname != "github.com" { - args = append(args, "--hostname", hostname) - } + args := downloadWorkflowRunLogsArgs(runID, owner, repo, hostname) output, err := workflow.RunGHContext(ctx, "Downloading workflow logs...", args...) if err != nil { @@ -363,6 +352,7 @@ func downloadWorkflowRunLogs(ctx context.Context, runID int64, outputDir string, } return nil } + return fmt.Errorf("failed to download workflow run logs for run %d: %w", runID, err) } @@ -389,6 +379,20 @@ func downloadWorkflowRunLogs(ctx context.Context, runID int64, outputDir string, return nil } +func downloadWorkflowRunLogsArgs(runID int64, owner, repo, hostname string) []string { + var endpoint string + if owner != "" && repo != "" { + endpoint = fmt.Sprintf("repos/%s/%s/actions/runs/%d/logs", owner, repo, runID) + } else { + endpoint = fmt.Sprintf("repos/{owner}/{repo}/actions/runs/%d/logs", runID) + } + args := []string{"api", endpoint} + if hostname != "" && hostname != "github.com" { + args = append(args, "--hostname", hostname) + } + return args +} + // unzipFile extracts a zip file to a destination directory func unzipFile(zipPath, destDir string, verbose bool) error { // Open the zip file @@ -410,20 +414,9 @@ func unzipFile(zipPath, destDir string, verbose bool) error { // extractZipFile extracts a single file from a zip archive func extractZipFile(f *zip.File, destDir string, verbose bool) (extractErr error) { - // #nosec G305 - Path traversal is prevented by filepath.Clean and prefix check below - // Validate file name doesn't contain path traversal attempts - cleanName := filepath.Clean(f.Name) - if strings.Contains(cleanName, "..") { - return fmt.Errorf("invalid file path in zip (contains ..): %s", f.Name) - } - - // Construct the full path for the file - filePath := filepath.Join(destDir, cleanName) - - // Prevent zip slip vulnerability - ensure extracted path is within destDir - cleanDest := filepath.Clean(destDir) - if !strings.HasPrefix(filepath.Clean(filePath), cleanDest+string(os.PathSeparator)) && filepath.Clean(filePath) != cleanDest { - return fmt.Errorf("invalid file path in zip (outside destination): %s", f.Name) + cleanName, filePath, err := extractZipFileValidatePath(f, destDir) + if err != nil { + return err } if verbose { @@ -435,30 +428,11 @@ func extractZipFile(f *zip.File, destDir string, verbose bool) (extractErr error return os.MkdirAll(filePath, constants.DirPermPublic) } - // Decompression bomb protection - limit individual file size to 1GB - // #nosec G110 - Decompression bomb is mitigated by size check below - const maxFileSize = 1 * 1024 * 1024 * 1024 // 1GB - if f.UncompressedSize64 > maxFileSize { - return fmt.Errorf("file too large in zip (>1GB): %s (%d bytes)", f.Name, f.UncompressedSize64) - } - - // Create parent directory if needed - if err := os.MkdirAll(filepath.Dir(filePath), constants.DirPermPublic); err != nil { - return fmt.Errorf("failed to create directory: %w", err) - } - - // Open the file in the zip - srcFile, err := f.Open() + srcFile, destFile, err := extractZipFileOpenFiles(f, filePath) if err != nil { - return fmt.Errorf("failed to open file in zip: %w", err) + return err } defer srcFile.Close() - - // Create the destination file - destFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return fmt.Errorf("failed to create destination file: %w", err) - } defer func() { // Handle errors from closing the writable file to prevent data loss // Data written to a file may be cached in memory and only flushed when the file is closed. @@ -468,21 +442,59 @@ func extractZipFile(f *zip.File, destDir string, verbose bool) (extractErr error } }() - // Copy the content with size limit enforcement - // Use LimitReader to prevent reading more than declared size - limitedReader := io.LimitReader(srcFile, int64(maxFileSize)) - written, err := io.Copy(destFile, limitedReader) - if err != nil { + if err := extractZipFileCopy(f, srcFile, destFile); err != nil { extractErr = fmt.Errorf("failed to extract file: %w", err) return extractErr } + return nil +} - // Verify we didn't exceed the size limit - if uint64(written) > maxFileSize { - extractErr = fmt.Errorf("file extraction exceeded size limit: %s", f.Name) - return extractErr +func extractZipFileValidatePath(f *zip.File, destDir string) (string, string, error) { + // #nosec G305 - Path traversal is prevented by filepath.Clean and prefix check below + cleanName := filepath.Clean(f.Name) + if strings.Contains(cleanName, "..") { + return "", "", fmt.Errorf("invalid file path in zip (contains ..): %s", f.Name) + } + filePath := filepath.Join(destDir, cleanName) + cleanDest := filepath.Clean(destDir) + if !strings.HasPrefix(filepath.Clean(filePath), cleanDest+string(os.PathSeparator)) && filepath.Clean(filePath) != cleanDest { + return "", "", fmt.Errorf("invalid file path in zip (outside destination): %s", f.Name) } + return cleanName, filePath, nil +} +func extractZipFileOpenFiles(f *zip.File, filePath string) (io.ReadCloser, *os.File, error) { + // Decompression bomb protection - limit individual file size to 1GB + // #nosec G110 - Decompression bomb is mitigated by size check below + if f.UncompressedSize64 > extractZipFileMaxSize { + return nil, nil, fmt.Errorf("file too large in zip (>1GB): %s (%d bytes)", f.Name, f.UncompressedSize64) + } + if err := os.MkdirAll(filepath.Dir(filePath), constants.DirPermPublic); err != nil { + return nil, nil, fmt.Errorf("failed to create directory: %w", err) + } + srcFile, err := f.Open() + if err != nil { + return nil, nil, fmt.Errorf("failed to open file in zip: %w", err) + } + destFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + _ = srcFile.Close() + return nil, nil, fmt.Errorf("failed to create destination file: %w", err) + } + return srcFile, destFile, nil +} + +const extractZipFileMaxSize = 1 * 1024 * 1024 * 1024 // 1GB + +func extractZipFileCopy(f *zip.File, srcFile io.Reader, destFile io.Writer) error { + limitedReader := io.LimitReader(srcFile, int64(extractZipFileMaxSize)) + written, err := io.Copy(destFile, limitedReader) + if err != nil { + return err + } + if uint64(written) > extractZipFileMaxSize { + return fmt.Errorf("file extraction exceeded size limit: %s", f.Name) + } return nil } @@ -683,59 +695,93 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er shouldLogProgress := IsRunningInCI() || opts.verbose // Check if artifacts already exist on disk (since they're immutable) - if fileutil.DirExists(opts.outputDir) && !fileutil.IsDirEmpty(opts.outputDir) { - if len(opts.artifactFilter) > 0 { - // A specific artifact set is requested. Check whether each requested - // artifact base name already has a matching directory on disk so we - // can avoid re-downloading artifacts that are already present and only - // fetch the ones that are missing. - missing := findMissingFilterEntries(opts.artifactFilter, opts.outputDir) - if len(missing) == 0 { - logsDownloadLog.Printf("All requested artifacts already on disk for run %d", opts.runID) - if shouldLogProgress { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("All requested artifacts already present for run %d, skipping download", opts.runID))) - } - ensureUsageAwInfoFallback(ctx, opts) - return nil - } - // Restrict the download to only the artifacts that are not yet on disk. - logsDownloadLog.Printf("Downloading missing artifacts for run %d: %v (already have: %v)", opts.runID, missing, opts.artifactFilter) + if done := downloadRunArtifactsCheckCache(ctx, &opts, shouldLogProgress); done { + return nil + } + + if err := downloadRunArtifactsEnsureDir(opts); err != nil { + return err + } + + dockerBuildArtifacts, downloadableNames := downloadRunArtifactsList(ctx, opts) + + // Start spinner for network operation + spinner := console.NewSpinner(fmt.Sprintf("Downloading artifacts for run %d...", opts.runID)) + if !opts.verbose { + spinner.Start() + } + + if len(dockerBuildArtifacts) > 0 || len(opts.artifactFilter) > 0 { + if err := downloadRunArtifactsSelected(ctx, opts, downloadableNames, spinner); err != nil { + return err + } + } else { + if err := downloadRunArtifactsBulk(ctx, opts, downloadableNames, spinner); err != nil { + return err + } + } + + // Stop spinner with success message + if !opts.verbose { + spinner.StopWithMessage(fmt.Sprintf("✓ Downloaded artifacts for run %d", opts.runID)) + } + + if err := downloadRunArtifactsPostprocess(ctx, opts); err != nil { + return err + } + + if opts.verbose { + downloadRunArtifactsVerboseSummary(opts) + } + + return nil +} + +func downloadRunArtifactsCheckCache(ctx context.Context, opts *downloadArtifactsOptions, shouldLogProgress bool) bool { + if !fileutil.DirExists(opts.outputDir) || fileutil.IsDirEmpty(opts.outputDir) { + return false + } + if len(opts.artifactFilter) > 0 { + missing := findMissingFilterEntries(opts.artifactFilter, opts.outputDir) + if len(missing) == 0 { + logsDownloadLog.Printf("All requested artifacts already on disk for run %d", opts.runID) if shouldLogProgress { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Downloading missing artifacts for run %d: %v", opts.runID, missing))) - } - opts.artifactFilter = missing - // Fall through to the download code below (MkdirAll is a no-op for existing dir). - } else { - // No filter — caller wants all artifacts. Keep the existing behaviour: - // if the directory is non-empty we assume the run was previously fully - // downloaded and skip the download. - if summary, ok := loadRunSummary(opts.outputDir, opts.verbose); ok { - // Valid cached summary exists, skip download - logsDownloadLog.Printf("Using cached artifacts for run %d", opts.runID) - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Using cached artifacts for run %d at %s (from %s)", opts.runID, opts.outputDir, summary.ProcessedAt.Format("2006-01-02 15:04:05")))) - } - return nil - } - // Summary doesn't exist or version mismatch - artifacts exist but need reprocessing - // Don't re-download, just reprocess what's there - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run folder exists with artifacts, will reprocess run %d without re-downloading", opts.runID))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("All requested artifacts already present for run %d, skipping download", opts.runID))) } - return nil + ensureUsageAwInfoFallback(ctx, *opts) + return true + } + logsDownloadLog.Printf("Downloading missing artifacts for run %d: %v (already have: %v)", opts.runID, missing, opts.artifactFilter) + if shouldLogProgress { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Downloading missing artifacts for run %d: %v", opts.runID, missing))) } + opts.artifactFilter = missing + return false } + if summary, ok := loadRunSummary(opts.outputDir, opts.verbose); ok { + logsDownloadLog.Printf("Using cached artifacts for run %d", opts.runID) + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Using cached artifacts for run %d at %s (from %s)", opts.runID, opts.outputDir, summary.ProcessedAt.Format("2006-01-02 15:04:05")))) + } + return true + } + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run folder exists with artifacts, will reprocess run %d without re-downloading", opts.runID))) + } + return true +} +func downloadRunArtifactsEnsureDir(opts downloadArtifactsOptions) error { if err := os.MkdirAll(opts.outputDir, constants.DirPermPublic); err != nil { return fmt.Errorf("failed to create run output directory: %w", err) } if opts.verbose { fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Created output directory "+opts.outputDir)) } + return nil +} - // Proactively list artifacts to detect .dockerbuild files that gh run download cannot - // extract (they are not zip archives). When found, skip them and download the - // remaining artifacts individually so the bulk download never encounters them. +func downloadRunArtifactsList(ctx context.Context, opts downloadArtifactsOptions) ([]string, []string) { artifactNames, listErr := listRunArtifactNames(ctx, opts.runID, opts.owner, opts.repo, opts.hostname, opts.verbose) var dockerBuildArtifacts, downloadableNames []string if listErr == nil { @@ -746,270 +792,213 @@ func downloadRunArtifacts(ctx context.Context, opts downloadArtifactsOptions) er downloadableNames = append(downloadableNames, name) } } - if len(dockerBuildArtifacts) > 0 { - skipDockerBuildMessage := fmt.Sprintf("Skipping %d .dockerbuild artifact(s) (not valid zip archives): %s", len(dockerBuildArtifacts), strings.Join(dockerBuildArtifacts, ", ")) - logsDownloadLog.Printf("Found %d .dockerbuild artifact(s) that will be skipped: %v", len(dockerBuildArtifacts), dockerBuildArtifacts) - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(skipDockerBuildMessage)) - } - } + downloadRunArtifactsLogDockerBuilds(dockerBuildArtifacts, opts.verbose) + } else if len(opts.artifactFilter) > 0 { + downloadableNames = append(downloadableNames, opts.artifactFilter...) + logsDownloadLog.Printf("Could not list artifacts (will try requested artifact names directly): %v", listErr) } else { - if len(opts.artifactFilter) > 0 { - // When artifact listing fails, still try the requested artifact names directly - // so filtered downloads don't incorrectly return ErrNoArtifacts. - downloadableNames = append(downloadableNames, opts.artifactFilter...) - logsDownloadLog.Printf("Could not list artifacts (will try requested artifact names directly): %v", listErr) - } else { - logsDownloadLog.Printf("Could not list artifacts (will use bulk download): %v", listErr) - } + logsDownloadLog.Printf("Could not list artifacts (will use bulk download): %v", listErr) } + return dockerBuildArtifacts, downloadableNames +} - // Start spinner for network operation - spinner := console.NewSpinner(fmt.Sprintf("Downloading artifacts for run %d...", opts.runID)) +func downloadRunArtifactsLogDockerBuilds(dockerBuildArtifacts []string, verbose bool) { + if len(dockerBuildArtifacts) == 0 { + return + } + skipDockerBuildMessage := fmt.Sprintf("Skipping %d .dockerbuild artifact(s) (not valid zip archives): %s", len(dockerBuildArtifacts), strings.Join(dockerBuildArtifacts, ", ")) + logsDownloadLog.Printf("Found %d .dockerbuild artifact(s) that will be skipped: %v", len(dockerBuildArtifacts), dockerBuildArtifacts) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(skipDockerBuildMessage)) + } +} + +func downloadRunArtifactsSelected(ctx context.Context, opts downloadArtifactsOptions, downloadableNames []string, spinner *console.SpinnerWrapper) error { if !opts.verbose { - spinner.Start() + spinner.Stop() + } + if len(downloadableNames) == 0 { + if !isUsageOnlyArtifactFilter(opts.artifactFilter) { + downloadRunArtifactsTryLogs(ctx, opts, true) + } + return ErrNoArtifacts + } + if err := downloadArtifactsByName(ctx, opts, downloadableNames); err != nil { + return err } + if fileutil.IsDirEmpty(opts.outputDir) { + return ErrNoArtifacts + } + return nil +} - if len(dockerBuildArtifacts) > 0 || len(opts.artifactFilter) > 0 { - // When .dockerbuild artifacts are present or an artifact filter is active, download - // only the selected artifacts individually instead of using the bulk downloader. - // The bulk downloader (gh run download without --name) cannot apply a name filter, - // and it aborts on non-zip artifacts. - if !opts.verbose { - spinner.Stop() +func downloadRunArtifactsTryLogs(ctx context.Context, opts downloadArtifactsOptions, cleanupEmpty bool) { + if logErr := downloadWorkflowRunLogs(ctx, opts.runID, opts.outputDir, opts.verbose, opts.owner, opts.repo, opts.hostname); logErr != nil { + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to download workflow run logs: %v", logErr))) } - if len(downloadableNames) == 0 { - // Nothing to download (all artifacts are either .dockerbuild or excluded by filter). - // For usage-only mode, skip workflow logs entirely to keep downloads lightweight. - if !isUsageOnlyArtifactFilter(opts.artifactFilter) { - // Attempt workflow run logs for diagnostics before returning. - if logErr := downloadWorkflowRunLogs(ctx, opts.runID, opts.outputDir, opts.verbose, opts.owner, opts.repo, opts.hostname); logErr != nil { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to download workflow run logs: %v", logErr))) - } - if fileutil.IsDirEmpty(opts.outputDir) { - if removeErr := os.RemoveAll(opts.outputDir); removeErr != nil && opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to clean up empty directory %s: %v", opts.outputDir, removeErr))) - } - } - } + if cleanupEmpty && fileutil.IsDirEmpty(opts.outputDir) { + if removeErr := os.RemoveAll(opts.outputDir); removeErr != nil && opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to clean up empty directory %s: %v", opts.outputDir, removeErr))) } - return ErrNoArtifacts } - if err := downloadArtifactsByName(ctx, opts, downloadableNames); err != nil { + } +} + +func downloadRunArtifactsBulk(ctx context.Context, opts downloadArtifactsOptions, downloadableNames []string, spinner *console.SpinnerWrapper) error { + ghArgs := downloadRunArtifactsGHArgs(opts) + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Executing: gh "+strings.Join(ghArgs, " "))) + } + output, err := workflow.ExecGHContext(ctx, ghArgs...).CombinedOutput() + skippedNonZipArtifacts, skippedCaseCollisionArtifacts, err := downloadRunArtifactsHandleBulkError(ctx, opts, output, err, spinner) + if err != nil { + return err + } + if skippedNonZipArtifacts { + retryCriticalArtifacts(ctx, opts) + } + if skippedCaseCollisionArtifacts { + if err := downloadRunArtifactsRetryCaseCollision(ctx, opts, downloadableNames); err != nil { return err } - if fileutil.IsDirEmpty(opts.outputDir) { - // Downloads were attempted but none succeeded; treat as no artifacts. - return ErrNoArtifacts - } - } else { - // No .dockerbuild artifacts detected (or listing failed) — use efficient bulk download. - // Build gh run download command with optional repo/hostname override for cross-repo and multi-host support - ghArgs := []string{"run", "download", strconv.FormatInt(opts.runID, 10), "--dir", opts.outputDir} - if opts.owner != "" && opts.repo != "" { - if opts.hostname != "" && opts.hostname != "github.com" { - ghArgs = append(ghArgs, "-R", opts.hostname+"/"+opts.owner+"/"+opts.repo) - } else { - ghArgs = append(ghArgs, "-R", opts.owner+"/"+opts.repo) - } - } + } + if skippedNonZipArtifacts && fileutil.IsDirEmpty(opts.outputDir) { + return ErrNoArtifacts + } + return nil +} - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Executing: gh "+strings.Join(ghArgs, " "))) +func downloadRunArtifactsGHArgs(opts downloadArtifactsOptions) []string { + ghArgs := []string{"run", "download", strconv.FormatInt(opts.runID, 10), "--dir", opts.outputDir} + if opts.owner != "" && opts.repo != "" { + if opts.hostname != "" && opts.hostname != "github.com" { + ghArgs = append(ghArgs, "-R", opts.hostname+"/"+opts.owner+"/"+opts.repo) + } else { + ghArgs = append(ghArgs, "-R", opts.owner+"/"+opts.repo) } + } + return ghArgs +} - cmd := workflow.ExecGHContext(ctx, ghArgs...) - output, err := cmd.CombinedOutput() - - // skippedNonZipArtifacts is set when gh run download fails due to non-zip artifacts - // that were not detected during the listing phase (e.g., listing failed). - var skippedNonZipArtifacts bool - // skippedCaseCollisionArtifacts is set when gh run download fails because a single - // artifact contains case-colliding paths on case-insensitive filesystems. - var skippedCaseCollisionArtifacts bool - - if err != nil { - // Stop spinner on error - if !opts.verbose { - spinner.Stop() - } - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(string(output))) - } - - // Check if it's because there are no artifacts - if strings.Contains(string(output), "no valid artifacts") || strings.Contains(string(output), "not found") { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No artifacts found for run %d (gh run download reported none)", opts.runID))) - } - // Even with no artifacts, attempt to download workflow run logs so that - // pre-agent step failures (e.g., activation job errors) can be diagnosed. - if logErr := downloadWorkflowRunLogs(ctx, opts.runID, opts.outputDir, opts.verbose, opts.owner, opts.repo, opts.hostname); logErr != nil { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to download workflow run logs: %v", logErr))) - } - // Clean up empty directory only if logs download also produced nothing - if fileutil.IsDirEmpty(opts.outputDir) { - if removeErr := os.RemoveAll(opts.outputDir); removeErr != nil && opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to clean up empty directory %s: %v", opts.outputDir, removeErr))) - } - } - } - return ErrNoArtifacts - } - // Check for authentication errors - if isPermissionError(err) { - return errors.New("GitHub CLI authentication required. Run 'gh auth login' first") - } - // Check if the error is due to non-zip artifacts (e.g., .dockerbuild files). - // The gh CLI fails when it encounters artifacts that are not valid zip archives. - // We warn and continue with any artifacts that were successfully downloaded. - if isNonZipArtifactError(output) { - // Show a concise warning; preserve legacy behavior of 200 chars + "...". - msg := stringutil.Truncate(string(output), 203) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Some artifacts could not be extracted (not a valid zip archive) and were skipped: "+msg)) - skippedNonZipArtifacts = true - } else if isCaseCollisionArtifactError(output) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Some artifacts could not be fully extracted due to case-colliding file paths. Retrying artifacts individually and continuing.")) - skippedCaseCollisionArtifacts = true - } else { - return fmt.Errorf("failed to download artifacts for run %d: %w (output: %s)", opts.runID, err, string(output)) - } +func downloadRunArtifactsHandleBulkError(ctx context.Context, opts downloadArtifactsOptions, output []byte, err error, spinner *console.SpinnerWrapper) (bool, bool, error) { + if err == nil { + return false, false, nil + } + if !opts.verbose { + spinner.Stop() + } + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(string(output))) + } + if strings.Contains(string(output), "no valid artifacts") || strings.Contains(string(output), "not found") { + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No artifacts found for run %d (gh run download reported none)", opts.runID))) } + downloadRunArtifactsTryLogs(ctx, opts, true) + return false, false, ErrNoArtifacts + } + if isPermissionError(err) { + return false, false, errors.New("GitHub CLI authentication required. Run 'gh auth login' first") + } + if isNonZipArtifactError(output) { + msg := stringutil.Truncate(string(output), 203) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Some artifacts could not be extracted (not a valid zip archive) and were skipped: "+msg)) + return true, false, nil + } + if isCaseCollisionArtifactError(output) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Some artifacts could not be fully extracted due to case-colliding file paths. Retrying artifacts individually and continuing.")) + return false, true, nil + } + return false, false, fmt.Errorf("failed to download artifacts for run %d: %w (output: %s)", opts.runID, err, string(output)) +} - // When the bulk download failed due to non-zip artifacts, gh CLI may have aborted - // before downloading all valid artifacts. Retry individually for critical artifacts - // that are missing, so flattening and audit analysis can proceed. - if skippedNonZipArtifacts { - retryCriticalArtifacts(ctx, opts) +func downloadRunArtifactsRetryCaseCollision(ctx context.Context, opts downloadArtifactsOptions, downloadableNames []string) error { + retryNames := downloadableNames + if len(retryNames) == 0 { + artifactNamesRetry, retryListErr := listRunArtifactNames(ctx, opts.runID, opts.owner, opts.repo, opts.hostname, opts.verbose) + if retryListErr != nil { + return fmt.Errorf("bulk artifact download hit case-colliding entries and could not list artifacts for individual retry: %w", retryListErr) } - - // When bulk download fails on case-colliding entries, gh CLI aborts and may skip - // artifacts that appear later in the run. Retry artifacts individually so one bad - // artifact does not block the rest. - if skippedCaseCollisionArtifacts { - retryNames := downloadableNames - if len(retryNames) == 0 { - // Initial artifact listing was unavailable, so fetch names now for targeted retry. - artifactNamesRetry, retryListErr := listRunArtifactNames(ctx, opts.runID, opts.owner, opts.repo, opts.hostname, opts.verbose) - if retryListErr != nil { - return fmt.Errorf("bulk artifact download hit case-colliding entries and could not list artifacts for individual retry: %w", retryListErr) - } - for _, name := range artifactNamesRetry { - if isDockerBuildArtifact(name) { - continue - } - if artifactMatchesFilter(name, opts.artifactFilter) { - retryNames = append(retryNames, name) - } - } + for _, name := range artifactNamesRetry { + if !isDockerBuildArtifact(name) && artifactMatchesFilter(name, opts.artifactFilter) { + retryNames = append(retryNames, name) } - if len(retryNames) > 0 { - if err := downloadArtifactsByName(ctx, opts, retryNames); err != nil { - return err - } - } - } - - if skippedNonZipArtifacts && fileutil.IsDirEmpty(opts.outputDir) { - // All artifacts were non-zip (none could be extracted) so nothing was downloaded. - // Treat this the same as a run with no artifacts — the audit will rely solely on - // workflow logs rather than artifact content. - return ErrNoArtifacts } } - - // Stop spinner with success message - if !opts.verbose { - spinner.StopWithMessage(fmt.Sprintf("✓ Downloaded artifacts for run %d", opts.runID)) + if len(retryNames) > 0 { + return downloadArtifactsByName(ctx, opts, retryNames) } + return nil +} - // Flatten single-file artifacts +func downloadRunArtifactsPostprocess(ctx context.Context, opts downloadArtifactsOptions) error { if err := flattenSingleFileArtifacts(opts.outputDir, opts.verbose); err != nil { return fmt.Errorf("failed to flatten artifacts: %w", err) } - - // Flatten activation artifact directory structure (contains aw_info.json and prompt.txt) if err := flattenActivationArtifact(opts.outputDir, opts.verbose); err != nil { return fmt.Errorf("failed to flatten activation artifact: %w", err) } - ensureUsageAwInfoFallback(ctx, opts) - - // Flatten unified agent directory structure if err := flattenUnifiedArtifact(opts.outputDir, opts.verbose); err != nil { return fmt.Errorf("failed to flatten unified artifact: %w", err) } - - // Flatten agent_outputs artifact if present if err := flattenAgentOutputsArtifact(opts.outputDir, opts.verbose); err != nil { return fmt.Errorf("failed to flatten agent_outputs artifact: %w", err) } - - // Flatten safe-outputs-items artifact if present. - // This artifact contains safe-output-items.jsonl and temporary-id-map.json. - // Flattening moves them to the run root so extractCreatedItemsFromManifest - // and loadResolvedTemporaryIDTargets can find them at their expected paths. if err := flattenSafeOutputsItemsArtifact(opts.outputDir, opts.verbose); err != nil { return fmt.Errorf("failed to flatten safe-outputs-items artifact: %w", err) } - - // Download and unzip workflow run logs unless caller requested usage-only mode. if !isUsageOnlyArtifactFilter(opts.artifactFilter) { - if err := downloadWorkflowRunLogs(ctx, opts.runID, opts.outputDir, opts.verbose, opts.owner, opts.repo, opts.hostname); err != nil { - // Log the error but don't fail the entire download process - // Logs may not be available for all runs (e.g., expired or deleted) - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to download workflow run logs: %v", err))) - } + downloadRunArtifactsTryLogs(ctx, opts, false) + } + return nil +} + +func downloadRunArtifactsVerboseSummary(opts downloadArtifactsOptions) { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Downloaded artifacts for run %d to %s", opts.runID, opts.outputDir))) + fileCount, firstFiles, walkFailed := downloadRunArtifactsListFiles(opts.outputDir) + if fileCount == 0 { + if walkFailed { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Download completed but artifact files could not be enumerated (filesystem error)")) + } else { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Download completed but no artifact files were created (empty run)")) } + return } + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Artifact file count: %d", fileCount))) + for _, f := range firstFiles { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(" • "+f)) + } + if fileCount > len(firstFiles) { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf(" … %d more files omitted", fileCount-len(firstFiles)))) + } +} - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Downloaded artifacts for run %d to %s", opts.runID, opts.outputDir))) - // Enumerate created files (shallow + summary) for immediate visibility - var fileCount int - var firstFiles []string - var walkFailed bool - if walkErr := filepath.Walk(opts.outputDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - logsDownloadLog.Printf("walk error at %s: %v", path, err) - walkFailed = true - return nil - } - if info.IsDir() { - return nil - } - fileCount++ - if len(firstFiles) < 12 { // capture a reasonable preview - rel, relErr := filepath.Rel(opts.outputDir, path) - if relErr == nil { - firstFiles = append(firstFiles, rel) - } - } +func downloadRunArtifactsListFiles(outputDir string) (int, []string, bool) { + var fileCount int + var firstFiles []string + var walkFailed bool + if walkErr := filepath.Walk(outputDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + logsDownloadLog.Printf("walk error at %s: %v", path, err) + walkFailed = true return nil - }); walkErr != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error enumerating artifacts in %s: %v", opts.outputDir, walkErr))) } - if fileCount == 0 { - if walkFailed { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Download completed but artifact files could not be enumerated (filesystem error)")) - } else { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Download completed but no artifact files were created (empty run)")) - } - } else { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Artifact file count: %d", fileCount))) - for _, f := range firstFiles { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(" • "+f)) - } - if fileCount > len(firstFiles) { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf(" … %d more files omitted", fileCount-len(firstFiles)))) + if info.IsDir() { + return nil + } + fileCount++ + if len(firstFiles) < 12 { + if rel, relErr := filepath.Rel(outputDir, path); relErr == nil { + firstFiles = append(firstFiles, rel) } } + return nil + }); walkErr != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error enumerating artifacts in %s: %v", outputDir, walkErr))) } - - return nil + return fileCount, firstFiles, walkFailed } func ensureUsageAwInfoFallback(ctx context.Context, opts downloadArtifactsOptions) { diff --git a/pkg/cli/logs_episode.go b/pkg/cli/logs_episode.go index 6df9d3badbb..10d711dd86c 100644 --- a/pkg/cli/logs_episode.go +++ b/pkg/cli/logs_episode.go @@ -102,6 +102,20 @@ type episodeSeed struct { func buildEpisodeData(runs []RunData, processedRuns []ProcessedRun) ([]EpisodeData, []EpisodeEdge) { logsEpisodeLog.Printf("Building episode data: runs=%d processed_runs=%d", len(runs), len(processedRuns)) + runsByID, processedByID, seedsByRunID, parents := buildEpisodeDataIndexes(runs, processedRuns) + edges := buildEpisodeDataEdges(runs, runsByID, parents) + rootMetadata := buildEpisodeDataRootMetadata(runs, parents, seedsByRunID) + episodeMap := buildEpisodeDataAccumulators(runs, processedByID, parents, rootMetadata) + buildEpisodeDataAssignEdgeEpisodes(edges, parents, rootMetadata) + episodes := buildEpisodeDataFinalizeEpisodes(episodeMap) + + buildEpisodeDataSort(episodes, edges) + + logsEpisodeLog.Printf("Built %d episodes and %d edges from %d runs", len(episodes), len(edges), len(runs)) + return episodes, edges +} + +func buildEpisodeDataIndexes(runs []RunData, processedRuns []ProcessedRun) (map[int64]RunData, map[int64]ProcessedRun, map[int64]episodeSeed, map[int64]int64) { runsByID := make(map[int64]RunData, len(runs)) processedByID := make(map[int64]ProcessedRun, len(processedRuns)) seedsByRunID := make(map[int64]episodeSeed, len(runs)) @@ -115,7 +129,10 @@ func buildEpisodeData(runs []RunData, processedRuns []ProcessedRun) ([]EpisodeDa for _, processedRun := range processedRuns { processedByID[processedRun.Run.DatabaseID] = processedRun } + return runsByID, processedByID, seedsByRunID, parents +} +func buildEpisodeDataEdges(runs []RunData, runsByID map[int64]RunData, parents map[int64]int64) []EpisodeEdge { edges := make([]EpisodeEdge, 0) for _, run := range runs { if edge, ok := buildEpisodeEdge(run, runs, runsByID); ok { @@ -123,8 +140,10 @@ func buildEpisodeData(runs []RunData, processedRuns []ProcessedRun) ([]EpisodeDa unionEpisodes(parents, edge.SourceRunID, edge.TargetRunID) } } + return edges +} - episodeMap := make(map[string]*episodeAccumulator) +func buildEpisodeDataRootMetadata(runs []RunData, parents map[int64]int64, seedsByRunID map[int64]episodeSeed) map[int64]episodeSeed { rootMetadata := make(map[int64]episodeSeed) for _, run := range runs { root := findEpisodeParent(parents, run.RunID) @@ -134,154 +153,212 @@ func buildEpisodeData(runs []RunData, processedRuns []ProcessedRun) ([]EpisodeDa rootMetadata[root] = seed } } + return rootMetadata +} +func buildEpisodeDataAccumulators(runs []RunData, processedByID map[int64]ProcessedRun, parents map[int64]int64, rootMetadata map[int64]episodeSeed) map[string]*episodeAccumulator { + episodeMap := make(map[string]*episodeAccumulator) for _, run := range runs { root := findEpisodeParent(parents, run.RunID) - selectedSeed := rootMetadata[root] - episodeID, kind, confidence, reasons := selectedSeed.EpisodeID, selectedSeed.Kind, selectedSeed.Confidence, selectedSeed.Reasons - acc, exists := episodeMap[episodeID] - if !exists { - acc = &episodeAccumulator{ - metadata: EpisodeData{ - EpisodeID: episodeID, - Kind: kind, - Confidence: confidence, - Reasons: append([]string(nil), reasons...), - RunIDs: []int64{}, - WorkflowNames: []string{}, - RiskDistribution: "none", - }, - runSet: make(map[int64]bool), - nameSet: make(map[string]bool), - rootTime: run.CreatedAt, - } - episodeMap[episodeID] = acc - } + acc := buildEpisodeDataAccumulatorForRun(episodeMap, run, rootMetadata[root]) + buildEpisodeDataAddRun(acc, run, processedByID) + } + return episodeMap +} - if !acc.runSet[run.RunID] { - acc.runSet[run.RunID] = true - acc.metadata.RunIDs = append(acc.metadata.RunIDs, run.RunID) - } - if run.WorkflowName != "" && !acc.nameSet[run.WorkflowName] { - acc.nameSet[run.WorkflowName] = true - acc.metadata.WorkflowNames = append(acc.metadata.WorkflowNames, run.WorkflowName) - } +func buildEpisodeDataAccumulatorForRun(episodeMap map[string]*episodeAccumulator, run RunData, selectedSeed episodeSeed) *episodeAccumulator { + acc, exists := episodeMap[selectedSeed.EpisodeID] + if exists { + return acc + } + acc = &episodeAccumulator{ + metadata: EpisodeData{ + EpisodeID: selectedSeed.EpisodeID, + Kind: selectedSeed.Kind, + Confidence: selectedSeed.Confidence, + Reasons: append([]string(nil), selectedSeed.Reasons...), + RunIDs: []int64{}, + WorkflowNames: []string{}, + RiskDistribution: "none", + }, + runSet: make(map[int64]bool), + nameSet: make(map[string]bool), + rootTime: run.CreatedAt, + } + episodeMap[selectedSeed.EpisodeID] = acc + return acc +} - acc.metadata.TotalRuns++ - acc.metadata.TotalTokens += run.TokenUsage - acc.metadata.TotalAIC += run.AIC - acc.metadata.ManifestEntryCount += run.ManifestEntryCount - acc.metadata.TemporaryIDMappings += run.TemporaryIDMappings - acc.metadata.ChainedTargetCount += run.ChainedTargetCount - acc.metadata.ChainedFollowupActionCount += run.ChainedFollowupActionCount - acc.metadata.DelegatedTempTargetCount += run.DelegatedTempTargetCount - acc.metadata.ClosedTempTargetCount += run.ClosedTempTargetCount - if run.Comparison != nil && run.Comparison.Classification != nil && run.Comparison.Classification.Label == "risky" { - acc.metadata.RiskyNodeCount++ - } - if run.Comparison != nil && run.Comparison.Classification != nil && run.Comparison.Classification.Label == "changed" { - acc.metadata.ChangedNodeCount++ - } - if run.BehaviorFingerprint != nil && run.BehaviorFingerprint.ActuationStyle != "read_only" { - acc.metadata.WriteCapableNodeCount++ - } - if run.Comparison != nil && run.Comparison.Baseline != nil && run.Comparison.Baseline.Selection == "latest_success" { - acc.metadata.LatestSuccessFallbackCount++ - } - if hasComparisonReasonCode(run.Comparison, "new_mcp_failure") { - acc.metadata.NewMCPFailureRunCount++ - } - if hasComparisonReasonCode(run.Comparison, "blocked_requests_increase") { - acc.metadata.BlockedRequestIncreaseRunCount++ - } - if hasAssessmentKindAtLeast(run.AgenticAssessments, "resource_heavy_for_domain", "medium") { - acc.metadata.ResourceHeavyNodeCount++ - } - if hasAssessmentKindAtLeast(run.AgenticAssessments, "poor_agentic_control", "medium") { - acc.metadata.PoorControlNodeCount++ - } - acc.metadata.MissingToolCount += run.MissingToolCount - if pr, ok := processedByID[run.RunID]; ok { - acc.metadata.MCPFailureCount += len(pr.MCPFailures) - if pr.FirewallAnalysis != nil { - acc.metadata.BlockedRequestCount += pr.FirewallAnalysis.BlockedRequests - if pr.FirewallAnalysis.BlockedRequests >= firewallBlockedRequestCap { - acc.metadata.BlockedRequestAtCap = true - } - } - // Collect per-tool-call metrics for this run. - if pr.MCPToolUsage != nil { - for _, tc := range pr.MCPToolUsage.ToolCalls { - acc.toolCalls = append(acc.toolCalls, mcpToolCallToEpisodeToolCall(tc)) - } - } - } - if !run.CreatedAt.IsZero() && (acc.metadata.RootRunID == 0 || run.CreatedAt.Before(acc.rootTime)) { - acc.rootTime = run.CreatedAt - acc.metadata.RootRunID = run.RunID - acc.metadata.PrimaryWorkflow = run.WorkflowName - } - if acc.metadata.PrimaryWorkflow == "" && run.WorkflowName != "" { - acc.metadata.PrimaryWorkflow = run.WorkflowName +func buildEpisodeDataAddRun(acc *episodeAccumulator, run RunData, processedByID map[int64]ProcessedRun) { + buildEpisodeDataAddIdentity(acc, run) + buildEpisodeDataAddMetrics(acc, run) + buildEpisodeDataAddProcessed(acc, run, processedByID) + buildEpisodeDataAddMetadata(acc, run, processedByID) +} + +func buildEpisodeDataAddIdentity(acc *episodeAccumulator, run RunData) { + if !acc.runSet[run.RunID] { + acc.runSet[run.RunID] = true + acc.metadata.RunIDs = append(acc.metadata.RunIDs, run.RunID) + } + if run.WorkflowName != "" && !acc.nameSet[run.WorkflowName] { + acc.nameSet[run.WorkflowName] = true + acc.metadata.WorkflowNames = append(acc.metadata.WorkflowNames, run.WorkflowName) + } +} + +func buildEpisodeDataAddMetrics(acc *episodeAccumulator, run RunData) { + acc.metadata.TotalRuns++ + acc.metadata.TotalTokens += run.TokenUsage + acc.metadata.TotalAIC += run.AIC + acc.metadata.ManifestEntryCount += run.ManifestEntryCount + acc.metadata.TemporaryIDMappings += run.TemporaryIDMappings + acc.metadata.ChainedTargetCount += run.ChainedTargetCount + acc.metadata.ChainedFollowupActionCount += run.ChainedFollowupActionCount + acc.metadata.DelegatedTempTargetCount += run.DelegatedTempTargetCount + acc.metadata.ClosedTempTargetCount += run.ClosedTempTargetCount + acc.metadata.MissingToolCount += run.MissingToolCount +} + +func buildEpisodeDataAddProcessed(acc *episodeAccumulator, run RunData, processedByID map[int64]ProcessedRun) { + if run.Comparison != nil && run.Comparison.Classification != nil && run.Comparison.Classification.Label == "risky" { + acc.metadata.RiskyNodeCount++ + } + if run.Comparison != nil && run.Comparison.Classification != nil && run.Comparison.Classification.Label == "changed" { + acc.metadata.ChangedNodeCount++ + } + if run.BehaviorFingerprint != nil && run.BehaviorFingerprint.ActuationStyle != "read_only" { + acc.metadata.WriteCapableNodeCount++ + } + if run.Comparison != nil && run.Comparison.Baseline != nil && run.Comparison.Baseline.Selection == "latest_success" { + acc.metadata.LatestSuccessFallbackCount++ + } + buildEpisodeDataAddAssessmentCounts(acc, run) + buildEpisodeDataAddProcessedRun(acc, run, processedByID) +} + +func buildEpisodeDataAddAssessmentCounts(acc *episodeAccumulator, run RunData) { + if hasComparisonReasonCode(run.Comparison, "new_mcp_failure") { + acc.metadata.NewMCPFailureRunCount++ + } + if hasComparisonReasonCode(run.Comparison, "blocked_requests_increase") { + acc.metadata.BlockedRequestIncreaseRunCount++ + } + if hasAssessmentKindAtLeast(run.AgenticAssessments, "resource_heavy_for_domain", "medium") { + acc.metadata.ResourceHeavyNodeCount++ + } + if hasAssessmentKindAtLeast(run.AgenticAssessments, "poor_agentic_control", "medium") { + acc.metadata.PoorControlNodeCount++ + } +} + +func buildEpisodeDataAddProcessedRun(acc *episodeAccumulator, run RunData, processedByID map[int64]ProcessedRun) { + pr, ok := processedByID[run.RunID] + if !ok { + return + } + acc.metadata.MCPFailureCount += len(pr.MCPFailures) + if pr.FirewallAnalysis != nil { + acc.metadata.BlockedRequestCount += pr.FirewallAnalysis.BlockedRequests + if pr.FirewallAnalysis.BlockedRequests >= firewallBlockedRequestCap { + acc.metadata.BlockedRequestAtCap = true } - if acc.metadata.Repository == "" && run.Repository != "" { - acc.metadata.Repository = run.Repository - if parts := strings.SplitN(run.Repository, "/", 2); len(parts) == 2 { - acc.metadata.Organization = parts[0] - } + } + if pr.MCPToolUsage != nil { + for _, tc := range pr.MCPToolUsage.ToolCalls { + acc.toolCalls = append(acc.toolCalls, mcpToolCallToEpisodeToolCall(tc)) } - if run.StartedAt.IsZero() && run.UpdatedAt.IsZero() { - acc.duration += run.CreatedAt.Sub(run.CreatedAt) - } else if !run.StartedAt.IsZero() && !run.UpdatedAt.IsZero() && run.UpdatedAt.After(run.StartedAt) { - acc.duration += run.UpdatedAt.Sub(run.StartedAt) - } else if pr, ok := processedByID[run.RunID]; ok && pr.Run.Duration > 0 { - acc.duration += pr.Run.Duration + } +} + +func buildEpisodeDataAddMetadata(acc *episodeAccumulator, run RunData, processedByID map[int64]ProcessedRun) { + if !run.CreatedAt.IsZero() && (acc.metadata.RootRunID == 0 || run.CreatedAt.Before(acc.rootTime)) { + acc.rootTime = run.CreatedAt + acc.metadata.RootRunID = run.RunID + acc.metadata.PrimaryWorkflow = run.WorkflowName + } + if acc.metadata.PrimaryWorkflow == "" && run.WorkflowName != "" { + acc.metadata.PrimaryWorkflow = run.WorkflowName + } + if acc.metadata.Repository == "" && run.Repository != "" { + acc.metadata.Repository = run.Repository + if parts := strings.SplitN(run.Repository, "/", 2); len(parts) == 2 { + acc.metadata.Organization = parts[0] } } + buildEpisodeDataAddDuration(acc, run, processedByID) +} +func buildEpisodeDataAddDuration(acc *episodeAccumulator, run RunData, processedByID map[int64]ProcessedRun) { + if run.StartedAt.IsZero() && run.UpdatedAt.IsZero() { + acc.duration += run.CreatedAt.Sub(run.CreatedAt) + } else if !run.StartedAt.IsZero() && !run.UpdatedAt.IsZero() && run.UpdatedAt.After(run.StartedAt) { + acc.duration += run.UpdatedAt.Sub(run.StartedAt) + } else if pr, ok := processedByID[run.RunID]; ok && pr.Run.Duration > 0 { + acc.duration += pr.Run.Duration + } +} + +func buildEpisodeDataAssignEdgeEpisodes(edges []EpisodeEdge, parents map[int64]int64, rootMetadata map[int64]episodeSeed) { for index := range edges { root := findEpisodeParent(parents, edges[index].TargetRunID) if selectedSeed, ok := rootMetadata[root]; ok { edges[index].EpisodeID = selectedSeed.EpisodeID } } +} +func buildEpisodeDataFinalizeEpisodes(episodeMap map[string]*episodeAccumulator) []EpisodeData { episodes := make([]EpisodeData, 0, len(episodeMap)) for _, acc := range episodeMap { - slices.Sort(acc.metadata.RunIDs) - slices.Sort(acc.metadata.WorkflowNames) - if acc.duration > 0 { - acc.metadata.TotalDuration = timeutil.FormatDuration(acc.duration) - } - if acc.metadata.PrimaryWorkflow == "" && len(acc.metadata.WorkflowNames) > 0 { - acc.metadata.PrimaryWorkflow = acc.metadata.WorkflowNames[0] - } - switch acc.metadata.RiskyNodeCount { - case 0: - acc.metadata.RiskDistribution = "none" - case 1: - acc.metadata.RiskDistribution = "concentrated" - default: - acc.metadata.RiskDistribution = "distributed" - } - acc.metadata.EscalationEligible, acc.metadata.EscalationReason = classifyEpisodeEscalation(acc.metadata) - acc.metadata.SuggestedRoute = buildSuggestedRoute(acc.metadata) - if len(acc.toolCalls) > 0 { - // Sort tool calls for deterministic output (server, then tool name, then status). - slices.SortFunc(acc.toolCalls, func(a, b EpisodeToolCall) int { - if a.Server != b.Server { - return cmp.Compare(a.Server, b.Server) - } - if a.Tool != b.Tool { - return cmp.Compare(a.Tool, b.Tool) - } - return cmp.Compare(a.Status, b.Status) - }) - acc.metadata.ToolCalls = acc.toolCalls - } + buildEpisodeDataFinalizeAccumulator(acc) episodes = append(episodes, acc.metadata) } + return episodes +} +func buildEpisodeDataFinalizeAccumulator(acc *episodeAccumulator) { + slices.Sort(acc.metadata.RunIDs) + slices.Sort(acc.metadata.WorkflowNames) + if acc.duration > 0 { + acc.metadata.TotalDuration = timeutil.FormatDuration(acc.duration) + } + if acc.metadata.PrimaryWorkflow == "" && len(acc.metadata.WorkflowNames) > 0 { + acc.metadata.PrimaryWorkflow = acc.metadata.WorkflowNames[0] + } + acc.metadata.RiskDistribution = buildEpisodeDataRiskDistribution(acc.metadata.RiskyNodeCount) + acc.metadata.EscalationEligible, acc.metadata.EscalationReason = classifyEpisodeEscalation(acc.metadata) + acc.metadata.SuggestedRoute = buildSuggestedRoute(acc.metadata) + if len(acc.toolCalls) > 0 { + buildEpisodeDataSortToolCalls(acc.toolCalls) + acc.metadata.ToolCalls = acc.toolCalls + } +} + +func buildEpisodeDataRiskDistribution(riskyNodeCount int) string { + switch riskyNodeCount { + case 0: + return "none" + case 1: + return "concentrated" + default: + return "distributed" + } +} + +func buildEpisodeDataSortToolCalls(toolCalls []EpisodeToolCall) { + slices.SortFunc(toolCalls, func(a, b EpisodeToolCall) int { + if a.Server != b.Server { + return cmp.Compare(a.Server, b.Server) + } + if a.Tool != b.Tool { + return cmp.Compare(a.Tool, b.Tool) + } + return cmp.Compare(a.Status, b.Status) + }) +} + +func buildEpisodeDataSort(episodes []EpisodeData, edges []EpisodeEdge) { slices.SortFunc(episodes, func(a, b EpisodeData) int { if a.RootRunID != b.RootRunID { return cmp.Compare(a.RootRunID, b.RootRunID) @@ -294,9 +371,6 @@ func buildEpisodeData(runs []RunData, processedRuns []ProcessedRun) ([]EpisodeDa } return cmp.Compare(a.TargetRunID, b.TargetRunID) }) - - logsEpisodeLog.Printf("Built %d episodes and %d edges from %d runs", len(episodes), len(edges), len(runs)) - return episodes, edges } // mcpToolCallToEpisodeToolCall converts an MCPToolCall record to the lightweight diff --git a/pkg/cli/logs_format_compact.go b/pkg/cli/logs_format_compact.go index 839c133501c..ebe5e3e6ff9 100644 --- a/pkg/cli/logs_format_compact.go +++ b/pkg/cli/logs_format_compact.go @@ -56,9 +56,19 @@ func workflowIDFromRun(path, name string) string { func renderLogsCompact(data LogsData) { logsCompactLog.Printf("Rendering %d runs in compact format", data.Summary.TotalRuns) - s := data.Summary + renderLogsCompactSummary(data.Summary) - // [summary] single line of key=value pairs + if len(data.Runs) == 0 { + return + } + + renderLogsCompactRuns(data.Runs) + + renderLogsCompactSections(data) + renderLogsCompactHint(data) +} + +func renderLogsCompactSummary(s LogsSummary) { summaryParts := []string{ "runs=" + strconv.Itoa(s.TotalRuns), "duration=" + s.TotalDuration, @@ -80,140 +90,170 @@ func renderLogsCompact(data LogsData) { if s.TotalGitHubAPICalls > 0 { summaryParts = append(summaryParts, "github_api="+strconv.Itoa(s.TotalGitHubAPICalls)) } - if len(s.EngineCounts) > 0 { - parts := make([]string, 0, len(s.EngineCounts)) - for engine, count := range s.EngineCounts { - parts = append(parts, engine+":"+strconv.Itoa(count)) - } - summaryParts = append(summaryParts, "engines="+strings.Join(parts, ",")) - } - // Outcome metrics if available - if s.OutcomeAccepted > 0 || s.OutcomeRejected > 0 { - summaryParts = append(summaryParts, - "accepted="+strconv.Itoa(s.OutcomeAccepted), - "rejected="+strconv.Itoa(s.OutcomeRejected), - ) - if s.OutcomeAcceptanceRate > 0 { - summaryParts = append(summaryParts, "acceptance="+fmt.Sprintf("%.0f%%", s.OutcomeAcceptanceRate*100)) - } - } + renderLogsCompactAppendEngineCounts(&summaryParts, s.EngineCounts) + renderLogsCompactAppendOutcome(&summaryParts, s, false) fmt.Fprintf(os.Stdout, "[summary] %s\n", strings.Join(summaryParts, " ")) +} - if len(data.Runs) == 0 { +func renderLogsCompactAppendEngineCounts(summaryParts *[]string, engineCounts map[string]int) { + if len(engineCounts) == 0 { return } + parts := make([]string, 0, len(engineCounts)) + for engine, count := range engineCounts { + parts = append(parts, engine+":"+strconv.Itoa(count)) + } + *summaryParts = append(*summaryParts, "engines="+strings.Join(parts, ",")) +} - // [runs] aligned table using tabwriter +func renderLogsCompactAppendOutcome(summaryParts *[]string, s LogsSummary, verbose bool) { + if s.OutcomeAccepted == 0 && s.OutcomeRejected == 0 { + return + } + *summaryParts = append(*summaryParts, "accepted="+strconv.Itoa(s.OutcomeAccepted), "rejected="+strconv.Itoa(s.OutcomeRejected)) + if verbose { + *summaryParts = append(*summaryParts, "ignored="+strconv.Itoa(s.OutcomeIgnored), "pending="+strconv.Itoa(s.OutcomePending)) + } + if s.OutcomeAcceptanceRate > 0 { + *summaryParts = append(*summaryParts, "acceptance="+fmt.Sprintf("%.0f%%", s.OutcomeAcceptanceRate*100)) + } + if verbose && s.OutcomeWasteRate > 0 { + *summaryParts = append(*summaryParts, "waste="+fmt.Sprintf("%.0f%%", s.OutcomeWasteRate*100)) + } +} + +func renderLogsCompactRuns(runs []RunData) { fmt.Fprintln(os.Stdout, "[runs]") w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) fmt.Fprintln(w, "RUNID\tWORKFLOW\tENGINE\tSTATUS\tDUR\tTOKENS\tAIC\tTURNS\tERR\tEVENT\tACTOR\tBRANCH") - - for _, r := range data.Runs { - status := r.Conclusion - if status == "" { - status = r.Status - } + for _, r := range runs { + status, dur, actor := renderLogsCompactRunFields(r) if status == "skipped" || status == "cancelled" { continue } - dur := r.Duration - if dur == "" { - dur = "-" - } branch := stringutil.Truncate(r.Branch, 30) - actor := r.Actor - if actor == "" { - actor = "-" - } wfID := workflowIDFromRun(r.WorkflowPath, r.WorkflowName) - fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%d\t%s\t%d\t%d\t%s\t%s\t%s\n", r.RunID, wfID, r.EngineID, status, dur, r.TokenUsage, formatCompactAIC(r.AIC), r.Turns, r.ErrorCount, r.Event, actor, branch) } w.Flush() +} - // [errors] — aggregated error/warning messages - if len(data.ErrorsAndWarnings) > 0 { - fmt.Fprintln(os.Stdout, "[errors]") - for _, ew := range data.ErrorsAndWarnings { - msg := stringutil.Truncate(ew.Message, 120) - fmt.Fprintf(os.Stdout, "%s run=%d count=%d: %s\n", ew.Type, ew.RunID, ew.Count, msg) - } +func renderLogsCompactRunFields(r RunData) (string, string, string) { + status := r.Conclusion + if status == "" { + status = r.Status } + dur := r.Duration + if dur == "" { + dur = "-" + } + actor := r.Actor + if actor == "" { + actor = "-" + } + return status, dur, actor +} - // [insights] — only medium/high severity (skip info-level noise) - if len(data.Observability) > 0 { - var hasActionable bool - for _, obs := range data.Observability { - if obs.Severity != "info" { - hasActionable = true - break - } - } - if hasActionable { - fmt.Fprintln(os.Stdout, "[insights]") - for _, obs := range data.Observability { - if obs.Severity == "info" { - continue - } - fmt.Fprintf(os.Stdout, "[%s] %s: %s\n", obs.Severity, obs.Title, obs.Summary) - } - } +func renderLogsCompactSections(data LogsData) { + renderLogsCompactErrors(data.ErrorsAndWarnings) + renderLogsCompactInsights(data.Observability) + renderLogsCompactFirewall(data.FirewallLog) + renderLogsCompactTools(data.ToolUsage) + renderLogsCompactMCPFailures(data.MCPFailures) + renderLogsCompactMissingTools(data.MissingTools) + if data.LogsLocation != "" { + fmt.Fprintf(os.Stdout, "[location] %s\n", data.LogsLocation) } +} - // [firewall] — summary + per-domain breakdown - if data.FirewallLog != nil && data.FirewallLog.TotalRequests > 0 { - fw := data.FirewallLog - fmt.Fprintf(os.Stdout, "[firewall] requests=%d allowed=%d blocked=%d\n", - fw.TotalRequests, fw.AllowedRequests, fw.BlockedRequests) - if len(fw.RequestsByDomain) > 0 { - for domain, counts := range fw.RequestsByDomain { - if counts.Blocked > 0 { - fmt.Fprintf(os.Stdout, " %s allowed=%d blocked=%d\n", domain, counts.Allowed, counts.Blocked) - } - } - } else if len(fw.BlockedDomains) > 0 { - fmt.Fprintf(os.Stdout, " blocked: %s\n", strings.Join(fw.BlockedDomains, " ")) - } +func renderLogsCompactErrors(errorsAndWarnings []ErrorSummary) { + if len(errorsAndWarnings) == 0 { + return } + fmt.Fprintln(os.Stdout, "[errors]") + for _, ew := range errorsAndWarnings { + msg := stringutil.Truncate(ew.Message, 120) + fmt.Fprintf(os.Stdout, "%s run=%d count=%d: %s\n", ew.Type, ew.RunID, ew.Count, msg) + } +} - // [tools] — top tools by call count - if len(data.ToolUsage) > 0 { - fmt.Fprintln(os.Stdout, "[tools]") - limit := min(10, len(data.ToolUsage)) - for i := range limit { - t := data.ToolUsage[i] - fmt.Fprintf(os.Stdout, "%s calls=%d runs=%d\n", t.Name, t.TotalCalls, t.Runs) +func renderLogsCompactInsights(observability []ObservabilityInsight) { + if len(observability) == 0 { + return + } + var hasActionable bool + for _, obs := range observability { + if obs.Severity != "info" { + hasActionable = true + break } - if len(data.ToolUsage) > limit { - fmt.Fprintf(os.Stdout, "... +%d more tools\n", len(data.ToolUsage)-limit) + } + if !hasActionable { + return + } + fmt.Fprintln(os.Stdout, "[insights]") + for _, obs := range observability { + if obs.Severity != "info" { + fmt.Fprintf(os.Stdout, "[%s] %s: %s\n", obs.Severity, obs.Title, obs.Summary) } } +} - // [mcp-failures] - if len(data.MCPFailures) > 0 { - fmt.Fprintln(os.Stdout, "[mcp-failures]") - for _, f := range data.MCPFailures { - fmt.Fprintf(os.Stdout, "server=%s count=%d runs=%v\n", f.ServerName, f.Count, f.RunIDs) +func renderLogsCompactFirewall(fw *FirewallLogSummary) { + if fw == nil || fw.TotalRequests == 0 { + return + } + fmt.Fprintf(os.Stdout, "[firewall] requests=%d allowed=%d blocked=%d\n", fw.TotalRequests, fw.AllowedRequests, fw.BlockedRequests) + if len(fw.RequestsByDomain) > 0 { + for domain, counts := range fw.RequestsByDomain { + if counts.Blocked > 0 { + fmt.Fprintf(os.Stdout, " %s allowed=%d blocked=%d\n", domain, counts.Allowed, counts.Blocked) + } } + } else if len(fw.BlockedDomains) > 0 { + fmt.Fprintf(os.Stdout, " blocked: %s\n", strings.Join(fw.BlockedDomains, " ")) } +} - // [missing-tools] — missing tool summary - if len(data.MissingTools) > 0 { - fmt.Fprintln(os.Stdout, "[missing-tools]") - for _, mt := range data.MissingTools { - fmt.Fprintf(os.Stdout, "%s count=%d runs=%v\n", mt.Tool, mt.Count, mt.RunIDs) - } +func renderLogsCompactTools(toolUsage []ToolUsageSummary) { + if len(toolUsage) == 0 { + return + } + fmt.Fprintln(os.Stdout, "[tools]") + limit := min(10, len(toolUsage)) + for i := range limit { + t := toolUsage[i] + fmt.Fprintf(os.Stdout, "%s calls=%d runs=%d\n", t.Name, t.TotalCalls, t.Runs) + } + if len(toolUsage) > limit { + fmt.Fprintf(os.Stdout, "... +%d more tools\n", len(toolUsage)-limit) } +} - // [location] - if data.LogsLocation != "" { - fmt.Fprintf(os.Stdout, "[location] %s\n", data.LogsLocation) +func renderLogsCompactMCPFailures(failures []MCPFailureSummary) { + if len(failures) == 0 { + return + } + fmt.Fprintln(os.Stdout, "[mcp-failures]") + for _, f := range failures { + fmt.Fprintf(os.Stdout, "server=%s count=%d runs=%v\n", f.ServerName, f.Count, f.RunIDs) + } +} + +func renderLogsCompactMissingTools(missingTools []MissingToolSummary) { + if len(missingTools) == 0 { + return } + fmt.Fprintln(os.Stdout, "[missing-tools]") + for _, mt := range missingTools { + fmt.Fprintf(os.Stdout, "%s count=%d runs=%v\n", mt.Tool, mt.Count, mt.RunIDs) + } +} - // [hint] — dynamic artifact hint + static usage guidance rendered as a single line +func renderLogsCompactHint(data LogsData) { hint := "use --json for full details, -v for verbose, --format console for tables" if data.Message != "" { hint = data.Message + " " + hint @@ -225,9 +265,17 @@ func renderLogsCompact(data LogsData) { func renderLogsCompactVerbose(data LogsData) { logsCompactLog.Printf("Rendering %d runs in verbose compact format", data.Summary.TotalRuns) - s := data.Summary + renderLogsCompactVerboseSummary(data.Summary) - // [summary] extended + if len(data.Runs) == 0 { + return + } + + renderLogsCompactVerboseRuns(data.Runs) + renderLogsCompactVerboseSections(data) +} + +func renderLogsCompactVerboseSummary(s LogsSummary) { summaryParts := []string{ "runs=" + strconv.Itoa(s.TotalRuns), "duration=" + s.TotalDuration, @@ -242,50 +290,20 @@ func renderLogsCompactVerbose(data LogsData) { if s.TotalAIC > 0 { summaryParts = append(summaryParts, "aic="+formatCompactAIC(s.TotalAIC)) } - if len(s.EngineCounts) > 0 { - parts := make([]string, 0, len(s.EngineCounts)) - for engine, count := range s.EngineCounts { - parts = append(parts, engine+":"+strconv.Itoa(count)) - } - summaryParts = append(summaryParts, "engines="+strings.Join(parts, ",")) - } - if s.OutcomeAccepted > 0 || s.OutcomeRejected > 0 { - summaryParts = append(summaryParts, - "accepted="+strconv.Itoa(s.OutcomeAccepted), - "rejected="+strconv.Itoa(s.OutcomeRejected), - "ignored="+strconv.Itoa(s.OutcomeIgnored), - "pending="+strconv.Itoa(s.OutcomePending), - ) - if s.OutcomeAcceptanceRate > 0 { - summaryParts = append(summaryParts, "acceptance="+fmt.Sprintf("%.0f%%", s.OutcomeAcceptanceRate*100)) - } - if s.OutcomeWasteRate > 0 { - summaryParts = append(summaryParts, "waste="+fmt.Sprintf("%.0f%%", s.OutcomeWasteRate*100)) - } - } + renderLogsCompactAppendEngineCounts(&summaryParts, s.EngineCounts) + renderLogsCompactAppendOutcome(&summaryParts, s, true) fmt.Fprintf(os.Stdout, "[summary] %s\n", strings.Join(summaryParts, " ")) +} - if len(data.Runs) == 0 { - return - } - - // [runs] verbose aligned table +func renderLogsCompactVerboseRuns(runs []RunData) { fmt.Fprintln(os.Stdout, "[runs]") w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) fmt.Fprintln(w, "RUNID\tWORKFLOW\tENGINE\tSTATUS\tDUR\tTOKENS\tAIC\tTURNS\tERR\tWARN\tEVENT\tACTOR\tTBT\tCLASS\tCREATED\tBRANCH") - - for _, r := range data.Runs { - status := r.Conclusion - if status == "" { - status = r.Status - } + for _, r := range runs { + status, dur, actor := renderLogsCompactRunFields(r) if status == "skipped" || status == "cancelled" { continue } - dur := r.Duration - if dur == "" { - dur = "-" - } tbt := r.AvgTimeBetweenTurns if tbt == "" { tbt = "-" @@ -294,12 +312,7 @@ func renderLogsCompactVerbose(data LogsData) { if classification == "" { classification = "-" } - actor := r.Actor - if actor == "" { - actor = "-" - } wfID := workflowIDFromRun(r.WorkflowPath, r.WorkflowName) - fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%d\t%s\t%d\t%d\t%d\t%s\t%s\t%s\t%s\t%s\t%s\n", r.RunID, wfID, r.EngineID, status, dur, r.TokenUsage, formatCompactAIC(r.AIC), @@ -308,79 +321,81 @@ func renderLogsCompactVerbose(data LogsData) { r.CreatedAt.Format("01-02 15:04"), r.Branch) } w.Flush() +} - // [errors] - if len(data.ErrorsAndWarnings) > 0 { - fmt.Fprintln(os.Stdout, "[errors]") - for _, ew := range data.ErrorsAndWarnings { - fmt.Fprintf(os.Stdout, "%s run=%d count=%d: %s\n", ew.Type, ew.RunID, ew.Count, ew.Message) - } +func renderLogsCompactVerboseSections(data LogsData) { + renderLogsCompactVerboseErrors(data.ErrorsAndWarnings) + renderLogsCompactVerboseInsights(data.Observability) + renderLogsCompactVerboseFirewall(data.FirewallLog) + renderLogsCompactVerboseTools(data.ToolUsage) + renderLogsCompactVerboseMCPTools(data.MCPToolUsage) + renderLogsCompactMCPFailures(data.MCPFailures) + renderLogsCompactMissingTools(data.MissingTools) + renderLogsCompactVerboseEpisodes(data.Episodes) + if data.LogsLocation != "" { + fmt.Fprintf(os.Stdout, "[location] %s\n", data.LogsLocation) } +} - // [insights] — all severities in verbose mode - if len(data.Observability) > 0 { - fmt.Fprintln(os.Stdout, "[insights]") - for _, obs := range data.Observability { - fmt.Fprintf(os.Stdout, "[%s] %s: %s\n", obs.Severity, obs.Title, obs.Summary) - } +func renderLogsCompactVerboseErrors(errorsAndWarnings []ErrorSummary) { + if len(errorsAndWarnings) == 0 { + return } - - // [firewall] — full breakdown - if data.FirewallLog != nil && data.FirewallLog.TotalRequests > 0 { - fw := data.FirewallLog - fmt.Fprintf(os.Stdout, "[firewall] requests=%d allowed=%d blocked=%d\n", - fw.TotalRequests, fw.AllowedRequests, fw.BlockedRequests) - if len(fw.RequestsByDomain) > 0 { - for domain, counts := range fw.RequestsByDomain { - fmt.Fprintf(os.Stdout, " %s allowed=%d blocked=%d\n", domain, counts.Allowed, counts.Blocked) - } - } + fmt.Fprintln(os.Stdout, "[errors]") + for _, ew := range errorsAndWarnings { + fmt.Fprintf(os.Stdout, "%s run=%d count=%d: %s\n", ew.Type, ew.RunID, ew.Count, ew.Message) } +} - // [tools] - if len(data.ToolUsage) > 0 { - fmt.Fprintln(os.Stdout, "[tools]") - for _, t := range data.ToolUsage { - fmt.Fprintf(os.Stdout, "%s calls=%d runs=%d\n", t.Name, t.TotalCalls, t.Runs) - } +func renderLogsCompactVerboseInsights(observability []ObservabilityInsight) { + if len(observability) == 0 { + return } - - // [mcp-tools] - if data.MCPToolUsage != nil && len(data.MCPToolUsage.Summary) > 0 { - fmt.Fprintln(os.Stdout, "[mcp-tools]") - for _, t := range data.MCPToolUsage.Summary { - fmt.Fprintf(os.Stdout, "%s.%s calls=%d\n", t.ServerName, t.ToolName, t.CallCount) - } + fmt.Fprintln(os.Stdout, "[insights]") + for _, obs := range observability { + fmt.Fprintf(os.Stdout, "[%s] %s: %s\n", obs.Severity, obs.Title, obs.Summary) } +} - // [mcp-failures] - if len(data.MCPFailures) > 0 { - fmt.Fprintln(os.Stdout, "[mcp-failures]") - for _, f := range data.MCPFailures { - fmt.Fprintf(os.Stdout, "server=%s count=%d runs=%v\n", f.ServerName, f.Count, f.RunIDs) +func renderLogsCompactVerboseFirewall(fw *FirewallLogSummary) { + if fw == nil || fw.TotalRequests == 0 { + return + } + fmt.Fprintf(os.Stdout, "[firewall] requests=%d allowed=%d blocked=%d\n", fw.TotalRequests, fw.AllowedRequests, fw.BlockedRequests) + if len(fw.RequestsByDomain) > 0 { + for domain, counts := range fw.RequestsByDomain { + fmt.Fprintf(os.Stdout, " %s allowed=%d blocked=%d\n", domain, counts.Allowed, counts.Blocked) } } +} - // [missing-tools] - if len(data.MissingTools) > 0 { - fmt.Fprintln(os.Stdout, "[missing-tools]") - for _, mt := range data.MissingTools { - fmt.Fprintf(os.Stdout, "%s count=%d runs=%v\n", mt.Tool, mt.Count, mt.RunIDs) - } +func renderLogsCompactVerboseTools(toolUsage []ToolUsageSummary) { + if len(toolUsage) == 0 { + return + } + fmt.Fprintln(os.Stdout, "[tools]") + for _, t := range toolUsage { + fmt.Fprintf(os.Stdout, "%s calls=%d runs=%d\n", t.Name, t.TotalCalls, t.Runs) } +} - // [episodes] - if len(data.Episodes) > 0 { - fmt.Fprintln(os.Stdout, "[episodes]") - for _, ep := range data.Episodes { - fmt.Fprintf(os.Stdout, "%s runs=%d conf=%s duration=%s\n", - ep.Kind, ep.TotalRuns, ep.Confidence, ep.TotalDuration) - } +func renderLogsCompactVerboseMCPTools(mcpToolUsage *MCPToolUsageSummary) { + if mcpToolUsage == nil || len(mcpToolUsage.Summary) == 0 { + return + } + fmt.Fprintln(os.Stdout, "[mcp-tools]") + for _, t := range mcpToolUsage.Summary { + fmt.Fprintf(os.Stdout, "%s.%s calls=%d\n", t.ServerName, t.ToolName, t.CallCount) } +} - // [location] - if data.LogsLocation != "" { - fmt.Fprintf(os.Stdout, "[location] %s\n", data.LogsLocation) +func renderLogsCompactVerboseEpisodes(episodes []EpisodeData) { + if len(episodes) == 0 { + return + } + fmt.Fprintln(os.Stdout, "[episodes]") + for _, ep := range episodes { + fmt.Fprintf(os.Stdout, "%s runs=%d conf=%s duration=%s\n", ep.Kind, ep.TotalRuns, ep.Confidence, ep.TotalDuration) } } diff --git a/pkg/cli/logs_format_tsv.go b/pkg/cli/logs_format_tsv.go index 20768dde1ce..3140b8ba25e 100644 --- a/pkg/cli/logs_format_tsv.go +++ b/pkg/cli/logs_format_tsv.go @@ -48,60 +48,64 @@ func renderLogsTSV(data LogsData) { } fmt.Fprintln(os.Stdout, strings.Join(headers, "\t")) - // Rows for _, r := range data.Runs { - conclusion := r.Conclusion - if conclusion == "" { - conclusion = r.Status - } - duration := r.Duration - if duration == "" { - duration = "-" - } - classification := r.Classification - if classification == "" { - classification = "-" - } - // Shorten URL to just the run path for density - url := r.URL - if idx := strings.Index(url, "/actions/runs/"); idx > 0 { - url = url[idx:] - } + fmt.Fprintln(os.Stdout, strings.Join(renderLogsTSVFields(r), "\t")) + } - fields := []string{ - strconv.FormatInt(r.RunID, 10), - r.WorkflowName, - r.EngineID, - conclusion, - duration, - strconv.Itoa(r.TokenUsage), - fmt.Sprintf("%.3f", r.AIC), - strconv.Itoa(r.Turns), - strconv.Itoa(r.ErrorCount), - r.Event, - r.Branch, - r.CreatedAt.Format("2006-01-02T15:04"), - classification, - url, - } - fmt.Fprintln(os.Stdout, strings.Join(fields, "\t")) + renderLogsTSVAppendices(data) +} + +func renderLogsTSVFields(r RunData) []string { + conclusion, duration, classification := renderLogsTSVRunStrings(r) + url := r.URL + if idx := strings.Index(url, "/actions/runs/"); idx > 0 { + url = url[idx:] + } + return []string{ + strconv.FormatInt(r.RunID, 10), + r.WorkflowName, + r.EngineID, + conclusion, + duration, + strconv.Itoa(r.TokenUsage), + fmt.Sprintf("%.3f", r.AIC), + strconv.Itoa(r.Turns), + strconv.Itoa(r.ErrorCount), + r.Event, + r.Branch, + r.CreatedAt.Format("2006-01-02T15:04"), + classification, + url, } +} - // Append observability insights as comments (high signal density) +func renderLogsTSVRunStrings(r RunData) (string, string, string) { + conclusion := r.Conclusion + if conclusion == "" { + conclusion = r.Status + } + duration := r.Duration + if duration == "" { + duration = "-" + } + classification := r.Classification + if classification == "" { + classification = "-" + } + return conclusion, duration, classification +} + +func renderLogsTSVAppendices(data LogsData) { if len(data.Observability) > 0 { fmt.Fprintln(os.Stdout, "# insights:") for _, obs := range data.Observability { fmt.Fprintf(os.Stdout, "# [%s] %s: %s\n", obs.Severity, obs.Title, obs.Summary) } } - - // Append firewall summary if present if data.FirewallLog != nil && data.FirewallLog.TotalRequests > 0 { fmt.Fprintf(os.Stdout, "# firewall: %d requests (%d allowed, %d blocked)\n", data.FirewallLog.TotalRequests, data.FirewallLog.AllowedRequests, data.FirewallLog.BlockedRequests) } - - // Append engine distribution if multiple engines if len(data.Summary.EngineCounts) > 1 { parts := make([]string, 0, len(data.Summary.EngineCounts)) for engine, count := range data.Summary.EngineCounts { @@ -133,55 +137,51 @@ func renderLogsTSVVerbose(data LogsData) { fmt.Fprintln(os.Stdout, strings.Join(headers, "\t")) for _, r := range data.Runs { - conclusion := r.Conclusion - if conclusion == "" { - conclusion = r.Status - } - duration := r.Duration - if duration == "" { - duration = "-" - } - tbt := r.AvgTimeBetweenTurns - if tbt == "" { - tbt = "-" - } - classification := r.Classification - if classification == "" { - classification = "-" - } - displayTitle := stringutil.Truncate(r.DisplayTitle, 50) - - fields := []string{ - strconv.FormatInt(r.RunID, 10), - r.WorkflowName, - r.EngineID, - conclusion, - duration, - strconv.Itoa(r.TokenUsage), - fmt.Sprintf("%.3f", r.AIC), - strconv.Itoa(r.Turns), - strconv.Itoa(r.ErrorCount), - strconv.Itoa(r.WarningCount), - strconv.Itoa(r.MissingToolCount), - strconv.Itoa(r.MissingDataCount), - strconv.Itoa(r.GitHubAPICalls), - r.Event, - r.Branch, - r.Actor, - r.CreatedAt.Format("2006-01-02T15:04"), - tbt, - classification, - fmt.Sprintf("%.0f", r.ActionMinutes), - displayTitle, - r.URL, - } - fmt.Fprintln(os.Stdout, strings.Join(fields, "\t")) + fmt.Fprintln(os.Stdout, strings.Join(renderLogsTSVVerboseFields(r), "\t")) } - if len(data.Observability) > 0 { - fmt.Fprintln(os.Stdout, "# insights:") - for _, obs := range data.Observability { - fmt.Fprintf(os.Stdout, "# [%s] %s: %s\n", obs.Severity, obs.Title, obs.Summary) - } + renderLogsTSVVerboseAppendices(data) +} + +func renderLogsTSVVerboseFields(r RunData) []string { + conclusion, duration, classification := renderLogsTSVRunStrings(r) + tbt := r.AvgTimeBetweenTurns + if tbt == "" { + tbt = "-" + } + displayTitle := stringutil.Truncate(r.DisplayTitle, 50) + return []string{ + strconv.FormatInt(r.RunID, 10), + r.WorkflowName, + r.EngineID, + conclusion, + duration, + strconv.Itoa(r.TokenUsage), + fmt.Sprintf("%.3f", r.AIC), + strconv.Itoa(r.Turns), + strconv.Itoa(r.ErrorCount), + strconv.Itoa(r.WarningCount), + strconv.Itoa(r.MissingToolCount), + strconv.Itoa(r.MissingDataCount), + strconv.Itoa(r.GitHubAPICalls), + r.Event, + r.Branch, + r.Actor, + r.CreatedAt.Format("2006-01-02T15:04"), + tbt, + classification, + fmt.Sprintf("%.0f", r.ActionMinutes), + displayTitle, + r.URL, + } +} + +func renderLogsTSVVerboseAppendices(data LogsData) { + if len(data.Observability) == 0 { + return + } + fmt.Fprintln(os.Stdout, "# insights:") + for _, obs := range data.Observability { + fmt.Fprintf(os.Stdout, "# [%s] %s: %s\n", obs.Severity, obs.Title, obs.Summary) } } diff --git a/pkg/cli/logs_github_api.go b/pkg/cli/logs_github_api.go index 7994475e50a..d73190d03a6 100644 --- a/pkg/cli/logs_github_api.go +++ b/pkg/cli/logs_github_api.go @@ -191,6 +191,31 @@ type ListWorkflowRunsOptions struct { // The processedCount and targetCount parameters are used to display progress in the spinner message. func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun, int, error) { logsGitHubAPILog.Printf("Listing workflow runs: workflow=%s, limit=%d, startDate=%s, endDate=%s, ref=%s", opts.WorkflowName, opts.Limit, opts.StartDate, opts.EndDate, opts.Ref) + args := listWorkflowRunsWithPaginationArgs(opts) + output, err := listWorkflowRunsWithPaginationRunCommand(opts, args) + if err != nil { + return nil, 0, listWorkflowRunsWithPaginationCommandError(opts, args, output, err) + } + + var runs []WorkflowRun + if err := json.Unmarshal(output, &runs); err != nil { + return nil, 0, fmt.Errorf("failed to parse workflow runs: %w", err) + } + + totalFetched := len(runs) + listWorkflowRunsWithPaginationStoreOldest(opts, runs) + + agenticRuns, err := listWorkflowRunsWithPaginationAgenticRuns(opts, runs) + if err != nil { + return nil, 0, err + } + agenticRuns = listWorkflowRunsWithPaginationFilterRunIDs(opts, agenticRuns) + agenticRuns = listWorkflowRunsWithPaginationFilterUsefulRuns(agenticRuns) + + return agenticRuns, totalFetched, nil +} + +func listWorkflowRunsWithPaginationArgs(opts ListWorkflowRunsOptions) []string { args := []string{"run", "list", "--json", "databaseId,number,url,status,conclusion,workflowName,createdAt,startedAt,updatedAt,event,headBranch,headSha,displayTitle"} // Add filters @@ -224,6 +249,10 @@ func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Executing: gh "+strings.Join(args, " "))) } + return args +} + +func listWorkflowRunsWithPaginationRunCommand(opts ListWorkflowRunsOptions, args []string) ([]byte, error) { // Start spinner for network operation spinnerMsg := workflowRunsSpinnerMessage(opts) spinner := console.NewSpinner(spinnerMsg) @@ -238,12 +267,15 @@ func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun cmd := workflow.ExecGHContext(cmdCtx, args...) output, err := cmd.CombinedOutput() - if err != nil { - // Stop spinner on error - if !opts.Verbose { - spinner.Stop() - } + if !opts.Verbose { + spinner.Stop() + } + + return output, err +} +func listWorkflowRunsWithPaginationCommandError(opts ListWorkflowRunsOptions, args []string, output []byte, err error) error { + if err != nil { // Extract detailed error information including exit code var exitCode int var exitErr *exec.ExitError @@ -275,7 +307,7 @@ func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun strings.Contains(combinedMsgLower, "unknown json") || strings.Contains(combinedMsgLower, "field not found") || strings.Contains(combinedMsgLower, "no such field") { - return nil, 0, fmt.Errorf("invalid field in JSON query (exit code %d): %s", exitCode, string(output)) + return fmt.Errorf("invalid field in JSON query (exit code %d): %s", exitCode, string(output)) } // Check for authentication errors. @@ -283,29 +315,18 @@ func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun // errors (e.g. unsupported JSON fields), so matching it caused misleading // "authentication required" messages for unrelated failures. if isPermissionErrorStr(combinedMsg) { - return nil, 0, errors.New("GitHub CLI authentication required. Run 'gh auth login' first") + return errors.New("GitHub CLI authentication required. Run 'gh auth login' first") } if len(output) > 0 { - return nil, 0, fmt.Errorf("failed to list workflow runs (exit code %d): %s", exitCode, string(output)) + return fmt.Errorf("failed to list workflow runs (exit code %d): %s", exitCode, string(output)) } - return nil, 0, fmt.Errorf("failed to list workflow runs (exit code %d): %w", exitCode, err) - } - - var runs []WorkflowRun - if err := json.Unmarshal(output, &runs); err != nil { - // Stop spinner on parse error - if !opts.Verbose { - spinner.Stop() - } - return nil, 0, fmt.Errorf("failed to parse workflow runs: %w", err) - } - - // Stop spinner silently - don't show per-iteration messages - if !opts.Verbose { - spinner.Stop() + return fmt.Errorf("failed to list workflow runs (exit code %d): %w", exitCode, err) } + return nil +} +func listWorkflowRunsWithPaginationStoreOldest(opts ListWorkflowRunsOptions, runs []WorkflowRun) { // Store the total count fetched from API before filtering totalFetched := len(runs) if opts.OldestFetchedCreatedAt != nil { @@ -315,7 +336,9 @@ func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun } *opts.OldestFetchedCreatedAt = oldest } +} +func listWorkflowRunsWithPaginationAgenticRuns(opts ListWorkflowRunsOptions, runs []WorkflowRun) ([]WorkflowRun, error) { // Filter only agentic workflow runs when no specific workflow is specified // If a workflow name was specified, we already filtered by it in the API call var agenticRuns []WorkflowRun @@ -324,7 +347,7 @@ func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun // Get the list of agentic workflow names from .lock.yml files agenticWorkflowNames, err := getAgenticWorkflowNames(opts.Verbose) if err != nil { - return nil, 0, fmt.Errorf("failed to get agentic workflow names: %w", err) + return nil, fmt.Errorf("failed to get agentic workflow names: %w", err) } for _, run := range runs { @@ -336,7 +359,10 @@ func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun // Specific workflow requested, return all runs (they're already filtered by GitHub API) agenticRuns = runs } + return agenticRuns, nil +} +func listWorkflowRunsWithPaginationFilterRunIDs(opts ListWorkflowRunsOptions, agenticRuns []WorkflowRun) []WorkflowRun { // Apply run ID filtering if specified if opts.BeforeRunID > 0 || opts.AfterRunID > 0 { var filteredRuns []WorkflowRun @@ -353,7 +379,10 @@ func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun } agenticRuns = filteredRuns } + return agenticRuns +} +func listWorkflowRunsWithPaginationFilterUsefulRuns(agenticRuns []WorkflowRun) []WorkflowRun { // Filter out skipped and cancelled runs — they carry no useful agentic data // and should not count toward the requested run count. { @@ -366,8 +395,7 @@ func listWorkflowRunsWithPagination(opts ListWorkflowRunsOptions) ([]WorkflowRun } agenticRuns = filtered } - - return agenticRuns, totalFetched, nil + return agenticRuns } func workflowRunsSpinnerMessage(opts ListWorkflowRunsOptions) string { diff --git a/pkg/cli/logs_github_rate_limit_usage.go b/pkg/cli/logs_github_rate_limit_usage.go index c922851386a..1601d3e43a0 100644 --- a/pkg/cli/logs_github_rate_limit_usage.go +++ b/pkg/cli/logs_github_rate_limit_usage.go @@ -121,6 +121,19 @@ func findGitHubRateLimitsFile(runDir string) string { // parseGitHubRateLimitsFile reads and parses the github_rate_limits.jsonl file, // returning an aggregated GitHubRateLimitUsage summary. +type parseGitHubRateLimitsFileResourceState struct { + requestsMade int + firstRemaining int + lastRemaining int + firstUsed int + lastUsed int + firstSnapshotUsed int + lastSnapshotUsed int + snapshotCount int + limit int + firstEntrySet bool +} + func parseGitHubRateLimitsFile(filePath string) (*GitHubRateLimitUsage, error) { gitHubRateLimitUsageLog.Printf("Parsing GitHub rate limits file: %s", filePath) @@ -130,24 +143,24 @@ func parseGitHubRateLimitsFile(filePath string) (*GitHubRateLimitUsage, error) { } defer file.Close() - type resourceState struct { - requestsMade int - firstRemaining int - lastRemaining int - firstUsed int - lastUsed int - firstSnapshotUsed int - lastSnapshotUsed int - snapshotCount int - limit int - firstEntrySet bool + byResource, totalRequests, err := parseGitHubRateLimitsFileScan(file) + if err != nil { + return nil, err + } + if len(byResource) == 0 && totalRequests == 0 { + // File was empty or had no parseable entries + return nil, nil } - byResource := make(map[string]*resourceState) - totalRequests := 0 + return parseGitHubRateLimitsFileBuildUsage(byResource, totalRequests), nil +} + +func parseGitHubRateLimitsFileScan(file *os.File) (map[string]*parseGitHubRateLimitsFileResourceState, int, error) { + byResource := make(map[string]*parseGitHubRateLimitsFileResourceState) scanner := bufio.NewScanner(file) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + totalRequests := 0 lineNum := 0 for scanner.Scan() { lineNum++ @@ -162,102 +175,82 @@ func parseGitHubRateLimitsFile(filePath string) (*GitHubRateLimitUsage, error) { continue } - resource := entry.Resource - if resource == "" { - resource = "core" - } - - // Only count actual API calls (not rate-limit-API snapshots) toward request totals - switch entry.Source { - case "response_headers": + if parseGitHubRateLimitsFileApplyEntry(byResource, entry) { totalRequests++ - - state, ok := byResource[resource] - if !ok { - state = &resourceState{} - byResource[resource] = state - } - state.requestsMade++ - - if !state.firstEntrySet { - state.firstRemaining = entry.Remaining - state.firstUsed = entry.Used - state.firstEntrySet = true - } - // Always update last values - state.lastRemaining = entry.Remaining - state.lastUsed = entry.Used - if entry.Limit > 0 { - state.limit = entry.Limit - } - case "rate_limit_api": - // Use rate-limit API snapshots to fill in limit and remaining, and to - // derive core_consumed via firstSnapshotUsed/lastSnapshotUsed when no - // response-header entries are present for this resource. - state, ok := byResource[resource] - if !ok { - state = &resourceState{} - byResource[resource] = state - } - state.snapshotCount++ - if state.snapshotCount == 1 { - state.firstSnapshotUsed = entry.Used - } - state.lastSnapshotUsed = entry.Used - if entry.Limit > 0 && state.limit == 0 { - state.limit = entry.Limit - } - // If we have no response-header entries, record the snapshot as an approximation - if !state.firstEntrySet { - state.lastRemaining = entry.Remaining - state.lastUsed = entry.Used - if entry.Limit > 0 { - state.limit = entry.Limit - } - } } } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading rate limits file: %w", err) + return nil, 0, fmt.Errorf("error reading rate limits file: %w", err) } + return byResource, totalRequests, nil +} - if len(byResource) == 0 && totalRequests == 0 { - // File was empty or had no parseable entries - return nil, nil +func parseGitHubRateLimitsFileApplyEntry(byResource map[string]*parseGitHubRateLimitsFileResourceState, entry GitHubRateLimitEntry) bool { + resource := entry.Resource + if resource == "" { + resource = "core" + } + state, ok := byResource[resource] + if !ok { + state = &parseGitHubRateLimitsFileResourceState{} + byResource[resource] = state + } + // Only count actual API calls (not rate-limit-API snapshots) toward request totals + switch entry.Source { + case "response_headers": + parseGitHubRateLimitsFileApplyResponseHeaders(state, entry) + return true + case "rate_limit_api": + parseGitHubRateLimitsFileApplySnapshot(state, entry) } + return false +} + +func parseGitHubRateLimitsFileApplyResponseHeaders(state *parseGitHubRateLimitsFileResourceState, entry GitHubRateLimitEntry) { + state.requestsMade++ + if !state.firstEntrySet { + state.firstRemaining = entry.Remaining + state.firstUsed = entry.Used + state.firstEntrySet = true + } + // Always update last values + state.lastRemaining = entry.Remaining + state.lastUsed = entry.Used + if entry.Limit > 0 { + state.limit = entry.Limit + } +} +func parseGitHubRateLimitsFileApplySnapshot(state *parseGitHubRateLimitsFileResourceState, entry GitHubRateLimitEntry) { + // Use rate-limit API snapshots to fill in limit and remaining, and to + // derive core_consumed via firstSnapshotUsed/lastSnapshotUsed when no + // response-header entries are present for this resource. + state.snapshotCount++ + if state.snapshotCount == 1 { + state.firstSnapshotUsed = entry.Used + } + state.lastSnapshotUsed = entry.Used + if entry.Limit > 0 && state.limit == 0 { + state.limit = entry.Limit + } + // If we have no response-header entries, record the snapshot as an approximation + if !state.firstEntrySet { + state.lastRemaining = entry.Remaining + state.lastUsed = entry.Used + if entry.Limit > 0 { + state.limit = entry.Limit + } + } +} + +func parseGitHubRateLimitsFileBuildUsage(byResource map[string]*parseGitHubRateLimitsFileResourceState, totalRequests int) *GitHubRateLimitUsage { usage := &GitHubRateLimitUsage{ TotalRequestsMade: totalRequests, } for resource, state := range byResource { - // Compute quota consumed within the same rate-limit window. - // If used values suggest a window reset occurred (lastUsed < firstUsed), - // fall back to using the absolute lastUsed value as the consumption metric. - var consumed int - consumedSource := "" - if state.requestsMade > 0 { - diff := state.lastUsed - state.firstUsed - if diff >= 0 { - consumed = diff - } else { - // Window reset mid-run; use lastUsed as a lower-bound estimate - consumed = state.lastUsed - } - consumedSource = "response_headers_delta" - } else if state.snapshotCount >= 2 { - diff := state.lastSnapshotUsed - state.firstSnapshotUsed - if diff >= 0 { - consumed = diff - } else { - // Window reset across snapshots; use last snapshot used as a lower-bound estimate - consumed = state.lastSnapshotUsed - } - consumedSource = "rate_limit_api_delta" - } else if state.snapshotCount == 1 { - consumedSource = "rate_limit_api_single_snapshot" - } + consumed, consumedSource := parseGitHubRateLimitsFileConsumed(state) row := &GitHubRateLimitResourceUsage{ Resource: resource, @@ -287,7 +280,33 @@ func parseGitHubRateLimitsFile(filePath string) (*GitHubRateLimitUsage, error) { return 0 }) - return usage, nil + return usage +} + +func parseGitHubRateLimitsFileConsumed(state *parseGitHubRateLimitsFileResourceState) (int, string) { + // Compute quota consumed within the same rate-limit window. + // If used values suggest a window reset occurred (lastUsed < firstUsed), + // fall back to using the absolute lastUsed value as the consumption metric. + if state.requestsMade > 0 { + diff := state.lastUsed - state.firstUsed + if diff >= 0 { + return diff, "response_headers_delta" + } + // Window reset mid-run; use lastUsed as a lower-bound estimate + return state.lastUsed, "response_headers_delta" + } + if state.snapshotCount >= 2 { + diff := state.lastSnapshotUsed - state.firstSnapshotUsed + if diff >= 0 { + return diff, "rate_limit_api_delta" + } + // Window reset across snapshots; use last snapshot used as a lower-bound estimate + return state.lastSnapshotUsed, "rate_limit_api_delta" + } + if state.snapshotCount == 1 { + return 0, "rate_limit_api_single_snapshot" + } + return 0, "" } // analyzeGitHubRateLimits locates and parses the github_rate_limits.jsonl file within diff --git a/pkg/cli/logs_metrics.go b/pkg/cli/logs_metrics.go index 99affedd258..86749cb1cdb 100644 --- a/pkg/cli/logs_metrics.go +++ b/pkg/cli/logs_metrics.go @@ -34,6 +34,11 @@ var logsMetricsLog = logger.New("cli:logs_metrics") // extractJSONMetrics is available as an alias var extractJSONMetrics = workflow.ExtractJSONMetrics +type logsMetricsSafeOutputItems struct { + Items []json.RawMessage `json:"items"` + Errors []string `json:"errors,omitempty"` +} + func buildReportProvenance(run WorkflowRun, timestamp, experimentName, variant string) ReportProvenance { return ReportProvenance{ Timestamp: timestamp, @@ -53,6 +58,26 @@ func extractLogMetrics(logDir string, verbose bool, workflowPath ...string) (Log fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Beginning metric extraction in "+logDir)) } + isGitHubCopilotCodingAgent := extractLogMetricsDetectCopilot(logDir, verbose, workflowPath...) + detectedEngine := extractLogMetricsDetectEngine(logDir, verbose) + extractLogMetricsReportArtifacts(logDir, verbose) + + eventsJSONLParsed := extractLogMetricsParseEventsJSONL(logDir, verbose, &metrics) + var err error + if !eventsJSONLParsed { + err = extractLogMetricsWalkLogFiles(logDir, detectedEngine, isGitHubCopilotCodingAgent, verbose, &metrics) + } + + extractLogMetricsParseGateway(logDir, verbose) + + if logsMetricsLog.Enabled() { + logsMetricsLog.Printf("Metrics extraction completed: tokens=%d, cost=%.4f, turns=%d", + metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns) + } + return metrics, err +} + +func extractLogMetricsDetectCopilot(logDir string, verbose bool, workflowPath ...string) bool { // First check if this is a GitHub Copilot coding agent run (not Copilot CLI) var detector *CopilotCodingAgentDetector if len(workflowPath) > 0 && workflowPath[0] != "" { @@ -67,6 +92,10 @@ func extractLogMetrics(logDir string, verbose bool, workflowPath ...string) (Log fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Detected GitHub Copilot coding agent run, using specialized parser")) } + return isGitHubCopilotCodingAgent +} + +func extractLogMetricsDetectEngine(logDir string, verbose bool) workflow.CodingAgentEngine { // First check for aw_info.json to determine the engine var detectedEngine workflow.CodingAgentEngine infoFilePath := filepath.Join(logDir, "aw_info.json") @@ -100,7 +129,10 @@ func extractLogMetrics(logDir string, verbose bool, workflowPath ...string) (Log } } } + return detectedEngine +} +func extractLogMetricsReportArtifacts(logDir string, verbose bool) { // Check for safe_output.jsonl artifact file awOutputPath := filepath.Join(logDir, "safe_output.jsonl") if fileutil.FileExists(awOutputPath) { @@ -153,12 +185,12 @@ func extractLogMetrics(logDir string, verbose bool, workflowPath ...string) (Log } } } +} +func extractLogMetricsParseEventsJSONL(logDir string, verbose bool, metrics *LogMetrics) bool { // Try events.jsonl first – it provides a precise, structured event list from the Copilot CLI // session state and is the most reliable source for tool calls, turns, and usage metrics. // Fall back to walking .log files if events.jsonl is not present or cannot be parsed. - var err error - eventsJSONLParsed := false if eventsJSONLPath := findEventsJSONLFile(logDir); eventsJSONLPath != "" { if verbose { fileInfo, statErr := os.Stat(eventsJSONLPath) @@ -170,10 +202,10 @@ func extractLogMetrics(logDir string, verbose bool, workflowPath ...string) (Log } eventsMetrics, eventsErr := parseEventsJSONLMetrics(eventsJSONLPath, verbose) if eventsErr == nil { - metrics = eventsMetrics - eventsJSONLParsed = true + *metrics = eventsMetrics logsMetricsLog.Printf("events.jsonl parsed: turns=%d tokens=%d toolCalls=%d", metrics.Turns, metrics.TokenUsage, len(metrics.ToolCalls)) + return true } else { logsMetricsLog.Printf("Failed to parse events.jsonl, falling back to log files: %v", eventsErr) if verbose { @@ -181,58 +213,70 @@ func extractLogMetrics(logDir string, verbose bool, workflowPath ...string) (Log } } } + return false +} +func extractLogMetricsWalkLogFiles(logDir string, detectedEngine workflow.CodingAgentEngine, isGitHubCopilotCodingAgent, verbose bool, metrics *LogMetrics) error { // Walk through all .log files when events.jsonl was not available or failed to parse - if !eventsJSONLParsed { - err = filepath.Walk(logDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // Skip directories. workflow-logs/ is explicitly excluded because it contains - // GitHub Actions runner captures of each job/step's stdout rather than the agent - // artifact data. Parsing those files would double-count token usage and turns; - // the same agent session output appears in both the agent artifact - // (e.g. agent-stdio.log) and the workflow run logs (workflow-logs/). - if info.IsDir() { - if info.Name() == "workflow-logs" { - logsMetricsLog.Printf("Skipping workflow-logs directory (GHA runner logs, not agent metrics): %s", path) - return filepath.SkipDir - } - return nil - } + return filepath.Walk(logDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if extractLogMetricsShouldSkipDir(path, info) { + return filepath.SkipDir + } + if info.IsDir() { + return nil + } + extractLogMetricsAddFileMetrics(path, info, detectedEngine, isGitHubCopilotCodingAgent, verbose, metrics) + return nil + }) +} - // Process log files - exclude output artifacts like aw_output.txt and agent_output.json - fileName := strings.ToLower(info.Name()) - if (strings.HasSuffix(fileName, ".log") || - (strings.HasSuffix(fileName, ".txt") && strings.Contains(fileName, "log"))) && - !strings.Contains(fileName, "aw_output") && - fileName != constants.AgentOutputFilename { - - fileMetrics, err := parseLogFileWithEngine(path, detectedEngine, isGitHubCopilotCodingAgent, verbose) - if err != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse log file %s: %v", path, err))) - return nil // Continue processing other files - } +func extractLogMetricsShouldSkipDir(path string, info os.FileInfo) bool { + // Skip directories. workflow-logs/ is explicitly excluded because it contains + // GitHub Actions runner captures of each job/step's stdout rather than the agent + // artifact data. Parsing those files would double-count token usage and turns; + // the same agent session output appears in both the agent artifact + // (e.g. agent-stdio.log) and the workflow run logs (workflow-logs/). + if info.IsDir() && info.Name() == "workflow-logs" { + logsMetricsLog.Printf("Skipping workflow-logs directory (GHA runner logs, not agent metrics): %s", path) + return true + } + return false +} - // Aggregate metrics - metrics.TokenUsage += fileMetrics.TokenUsage - metrics.EstimatedCost += fileMetrics.EstimatedCost - if fileMetrics.Turns > metrics.Turns { - // For turns, take the maximum rather than summing, since turns represent - // the total conversation turns for the entire workflow run - metrics.Turns = fileMetrics.Turns - } +func extractLogMetricsAddFileMetrics(path string, info os.FileInfo, detectedEngine workflow.CodingAgentEngine, isGitHubCopilotCodingAgent, verbose bool, metrics *LogMetrics) { + // Process log files - exclude output artifacts like aw_output.txt and agent_output.json + fileName := strings.ToLower(info.Name()) + if (!strings.HasSuffix(fileName, ".log") && + !(strings.HasSuffix(fileName, ".txt") && strings.Contains(fileName, "log"))) || + strings.Contains(fileName, "aw_output") || + fileName == constants.AgentOutputFilename { + return + } - // Aggregate tool sequences and tool calls - metrics.ToolSequences = append(metrics.ToolSequences, fileMetrics.ToolSequences...) - metrics.ToolCalls = append(metrics.ToolCalls, fileMetrics.ToolCalls...) - } + fileMetrics, err := parseLogFileWithEngine(path, detectedEngine, isGitHubCopilotCodingAgent, verbose) + if err != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse log file %s: %v", path, err))) + return // Continue processing other files + } - return nil - }) + // Aggregate metrics + metrics.TokenUsage += fileMetrics.TokenUsage + metrics.EstimatedCost += fileMetrics.EstimatedCost + if fileMetrics.Turns > metrics.Turns { + // For turns, take the maximum rather than summing, since turns represent + // the total conversation turns for the entire workflow run + metrics.Turns = fileMetrics.Turns } + // Aggregate tool sequences and tool calls + metrics.ToolSequences = append(metrics.ToolSequences, fileMetrics.ToolSequences...) + metrics.ToolCalls = append(metrics.ToolCalls, fileMetrics.ToolCalls...) +} + +func extractLogMetricsParseGateway(logDir string, verbose bool) { // Try to parse gateway.jsonl if it exists gatewayMetrics, gatewayErr := parseGatewayLogs(logDir, verbose) if gatewayErr == nil && gatewayMetrics != nil { @@ -250,12 +294,6 @@ func extractLogMetrics(logDir string, verbose bool, workflowPath ...string) (Log fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse gateway.jsonl: %v", gatewayErr))) } } - - if logsMetricsLog.Enabled() { - logsMetricsLog.Printf("Metrics extraction completed: tokens=%d, cost=%.4f, turns=%d", - metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns) - } - return metrics, err } // ExtractLogMetricsFromRun extracts log metrics from a processed run's log directory @@ -281,11 +319,33 @@ func extractMissingToolsFromRun(runDir string, run WorkflowRun, verbose bool, ex logsMetricsLog.Printf("Extracting missing tools from run: %d", run.DatabaseID) var missingTools []MissingToolReport + agentOutputJSONPath := filepath.Join(runDir, constants.AgentOutputFilename) + resolvedAgentOutputFile := extractMissingToolsFromRunAgentOutputFile(runDir, agentOutputJSONPath, verbose) + if resolvedAgentOutputFile == "" { + logsMetricsLog.Print("No safe output artifact found") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No safe output artifact found at %s for run %d", agentOutputJSONPath, run.DatabaseID))) + } + return missingTools, nil + } + + safeOutput, ok := extractMissingToolsFromRunReadSafeOutput(resolvedAgentOutputFile, verbose) + if !ok { + return missingTools, nil // Continue processing without this file + } + + missingTools = extractMissingToolsFromRunItems(safeOutput.Items, run, verbose, experimentName, variant) + if verbose && len(missingTools) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d missing tool reports in safe output artifact for run %d", len(missingTools), run.DatabaseID))) + } + logsMetricsLog.Printf("Found %d missing tool reports", len(missingTools)) + return missingTools, nil +} + +func extractMissingToolsFromRunAgentOutputFile(runDir, agentOutputJSONPath string, verbose bool) string { // Look for the safe output artifact file that contains structured JSON with items array // This file is created by the collect_ndjson_output.cjs script during workflow execution // After artifact refactoring, the file is flattened to agent_output.json at root - agentOutputJSONPath := filepath.Join(runDir, constants.AgentOutputFilename) - // Support both new flattened form (agent_output.json) and old forms for backward compatibility: // 1. New: agent_output.json at root (after flattening) // 2. Old: agent-output directory with nested agent-output file @@ -332,78 +392,65 @@ func extractMissingToolsFromRun(runDir string, run WorkflowRun, verbose bool, ex } } } + return resolvedAgentOutputFile +} - if resolvedAgentOutputFile != "" { - // Sanitize the path to prevent path traversal attacks - cleanPath := filepath.Clean(resolvedAgentOutputFile) +func extractMissingToolsFromRunReadSafeOutput(resolvedAgentOutputFile string, verbose bool) (logsMetricsSafeOutputItems, bool) { + // Sanitize the path to prevent path traversal attacks + cleanPath := filepath.Clean(resolvedAgentOutputFile) - // Read the safe output artifact file - content, readErr := os.ReadFile(cleanPath) - if readErr != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read safe output file %s: %v", cleanPath, readErr))) - } - return missingTools, nil // Continue processing without this file + // Read the safe output artifact file + content, readErr := os.ReadFile(cleanPath) + if readErr != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read safe output file %s: %v", cleanPath, readErr))) } + return logsMetricsSafeOutputItems{}, false + } - // Parse the structured JSON output from the collect script - var safeOutput struct { - Items []json.RawMessage `json:"items"` - Errors []string `json:"errors,omitempty"` + // Parse the structured JSON output from the collect script + var safeOutput logsMetricsSafeOutputItems + if err := json.Unmarshal(content, &safeOutput); err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse safe output JSON from %s: %v", cleanPath, err))) } + return logsMetricsSafeOutputItems{}, false + } + return safeOutput, true +} - if err := json.Unmarshal(content, &safeOutput); err != nil { +func extractMissingToolsFromRunItems(items []json.RawMessage, run WorkflowRun, verbose bool, experimentName, variant string) []MissingToolReport { + var missingTools []MissingToolReport + // Extract missing-tool entries from the items array + for _, itemRaw := range items { + var item struct { + Type string `json:"type"` + Tool string `json:"tool,omitempty"` + Reason string `json:"reason,omitempty"` + Alternatives string `json:"alternatives,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + } + if err := json.Unmarshal(itemRaw, &item); err != nil { if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse safe output JSON from %s: %v", cleanPath, err))) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse item from safe output: %v", err))) } - return missingTools, nil // Continue processing without this file + continue // Skip malformed items } - - // Extract missing-tool entries from the items array - for _, itemRaw := range safeOutput.Items { - var item struct { - Type string `json:"type"` - Tool string `json:"tool,omitempty"` - Reason string `json:"reason,omitempty"` - Alternatives string `json:"alternatives,omitempty"` - Timestamp string `json:"timestamp,omitempty"` - } - - if err := json.Unmarshal(itemRaw, &item); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse item from safe output: %v", err))) - } - continue // Skip malformed items + // Check if this is a missing-tool entry + if item.Type == "missing_tool" { + missingTool := MissingToolReport{ + Tool: item.Tool, + Reason: item.Reason, + Alternatives: item.Alternatives, + ReportProvenance: buildReportProvenance(run, item.Timestamp, experimentName, variant), } - - // Check if this is a missing-tool entry - if item.Type == "missing_tool" { - missingTool := MissingToolReport{ - Tool: item.Tool, - Reason: item.Reason, - Alternatives: item.Alternatives, - ReportProvenance: buildReportProvenance(run, item.Timestamp, experimentName, variant), - } - missingTools = append(missingTools, missingTool) - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found missing_tool entry: %s (%s)", item.Tool, item.Reason))) - } + missingTools = append(missingTools, missingTool) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found missing_tool entry: %s (%s)", item.Tool, item.Reason))) } } - - if verbose && len(missingTools) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d missing tool reports in safe output artifact for run %d", len(missingTools), run.DatabaseID))) - } - logsMetricsLog.Printf("Found %d missing tool reports", len(missingTools)) - } else { - logsMetricsLog.Print("No safe output artifact found") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No safe output artifact found at %s for run %d", agentOutputJSONPath, run.DatabaseID))) - } } - - return missingTools, nil + return missingTools } // extractNoopsFromRun extracts noop messages from a workflow run's artifacts. @@ -413,11 +460,33 @@ func extractNoopsFromRun(runDir string, run WorkflowRun, verbose bool, experimen logsMetricsLog.Printf("Extracting noops from run: %d", run.DatabaseID) var noops []NoopReport + agentOutputJSONPath := filepath.Join(runDir, constants.AgentOutputFilename) + resolvedAgentOutputFile := extractNoopsFromRunAgentOutputFile(runDir, agentOutputJSONPath, verbose) + if resolvedAgentOutputFile == "" { + logsMetricsLog.Print("No safe output artifact found") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No safe output artifact found at %s for run %d", agentOutputJSONPath, run.DatabaseID))) + } + return noops, nil + } + + safeOutput, ok := extractNoopsFromRunReadSafeOutput(resolvedAgentOutputFile, verbose) + if !ok { + return noops, nil // Continue processing without this file + } + + noops = extractNoopsFromRunItems(safeOutput.Items, run, verbose, experimentName, variant) + if verbose && len(noops) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d noop messages in safe output artifact for run %d", len(noops), run.DatabaseID))) + } + logsMetricsLog.Printf("Found %d noop messages", len(noops)) + return noops, nil +} + +func extractNoopsFromRunAgentOutputFile(runDir, agentOutputJSONPath string, verbose bool) string { // Look for the safe output artifact file that contains structured JSON with items array // This file is created by the collect_ndjson_output.cjs script during workflow execution // After artifact refactoring, the file is flattened to agent_output.json at root - agentOutputJSONPath := filepath.Join(runDir, constants.AgentOutputFilename) - // Support both new flattened form (agent_output.json) and old forms for backward compatibility: // 1. New: agent_output.json at root (after flattening) // 2. Old: agent-output directory with nested agent-output file @@ -464,74 +533,61 @@ func extractNoopsFromRun(runDir string, run WorkflowRun, verbose bool, experimen } } } + return resolvedAgentOutputFile +} - if resolvedAgentOutputFile != "" { - // Sanitize the path to prevent path traversal attacks - cleanPath := filepath.Clean(resolvedAgentOutputFile) +func extractNoopsFromRunReadSafeOutput(resolvedAgentOutputFile string, verbose bool) (logsMetricsSafeOutputItems, bool) { + // Sanitize the path to prevent path traversal attacks + cleanPath := filepath.Clean(resolvedAgentOutputFile) - // Read the safe output artifact file - content, readErr := os.ReadFile(cleanPath) - if readErr != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read safe output file %s: %v", cleanPath, readErr))) - } - return noops, nil // Continue processing without this file + // Read the safe output artifact file + content, readErr := os.ReadFile(cleanPath) + if readErr != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read safe output file %s: %v", cleanPath, readErr))) } + return logsMetricsSafeOutputItems{}, false + } - // Parse the structured JSON output from the collect script - var safeOutput struct { - Items []json.RawMessage `json:"items"` - Errors []string `json:"errors,omitempty"` + // Parse the structured JSON output from the collect script + var safeOutput logsMetricsSafeOutputItems + if err := json.Unmarshal(content, &safeOutput); err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse safe output JSON from %s: %v", cleanPath, err))) } + return logsMetricsSafeOutputItems{}, false + } + return safeOutput, true +} - if err := json.Unmarshal(content, &safeOutput); err != nil { +func extractNoopsFromRunItems(items []json.RawMessage, run WorkflowRun, verbose bool, experimentName, variant string) []NoopReport { + var noops []NoopReport + // Extract noop entries from the items array + for _, itemRaw := range items { + var item struct { + Type string `json:"type"` + Message string `json:"message,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + } + if err := json.Unmarshal(itemRaw, &item); err != nil { if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse safe output JSON from %s: %v", cleanPath, err))) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse item from safe output: %v", err))) } - return noops, nil // Continue processing without this file + continue // Skip malformed items } - - // Extract noop entries from the items array - for _, itemRaw := range safeOutput.Items { - var item struct { - Type string `json:"type"` - Message string `json:"message,omitempty"` - Timestamp string `json:"timestamp,omitempty"` + // Check if this is a noop entry + if item.Type == "noop" { + noop := NoopReport{ + Message: item.Message, + ReportProvenance: buildReportProvenance(run, item.Timestamp, experimentName, variant), } - - if err := json.Unmarshal(itemRaw, &item); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse item from safe output: %v", err))) - } - continue // Skip malformed items - } - - // Check if this is a noop entry - if item.Type == "noop" { - noop := NoopReport{ - Message: item.Message, - ReportProvenance: buildReportProvenance(run, item.Timestamp, experimentName, variant), - } - noops = append(noops, noop) - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found noop entry: "+item.Message)) - } + noops = append(noops, noop) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found noop entry: "+item.Message)) } } - - if verbose && len(noops) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d noop messages in safe output artifact for run %d", len(noops), run.DatabaseID))) - } - logsMetricsLog.Printf("Found %d noop messages", len(noops)) - } else { - logsMetricsLog.Print("No safe output artifact found") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No safe output artifact found at %s for run %d", agentOutputJSONPath, run.DatabaseID))) - } } - - return noops, nil + return noops } // extractMissingDataFromRun extracts missing data reports from a workflow run's artifacts. @@ -541,11 +597,33 @@ func extractMissingDataFromRun(runDir string, run WorkflowRun, verbose bool, exp logsMetricsLog.Printf("Extracting missing data from run: %d", run.DatabaseID) var missingData []MissingDataReport + agentOutputJSONPath := filepath.Join(runDir, constants.AgentOutputFilename) + resolvedAgentOutputFile := extractMissingDataFromRunAgentOutputFile(runDir, agentOutputJSONPath, verbose) + if resolvedAgentOutputFile == "" { + logsMetricsLog.Print("No safe output artifact found") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No safe output artifact found at %s for run %d", agentOutputJSONPath, run.DatabaseID))) + } + return missingData, nil + } + + safeOutput, ok := extractMissingDataFromRunReadSafeOutput(resolvedAgentOutputFile, verbose) + if !ok { + return missingData, nil // Continue processing without this file + } + + missingData = extractMissingDataFromRunItems(safeOutput.Items, run, verbose, experimentName, variant) + if verbose && len(missingData) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d missing data reports in safe output artifact for run %d", len(missingData), run.DatabaseID))) + } + logsMetricsLog.Printf("Found %d missing data reports", len(missingData)) + return missingData, nil +} + +func extractMissingDataFromRunAgentOutputFile(runDir, agentOutputJSONPath string, verbose bool) string { // Look for the safe output artifact file that contains structured JSON with items array // This file is created by the collect_ndjson_output.cjs script during workflow execution // After artifact refactoring, the file is flattened to agent_output.json at root - agentOutputJSONPath := filepath.Join(runDir, constants.AgentOutputFilename) - // Support both new flattened form (agent_output.json) and old forms for backward compatibility: // 1. New: agent_output.json at root (after flattening) // 2. Old: agent-output directory with nested agent-output file @@ -592,80 +670,67 @@ func extractMissingDataFromRun(runDir string, run WorkflowRun, verbose bool, exp } } } + return resolvedAgentOutputFile +} - if resolvedAgentOutputFile != "" { - // Sanitize the path to prevent path traversal attacks - cleanPath := filepath.Clean(resolvedAgentOutputFile) +func extractMissingDataFromRunReadSafeOutput(resolvedAgentOutputFile string, verbose bool) (logsMetricsSafeOutputItems, bool) { + // Sanitize the path to prevent path traversal attacks + cleanPath := filepath.Clean(resolvedAgentOutputFile) - // Read the safe output artifact file - content, readErr := os.ReadFile(cleanPath) - if readErr != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read safe output file %s: %v", cleanPath, readErr))) - } - return missingData, nil // Continue processing without this file + // Read the safe output artifact file + content, readErr := os.ReadFile(cleanPath) + if readErr != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read safe output file %s: %v", cleanPath, readErr))) } + return logsMetricsSafeOutputItems{}, false + } - // Parse the structured JSON output from the collect script - var safeOutput struct { - Items []json.RawMessage `json:"items"` - Errors []string `json:"errors,omitempty"` + // Parse the structured JSON output from the collect script + var safeOutput logsMetricsSafeOutputItems + if err := json.Unmarshal(content, &safeOutput); err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse safe output JSON from %s: %v", cleanPath, err))) } + return logsMetricsSafeOutputItems{}, false + } + return safeOutput, true +} - if err := json.Unmarshal(content, &safeOutput); err != nil { +func extractMissingDataFromRunItems(items []json.RawMessage, run WorkflowRun, verbose bool, experimentName, variant string) []MissingDataReport { + var missingData []MissingDataReport + // Extract missing_data entries from the items array + for _, itemRaw := range items { + var item struct { + Type string `json:"type"` + DataType string `json:"data_type,omitempty"` + Reason string `json:"reason,omitempty"` + Context string `json:"context,omitempty"` + Alternatives string `json:"alternatives,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + } + if err := json.Unmarshal(itemRaw, &item); err != nil { if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse safe output JSON from %s: %v", cleanPath, err))) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse item from safe output: %v", err))) } - return missingData, nil // Continue processing without this file - } - - // Extract missing_data entries from the items array - for _, itemRaw := range safeOutput.Items { - var item struct { - Type string `json:"type"` - DataType string `json:"data_type,omitempty"` - Reason string `json:"reason,omitempty"` - Context string `json:"context,omitempty"` - Alternatives string `json:"alternatives,omitempty"` - Timestamp string `json:"timestamp,omitempty"` - } - - if err := json.Unmarshal(itemRaw, &item); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse item from safe output: %v", err))) - } - continue // Skip malformed items + continue // Skip malformed items + } + // Check if this is a missing_data entry + if item.Type == "missing_data" { + missingDataItem := MissingDataReport{ + DataType: item.DataType, + Reason: item.Reason, + Context: item.Context, + Alternatives: item.Alternatives, + ReportProvenance: buildReportProvenance(run, item.Timestamp, experimentName, variant), } - - // Check if this is a missing_data entry - if item.Type == "missing_data" { - missingDataItem := MissingDataReport{ - DataType: item.DataType, - Reason: item.Reason, - Context: item.Context, - Alternatives: item.Alternatives, - ReportProvenance: buildReportProvenance(run, item.Timestamp, experimentName, variant), - } - missingData = append(missingData, missingDataItem) - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found missing_data entry: %s (%s)", item.DataType, item.Reason))) - } + missingData = append(missingData, missingDataItem) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found missing_data entry: %s (%s)", item.DataType, item.Reason))) } } - - if verbose && len(missingData) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d missing data reports in safe output artifact for run %d", len(missingData), run.DatabaseID))) - } - logsMetricsLog.Printf("Found %d missing data reports", len(missingData)) - } else { - logsMetricsLog.Print("No safe output artifact found") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No safe output artifact found at %s for run %d", agentOutputJSONPath, run.DatabaseID))) - } } - - return missingData, nil + return missingData } // extractMCPFailuresFromRun extracts MCP server failure reports from a workflow run's logs. diff --git a/pkg/cli/logs_orchestrator_download.go b/pkg/cli/logs_orchestrator_download.go index a221780fee4..2241a881b3e 100644 --- a/pkg/cli/logs_orchestrator_download.go +++ b/pkg/cli/logs_orchestrator_download.go @@ -204,17 +204,8 @@ func collectProcessedWorkflowRuns(runtime logsDownloadRuntime, opts LogsDownload } } logWorkflowRunBatchFound(batch, iteration, opts.Verbose) - processedRuns, batchProcessed, allRunsConsumed, timedOut := processWorkflowRunBatch(runtime.activeCtx, batch, processedRuns, processWorkflowRunBatchOptions{ - count: opts.Count, - outputDir: opts.OutputDir, - verbose: opts.Verbose, - repoOverride: opts.RepoOverride, - artifactFilter: runtime.artifactFilter, - evalsOnly: opts.EvalsOnly, - artifactSets: opts.ArtifactSets, - parse: opts.Parse, - filters: runtime.filters, - }) + batchOpts := collectProcessedWorkflowRunsBatchOptions(runtime, opts) + processedRuns, batchProcessed, allRunsConsumed, timedOut := processWorkflowRunBatch(runtime.activeCtx, batch, processedRuns, batchOpts) timeoutReached = timeoutReached || timedOut logProcessedWorkflowRunBatch(opts, runtime.fetchAllInRange, iteration, batchProcessed, len(processedRuns), opts.Verbose) if allRunsConsumed { @@ -231,6 +222,20 @@ func collectProcessedWorkflowRuns(runtime logsDownloadRuntime, opts LogsDownload return processedRuns, timeoutReached, countLimitReached, nil } +func collectProcessedWorkflowRunsBatchOptions(runtime logsDownloadRuntime, opts LogsDownloadOptions) processWorkflowRunBatchOptions { + return processWorkflowRunBatchOptions{ + count: opts.Count, + outputDir: opts.OutputDir, + verbose: opts.Verbose, + repoOverride: opts.RepoOverride, + artifactFilter: runtime.artifactFilter, + evalsOnly: opts.EvalsOnly, + artifactSets: opts.ArtifactSets, + parse: opts.Parse, + filters: runtime.filters, + } +} + func shouldStopLogsIteration(runtime logsDownloadRuntime, opts LogsDownloadOptions) (bool, bool, error) { select { case <-runtime.activeCtx.Done(): diff --git a/pkg/cli/logs_orchestrator_filters.go b/pkg/cli/logs_orchestrator_filters.go index 195d3dac165..41306b0c92a 100644 --- a/pkg/cli/logs_orchestrator_filters.go +++ b/pkg/cli/logs_orchestrator_filters.go @@ -51,102 +51,147 @@ func applyRunFilters(ctx context.Context, result DownloadResult, opts runFilterO } // Apply engine filtering if specified. - if opts.engine != "" { - engineMatches, detectedEngineID := matchEngineFilter(awInfo, awInfoErr, opts.engine) - if !engineMatches { - if detectedEngineID == "" { - detectedEngineID = "unknown" - } - logsOrchestratorLog.Printf("Skipping run %d: engine filter=%s, detected=%s", result.Run.DatabaseID, opts.engine, detectedEngineID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: engine '%s' does not match filter '%s'", result.Run.DatabaseID, detectedEngineID, opts.engine))) - } - return true - } + if applyRunFiltersEngine(result, opts, awInfo, awInfoErr, verbose) { + return true } // Apply staged filtering if --exclude-staged flag is specified. - if opts.noStaged { - var isStaged bool - if awInfoErr == nil && awInfo != nil { - isStaged = awInfo.Staged - } - if isStaged { - logsOrchestratorLog.Printf("Skipping run %d: staged workflow filtered by --exclude-staged", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow is staged (filtered out by --exclude-staged)", result.Run.DatabaseID))) - } - return true - } + if applyRunFiltersStaged(result, opts, awInfo, awInfoErr, verbose) { + return true } // Apply firewall filtering if --firewall or --no-firewall flag is specified. - if opts.firewallOnly || opts.noFirewall { - var hasFirewall bool - if awInfoErr == nil && awInfo != nil { - // Firewall is enabled if steps.firewall is non-empty (e.g. "squid"). - hasFirewall = awInfo.Steps.Firewall != "" - } - if opts.firewallOnly && !hasFirewall { - logsOrchestratorLog.Printf("Skipping run %d: no firewall detected, filtered by --firewall", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not use firewall (filtered by --firewall)", result.Run.DatabaseID))) - } - return true - } - if opts.noFirewall && hasFirewall { - logsOrchestratorLog.Printf("Skipping run %d: firewall detected, filtered by --no-firewall", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow uses firewall (filtered by --no-firewall)", result.Run.DatabaseID))) - } - return true - } + if applyRunFiltersFirewall(result, opts, awInfo, awInfoErr, verbose) { + return true } // Apply safe output type filtering if --safe-output flag is specified. - if opts.safeOutputType != "" { - hasSafeOutputType, checkErr := runContainsSafeOutputType(result.LogsPath, opts.safeOutputType, verbose) - if checkErr != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check safe output type for run %d: %v", result.Run.DatabaseID, checkErr))) - } - if !hasSafeOutputType { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: no '%s' safe output messages found", result.Run.DatabaseID, opts.safeOutputType))) - } - return true - } + if applyRunFiltersSafeOutput(result, opts, verbose) { + return true } // Apply filtered-integrity filtering if --filtered-integrity flag is specified. - if opts.filteredIntegrity { - hasFiltered, checkErr := runHasDifcFilteredItems(result.LogsPath, verbose) - if checkErr != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check DIFC filtered items for run %d: %v", result.Run.DatabaseID, checkErr))) - return true - } - if !hasFiltered { - logsOrchestratorLog.Printf("Skipping run %d: no DIFC filtered items found", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: no DIFC integrity-filtered items found in gateway logs", result.Run.DatabaseID))) - } - return true - } + if applyRunFiltersFilteredIntegrity(result, opts, verbose) { + return true } // Apply evals filtering if --evals flag is specified. - if opts.evalsOnly { - if !runHasEvals(result.LogsPath, verbose) && !ensureEvalsResultsFromBranch(ctx, result.Run, result.LogsPath, "", "", "", verbose) { - logsOrchestratorLog.Printf("Skipping run %d: no evals results found, filtered by --evals", result.Run.DatabaseID) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", result.Run.DatabaseID))) - } - return true - } + if applyRunFiltersEvals(ctx, result, opts, verbose) { + return true + } + + return false +} + +func applyRunFiltersEngine(result DownloadResult, opts runFilterOpts, awInfo *AwInfo, awInfoErr error, verbose bool) bool { + if opts.engine == "" { + return false + } + engineMatches, detectedEngineID := matchEngineFilter(awInfo, awInfoErr, opts.engine) + if engineMatches { + return false + } + if detectedEngineID == "" { + detectedEngineID = "unknown" + } + logsOrchestratorLog.Printf("Skipping run %d: engine filter=%s, detected=%s", result.Run.DatabaseID, opts.engine, detectedEngineID) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: engine '%s' does not match filter '%s'", result.Run.DatabaseID, detectedEngineID, opts.engine))) + } + return true +} + +func applyRunFiltersStaged(result DownloadResult, opts runFilterOpts, awInfo *AwInfo, awInfoErr error, verbose bool) bool { + var isStaged bool + if opts.noStaged && awInfoErr == nil && awInfo != nil { + isStaged = awInfo.Staged + } + if !isStaged { + return false + } + logsOrchestratorLog.Printf("Skipping run %d: staged workflow filtered by --exclude-staged", result.Run.DatabaseID) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow is staged (filtered out by --exclude-staged)", result.Run.DatabaseID))) } + return true +} +func applyRunFiltersFirewall(result DownloadResult, opts runFilterOpts, awInfo *AwInfo, awInfoErr error, verbose bool) bool { + if !opts.firewallOnly && !opts.noFirewall { + return false + } + var hasFirewall bool + if awInfoErr == nil && awInfo != nil { + // Firewall is enabled if steps.firewall is non-empty (e.g. "squid"). + hasFirewall = awInfo.Steps.Firewall != "" + } + if opts.firewallOnly && !hasFirewall { + logsOrchestratorLog.Printf("Skipping run %d: no firewall detected, filtered by --firewall", result.Run.DatabaseID) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not use firewall (filtered by --firewall)", result.Run.DatabaseID))) + } + return true + } + if opts.noFirewall && hasFirewall { + logsOrchestratorLog.Printf("Skipping run %d: firewall detected, filtered by --no-firewall", result.Run.DatabaseID) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow uses firewall (filtered by --no-firewall)", result.Run.DatabaseID))) + } + return true + } return false } +func applyRunFiltersSafeOutput(result DownloadResult, opts runFilterOpts, verbose bool) bool { + if opts.safeOutputType == "" { + return false + } + hasSafeOutputType, checkErr := runContainsSafeOutputType(result.LogsPath, opts.safeOutputType, verbose) + if checkErr != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check safe output type for run %d: %v", result.Run.DatabaseID, checkErr))) + } + if hasSafeOutputType { + return false + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: no '%s' safe output messages found", result.Run.DatabaseID, opts.safeOutputType))) + } + return true +} + +func applyRunFiltersFilteredIntegrity(result DownloadResult, opts runFilterOpts, verbose bool) bool { + if !opts.filteredIntegrity { + return false + } + hasFiltered, checkErr := runHasDifcFilteredItems(result.LogsPath, verbose) + if checkErr != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check DIFC filtered items for run %d: %v", result.Run.DatabaseID, checkErr))) + return true + } + if hasFiltered { + return false + } + logsOrchestratorLog.Printf("Skipping run %d: no DIFC filtered items found", result.Run.DatabaseID) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: no DIFC integrity-filtered items found in gateway logs", result.Run.DatabaseID))) + } + return true +} + +func applyRunFiltersEvals(ctx context.Context, result DownloadResult, opts runFilterOpts, verbose bool) bool { + if !opts.evalsOnly { + return false + } + if runHasEvals(result.LogsPath, verbose) || ensureEvalsResultsFromBranch(ctx, result.Run, result.LogsPath, "", "", "", verbose) { + return false + } + logsOrchestratorLog.Printf("Skipping run %d: no evals results found, filtered by --evals", result.Run.DatabaseID) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping run %d: workflow does not have evals results (filtered by --evals)", result.Run.DatabaseID))) + } + return true +} + // buildProcessedRun constructs a ProcessedRun from a DownloadResult, computing // duration, action minutes, effective tokens, and job-failure counts. func buildProcessedRun(result DownloadResult, verbose, logFailedJobs bool) ProcessedRun { diff --git a/pkg/cli/logs_orchestrator_render.go b/pkg/cli/logs_orchestrator_render.go index 5bd5a7e37ba..b3d26f3280e 100644 --- a/pkg/cli/logs_orchestrator_render.go +++ b/pkg/cli/logs_orchestrator_render.go @@ -17,12 +17,7 @@ import ( // format: JSON, console metrics table, or cross-run audit report (pretty/markdown). // continuation is optional and only set when a timeout was reached during a paginated download. func renderLogsOutput(processedRuns []ProcessedRun, opts renderLogsOutputOptions) error { - // Update MissingToolCount, MissingDataCount, and NoopCount in runs - for i := range processedRuns { - processedRuns[i].Run.MissingToolCount = len(processedRuns[i].MissingTools) - processedRuns[i].Run.MissingDataCount = len(processedRuns[i].MissingData) - processedRuns[i].Run.NoopCount = len(processedRuns[i].Noops) - } + renderLogsOutputUpdateCounts(processedRuns) // Build structured logs data logsOrchestratorLog.Printf("Building logs data from %d processed runs (continuation=%t)", len(processedRuns), opts.continuation != nil) @@ -61,88 +56,117 @@ func renderLogsOutput(processedRuns []ProcessedRun, opts renderLogsOutputOptions return nil case "markdown", "pretty": - inputs := make([]crossRunInput, 0, len(processedRuns)) - for _, pr := range processedRuns { - inputs = append(inputs, crossRunInput{ - RunID: pr.Run.DatabaseID, - WorkflowName: pr.Run.WorkflowName, - Conclusion: pr.Run.Conclusion, - Duration: pr.Run.Duration, - FirewallAnalysis: pr.FirewallAnalysis, - Metrics: LogMetrics{ - TokenUsage: pr.Run.TokenUsage, - Turns: pr.Run.Turns, - }, - MCPToolUsage: pr.MCPToolUsage, - MCPFailures: pr.MCPFailures, - ErrorCount: pr.Run.ErrorCount, - }) - } - report := buildCrossRunAuditReport(inputs) - if opts.jsonOutput { - return renderCrossRunReportJSON(report) - } - if opts.format == "pretty" { - renderCrossRunReportPretty(report) - renderLogsArtifactHint(os.Stderr, logsData.Message) - return nil - } - if opts.reportFile != "" { - if err := os.MkdirAll(filepath.Dir(opts.reportFile), constants.DirPermPublic); err != nil { - return fmt.Errorf("failed to create report file directory: %w", err) - } - f, err := os.Create(opts.reportFile) - if err != nil { - return fmt.Errorf("failed to create report file: %w", err) - } - if err := func() (retErr error) { - defer func() { - if cerr := f.Close(); cerr != nil && retErr == nil { - retErr = cerr - } - }() - renderCrossRunReportMarkdownToWriter(f, report) - return nil - }(); err != nil { - return fmt.Errorf("failed to write report file: %w", err) - } - } else { - renderCrossRunReportMarkdown(report) - } + return renderLogsOutputCrossRun(processedRuns, logsData, opts) + + case "console": + return renderLogsOutputConsole(processedRuns, logsData, opts) + } + + // Default: compact format optimised for agentic consumption + return renderLogsOutputCompact(logsData, opts) +} + +func renderLogsOutputUpdateCounts(processedRuns []ProcessedRun) { + // Update MissingToolCount, MissingDataCount, and NoopCount in runs + for i := range processedRuns { + processedRuns[i].Run.MissingToolCount = len(processedRuns[i].MissingTools) + processedRuns[i].Run.MissingDataCount = len(processedRuns[i].MissingData) + processedRuns[i].Run.NoopCount = len(processedRuns[i].Noops) + } +} + +func renderLogsOutputCrossRun(processedRuns []ProcessedRun, logsData LogsData, opts renderLogsOutputOptions) error { + report := buildCrossRunAuditReport(renderLogsOutputCrossRunInputs(processedRuns)) + if opts.jsonOutput { + return renderCrossRunReportJSON(report) + } + if opts.format == "pretty" { + renderCrossRunReportPretty(report) renderLogsArtifactHint(os.Stderr, logsData.Message) return nil + } + if opts.reportFile != "" { + if err := renderLogsOutputWriteCrossRunReport(opts.reportFile, report); err != nil { + return err + } + } else { + renderCrossRunReportMarkdown(report) + } + renderLogsArtifactHint(os.Stderr, logsData.Message) + return nil +} - case "console": - // Explicit console format: decorated tables for human reading - if opts.jsonOutput { - if err := renderLogsJSON(logsData, opts.verbose); err != nil { - return fmt.Errorf("failed to render JSON output: %w", err) - } - } else { - renderLogsConsole(logsData) - displayAggregatedGatewayMetrics(processedRuns, opts.outputDir, opts.verbose) - displayUnifiedTimeline(processedRuns, opts.verbose) - if opts.toolGraph { - generateToolGraph(processedRuns, opts.verbose) +func renderLogsOutputCrossRunInputs(processedRuns []ProcessedRun) []crossRunInput { + inputs := make([]crossRunInput, 0, len(processedRuns)) + for _, pr := range processedRuns { + inputs = append(inputs, crossRunInput{ + RunID: pr.Run.DatabaseID, + WorkflowName: pr.Run.WorkflowName, + Conclusion: pr.Run.Conclusion, + Duration: pr.Run.Duration, + FirewallAnalysis: pr.FirewallAnalysis, + Metrics: LogMetrics{ + TokenUsage: pr.Run.TokenUsage, + Turns: pr.Run.Turns, + }, + MCPToolUsage: pr.MCPToolUsage, + MCPFailures: pr.MCPFailures, + ErrorCount: pr.Run.ErrorCount, + }) + } + return inputs +} + +func renderLogsOutputWriteCrossRunReport(reportFile string, report *CrossRunAuditReport) error { + if err := os.MkdirAll(filepath.Dir(reportFile), constants.DirPermPublic); err != nil { + return fmt.Errorf("failed to create report file directory: %w", err) + } + f, err := os.Create(reportFile) + if err != nil { + return fmt.Errorf("failed to create report file: %w", err) + } + if err := func() (retErr error) { + defer func() { + if cerr := f.Close(); cerr != nil && retErr == nil { + retErr = cerr } - renderLogsArtifactHint(os.Stderr, logsData.Message) - } + }() + renderCrossRunReportMarkdownToWriter(f, report) return nil + }(); err != nil { + return fmt.Errorf("failed to write report file: %w", err) } + return nil +} - // Default: compact format optimised for agentic consumption +func renderLogsOutputConsole(processedRuns []ProcessedRun, logsData LogsData, opts renderLogsOutputOptions) error { + // Explicit console format: decorated tables for human reading if opts.jsonOutput { if err := renderLogsJSON(logsData, opts.verbose); err != nil { return fmt.Errorf("failed to render JSON output: %w", err) } } else { - if opts.verbose { - renderLogsCompactVerbose(logsData) - } else { - renderLogsCompact(logsData) + renderLogsConsole(logsData) + displayAggregatedGatewayMetrics(processedRuns, opts.outputDir, opts.verbose) + displayUnifiedTimeline(processedRuns, opts.verbose) + if opts.toolGraph { + generateToolGraph(processedRuns, opts.verbose) } + renderLogsArtifactHint(os.Stderr, logsData.Message) } + return nil +} +func renderLogsOutputCompact(logsData LogsData, opts renderLogsOutputOptions) error { + if opts.jsonOutput { + if err := renderLogsJSON(logsData, opts.verbose); err != nil { + return fmt.Errorf("failed to render JSON output: %w", err) + } + } else if opts.verbose { + renderLogsCompactVerbose(logsData) + } else { + renderLogsCompact(logsData) + } return nil } diff --git a/pkg/cli/logs_orchestrator_stdin.go b/pkg/cli/logs_orchestrator_stdin.go index 9c907502a53..8c54cc80bae 100644 --- a/pkg/cli/logs_orchestrator_stdin.go +++ b/pkg/cli/logs_orchestrator_stdin.go @@ -24,9 +24,51 @@ import ( func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) error { logsOrchestratorLog.Printf("Starting stdin log download: runs=%d, outputDir=%s", len(opts.RunURLs), opts.OutputDir) - if err := ValidateArtifactSets(opts.ArtifactSets); err != nil { + artifactFilter, stop, err := downloadWorkflowLogsFromStdinPrepare(ctx, opts) + if err != nil || stop { + return err + } + + repoInfo, err := downloadWorkflowLogsFromStdinRepoInfo(opts.RepoOverride) + if err != nil { return err } + startTime := downloadWorkflowLogsFromStdinStartTime(opts) + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Fetching metadata for %d runs from stdin...", len(opts.RunURLs)))) + } + + runs, err := downloadWorkflowLogsFromStdinFetchRuns(ctx, opts, repoInfo, startTime) + if err != nil { + return err + } + if len(runs) == 0 { + return downloadWorkflowLogsFromStdinNoRuns(opts, "No runs found. No valid runs could be loaded from the provided input.", "No valid runs could be loaded from stdin") + } + + downloadResults := downloadRunArtifactsConcurrent(ctx, runs, runArtifactsConcurrentOptions{outputDir: opts.OutputDir, verbose: opts.Verbose, maxRuns: len(runs), repoOverride: opts.RepoOverride, artifactFilter: artifactFilter, evalsOnly: opts.EvalsOnly, artifactSets: opts.ArtifactSets}) + processedRuns := downloadWorkflowLogsFromStdinProcessResults(ctx, downloadResults, opts) + if len(processedRuns) == 0 { + return downloadWorkflowLogsFromStdinNoRuns(opts, "No runs found matching the specified criteria.", "No workflow runs with artifacts found matching the specified criteria") + } + + return renderLogsOutput(processedRuns, renderLogsOutputOptions{ + outputDir: opts.OutputDir, + summaryFile: opts.SummaryFile, + format: opts.Format, + reportFile: opts.ReportFile, + jsonOutput: opts.JSONOutput, + toolGraph: opts.ToolGraph, + train: opts.Train, + verbose: opts.Verbose, + artifactFilter: artifactFilter, + }) +} + +func downloadWorkflowLogsFromStdinPrepare(ctx context.Context, opts StdinLogsOptions) ([]string, bool, error) { + if err := ValidateArtifactSets(opts.ArtifactSets); err != nil { + return nil, false, err + } artifactFilter := ResolveArtifactFilter(opts.ArtifactSets) if len(artifactFilter) > 0 { logsOrchestratorLog.Printf("Artifact filter active: %v", artifactFilter) @@ -45,36 +87,49 @@ func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) e select { case <-ctx.Done(): fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) - return ctx.Err() + return nil, false, ctx.Err() default: } if len(opts.RunURLs) == 0 { fmt.Fprintln(os.Stderr, console.FormatWarningMessage("No run IDs or URLs provided on stdin")) - return nil + return artifactFilter, true, nil } + return artifactFilter, false, nil +} + +type downloadWorkflowLogsFromStdinRepoOverrides struct { + host string + owner string + repo string +} + +func downloadWorkflowLogsFromStdinRepoInfo(repoOverride string) (downloadWorkflowLogsFromStdinRepoOverrides, error) { // Parse owner/repo (and optional GHES host) from --repo override if provided. // Accepted formats: "owner/repo" or "HOST/owner/repo". - var hostOverride, ownerOverride, repoNameOverride string - if opts.RepoOverride != "" { - parts := strings.SplitN(opts.RepoOverride, "/", 3) + var repoInfo downloadWorkflowLogsFromStdinRepoOverrides + if repoOverride != "" { + parts := strings.SplitN(repoOverride, "/", 3) switch len(parts) { case 3: // HOST/owner/repo if parts[0] == "" || parts[1] == "" || parts[2] == "" { - return fmt.Errorf("invalid repository format '%s': expected '[HOST/]owner/repo'", opts.RepoOverride) + return repoInfo, fmt.Errorf("invalid repository format '%s': expected '[HOST/]owner/repo'", repoOverride) } - hostOverride, ownerOverride, repoNameOverride = parts[0], parts[1], parts[2] + repoInfo.host, repoInfo.owner, repoInfo.repo = parts[0], parts[1], parts[2] case 2: // owner/repo if parts[0] == "" || parts[1] == "" { - return fmt.Errorf("invalid repository format '%s': expected '[HOST/]owner/repo'", opts.RepoOverride) + return repoInfo, fmt.Errorf("invalid repository format '%s': expected '[HOST/]owner/repo'", repoOverride) } - ownerOverride, repoNameOverride = parts[0], parts[1] + repoInfo.owner, repoInfo.repo = parts[0], parts[1] default: - return fmt.Errorf("invalid repository format '%s': expected '[HOST/]owner/repo'", opts.RepoOverride) + return repoInfo, fmt.Errorf("invalid repository format '%s': expected '[HOST/]owner/repo'", repoOverride) } } + return repoInfo, nil +} +func downloadWorkflowLogsFromStdinStartTime(opts StdinLogsOptions) time.Time { // Start timeout timer if specified. var startTime time.Time if opts.Timeout > 0 { @@ -83,18 +138,17 @@ func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) e fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Timeout set to %d minutes", opts.Timeout))) } } + return startTime +} - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Fetching metadata for %d runs from stdin...", len(opts.RunURLs)))) - } - +func downloadWorkflowLogsFromStdinFetchRuns(ctx context.Context, opts StdinLogsOptions, repoInfo downloadWorkflowLogsFromStdinRepoOverrides, startTime time.Time) ([]WorkflowRun, error) { // Build WorkflowRun objects by fetching metadata for each provided URL. var runs []WorkflowRun for _, rawURL := range opts.RunURLs { select { case <-ctx.Done(): fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) - return ctx.Err() + return runs, ctx.Err() default: } @@ -116,14 +170,14 @@ func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) e repo := components.Repo host := components.Host if owner == "" { - owner = ownerOverride - repo = repoNameOverride + owner = repoInfo.owner + repo = repoInfo.repo if host == "" { - host = hostOverride + host = repoInfo.host } } if owner == "" { - return fmt.Errorf("run %q does not include repository information; pass --repo owner/repo or provide full run URLs", rawURL) + return runs, fmt.Errorf("run %q does not include repository information; pass --repo owner/repo or provide full run URLs", rawURL) } run, err := fetchWorkflowRunMetadata(ctx, components.Number, owner, repo, host, opts.Verbose) @@ -133,22 +187,10 @@ func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) e } runs = append(runs, run) } + return runs, nil +} - if len(runs) == 0 { - if opts.JSONOutput { - logsData := buildLogsData([]ProcessedRun{}, opts.OutputDir, nil) - logsData.Message = "No runs found. No valid runs could be loaded from the provided input." - if err := renderLogsJSON(logsData, opts.Verbose); err != nil { - return fmt.Errorf("failed to render JSON output: %w", err) - } - } - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("No valid runs could be loaded from stdin")) - return nil - } - - // Download artifacts for all runs concurrently. - downloadResults := downloadRunArtifactsConcurrent(ctx, runs, runArtifactsConcurrentOptions{outputDir: opts.OutputDir, verbose: opts.Verbose, maxRuns: len(runs), repoOverride: opts.RepoOverride, artifactFilter: artifactFilter, evalsOnly: opts.EvalsOnly, artifactSets: opts.ArtifactSets}) - +func downloadWorkflowLogsFromStdinProcessResults(ctx context.Context, downloadResults []DownloadResult, opts StdinLogsOptions) []ProcessedRun { filters := runFilterOpts{ engine: opts.Engine, noStaged: opts.NoStaged, @@ -162,69 +204,68 @@ func DownloadWorkflowLogsFromStdin(ctx context.Context, opts StdinLogsOptions) e // Process download results applying the same filters as DownloadWorkflowLogs. var processedRuns []ProcessedRun for _, result := range downloadResults { - if result.Skipped { - if opts.Verbose && result.Error != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping run %d: %v", result.Run.DatabaseID, result.Error))) - } + processedRun, ok := downloadWorkflowLogsFromStdinProcessResult(ctx, result, filters, opts) + if !ok { continue } + processedRuns = append(processedRuns, processedRun) + } + return processedRuns +} - if result.Error != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to download artifacts for run %d: %v", result.Run.DatabaseID, result.Error))) - continue +func downloadWorkflowLogsFromStdinProcessResult(ctx context.Context, result DownloadResult, filters runFilterOpts, opts StdinLogsOptions) (ProcessedRun, bool) { + if result.Skipped { + if opts.Verbose && result.Error != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping run %d: %v", result.Run.DatabaseID, result.Error))) } + return ProcessedRun{}, false + } - if applyRunFilters(ctx, result, filters, opts.Verbose) { - continue - } + if result.Error != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to download artifacts for run %d: %v", result.Run.DatabaseID, result.Error))) + return ProcessedRun{}, false + } - processedRun := buildProcessedRun(result, opts.Verbose, false) - - if opts.Parse { - awInfoPath := filepath.Join(result.LogsPath, "aw_info.json") - detectedEngine := extractEngineFromAwInfo(awInfoPath, opts.Verbose) - if err := parseAgentLog(result.LogsPath, detectedEngine, opts.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse log for run %d: %v", processedRun.Run.DatabaseID, err))) - } else { - logMdPath := filepath.Join(result.LogsPath, "log.md") - if fileutil.FileExists(logMdPath) { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Parsed log for run %d → %s", processedRun.Run.DatabaseID, logMdPath))) - } - } - if err := parseFirewallLogs(result.LogsPath, opts.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse firewall logs for run %d: %v", processedRun.Run.DatabaseID, err))) - } else { - firewallMdPath := filepath.Join(result.LogsPath, "firewall.md") - if fileutil.FileExists(firewallMdPath) { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Parsed firewall logs for run %d → %s", processedRun.Run.DatabaseID, firewallMdPath))) - } - } - } + if applyRunFilters(ctx, result, filters, opts.Verbose) { + return ProcessedRun{}, false + } - processedRuns = append(processedRuns, processedRun) + processedRun := buildProcessedRun(result, opts.Verbose, false) + if opts.Parse { + downloadWorkflowLogsFromStdinParseRun(result, processedRun, opts.Verbose) } + return processedRun, true +} - if len(processedRuns) == 0 { - if opts.JSONOutput { - logsData := buildLogsData([]ProcessedRun{}, opts.OutputDir, nil) - logsData.Message = "No runs found matching the specified criteria." - if err := renderLogsJSON(logsData, opts.Verbose); err != nil { - return fmt.Errorf("failed to render JSON output: %w", err) - } +func downloadWorkflowLogsFromStdinParseRun(result DownloadResult, processedRun ProcessedRun, verbose bool) { + awInfoPath := filepath.Join(result.LogsPath, "aw_info.json") + detectedEngine := extractEngineFromAwInfo(awInfoPath, verbose) + if err := parseAgentLog(result.LogsPath, detectedEngine, verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse log for run %d: %v", processedRun.Run.DatabaseID, err))) + } else { + logMdPath := filepath.Join(result.LogsPath, "log.md") + if fileutil.FileExists(logMdPath) { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Parsed log for run %d → %s", processedRun.Run.DatabaseID, logMdPath))) + } + } + if err := parseFirewallLogs(result.LogsPath, verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse firewall logs for run %d: %v", processedRun.Run.DatabaseID, err))) + } else { + firewallMdPath := filepath.Join(result.LogsPath, "firewall.md") + if fileutil.FileExists(firewallMdPath) { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Parsed firewall logs for run %d → %s", processedRun.Run.DatabaseID, firewallMdPath))) } - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("No workflow runs with artifacts found matching the specified criteria")) - return nil } +} - return renderLogsOutput(processedRuns, renderLogsOutputOptions{ - outputDir: opts.OutputDir, - summaryFile: opts.SummaryFile, - format: opts.Format, - reportFile: opts.ReportFile, - jsonOutput: opts.JSONOutput, - toolGraph: opts.ToolGraph, - train: opts.Train, - verbose: opts.Verbose, - artifactFilter: artifactFilter, - }) +func downloadWorkflowLogsFromStdinNoRuns(opts StdinLogsOptions, jsonMessage, warningMessage string) error { + if opts.JSONOutput { + logsData := buildLogsData([]ProcessedRun{}, opts.OutputDir, nil) + logsData.Message = jsonMessage + if err := renderLogsJSON(logsData, opts.Verbose); err != nil { + return fmt.Errorf("failed to render JSON output: %w", err) + } + } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(warningMessage)) + return nil } diff --git a/pkg/cli/logs_parsing_firewall.go b/pkg/cli/logs_parsing_firewall.go index 353f52e65bd..4ea384235bd 100644 --- a/pkg/cli/logs_parsing_firewall.go +++ b/pkg/cli/logs_parsing_firewall.go @@ -26,50 +26,7 @@ import ( var logsParsingFirewallLog = logger.New("cli:logs_parsing_firewall") -// parseFirewallLogs runs the JavaScript firewall log parser and writes markdown to firewall.md -func parseFirewallLogs(runDir string, verbose bool) error { - logsParsingFirewallLog.Printf("Parsing firewall logs in: %s", runDir) - // Get the firewall log parser script - jsScript := workflow.GetLogParserScript("parse_firewall_logs") - if jsScript == "" { - logsParsingFirewallLog.Print("Failed to get firewall log parser script") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Failed to get firewall log parser script")) - } - return nil - } - - // Check if squid logs directory exists in the run directory - // The logs could be in several locations depending on the AWF version and artifact structure. - - logsDir, err := findFirewallLogsDir(runDir) - if err != nil { - return err - } - if logsDir == "" { - logsParsingFirewallLog.Print("No firewall logs found, skipping parsing") - // No firewall logs found - this is not an error, just skip parsing - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No firewall logs found in %s, skipping firewall log parsing", filepath.Base(runDir)))) - } - return nil - } - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found firewall logs in "+logsDir)) - } - - // Create a temporary directory for running the parser - tempDir, err := os.MkdirTemp("", "firewall_log_parser") - if err != nil { - return fmt.Errorf("failed to create temp dir: %w", err) - } - defer os.RemoveAll(tempDir) - - // Create a Node.js script that mimics the GitHub Actions environment - // The firewall parser expects logs in /tmp/gh-aw/squid-logs-{workflow}/ - // We'll set GITHUB_WORKFLOW to a value that makes the parser look in our temp directory - nodeScript := fmt.Sprintf(` +const parseFirewallLogsNodeScriptTemplate = ` const fs = require('fs'); const path = require('path'); @@ -205,8 +162,60 @@ const originalMain = function() { // Replace main() call with our custom version originalMain(); -`, logsDir, jsScript) +` + +// parseFirewallLogs runs the JavaScript firewall log parser and writes markdown to firewall.md +func parseFirewallLogs(runDir string, verbose bool) error { + logsParsingFirewallLog.Printf("Parsing firewall logs in: %s", runDir) + // Get the firewall log parser script + jsScript := workflow.GetLogParserScript("parse_firewall_logs") + if jsScript == "" { + logsParsingFirewallLog.Print("Failed to get firewall log parser script") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Failed to get firewall log parser script")) + } + return nil + } + + // Check if squid logs directory exists in the run directory + // The logs could be in several locations depending on the AWF version and artifact structure. + + logsDir, err := findFirewallLogsDir(runDir) + if err != nil { + return err + } + if logsDir == "" { + logsParsingFirewallLog.Print("No firewall logs found, skipping parsing") + // No firewall logs found - this is not an error, just skip parsing + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No firewall logs found in %s, skipping firewall log parsing", filepath.Base(runDir)))) + } + return nil + } + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found firewall logs in "+logsDir)) + } + + // Create a temporary directory for running the parser + tempDir, err := os.MkdirTemp("", "firewall_log_parser") + if err != nil { + return fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tempDir) + + nodeScript := parseFirewallLogsNodeScript(logsDir, jsScript) + return parseFirewallLogsRunNodeScript(runDir, tempDir, nodeScript) +} + +func parseFirewallLogsNodeScript(logsDir, jsScript string) string { + // Create a Node.js script that mimics the GitHub Actions environment. + // The firewall parser expects logs in /tmp/gh-aw/squid-logs-{workflow}/; + // use GITHUB_WORKFLOW to make the parser look in our temp directory. + return fmt.Sprintf(parseFirewallLogsNodeScriptTemplate, logsDir, jsScript) +} +func parseFirewallLogsRunNodeScript(runDir, tempDir, nodeScript string) error { // Write the Node.js script nodeFile := filepath.Join(tempDir, "parser.js") if err := os.WriteFile(nodeFile, []byte(nodeScript), constants.FilePermPublic); err != nil { diff --git a/pkg/cli/logs_parsing_javascript.go b/pkg/cli/logs_parsing_javascript.go index e38605da18e..3b43b699929 100644 --- a/pkg/cli/logs_parsing_javascript.go +++ b/pkg/cli/logs_parsing_javascript.go @@ -25,14 +25,63 @@ import ( var logsParsingJsLog = logger.New("cli:logs_parsing_js") +const parseAgentLogNodeScriptTemplate = ` +const fs = require('fs'); + +// Mock @actions/core for the parser +const core = { + summary: { + addRaw: function(content) { + this._content = content; + return this; + }, + write: function() { + console.log(this._content); + }, + _content: '' + }, + setFailed: function(message) { + console.error('FAILED:', message); + process.exit(1); + }, + info: function(message) { + // Silent in CLI mode + } +}; + +// Set global core for the parser scripts +global.core = core; + +// Set up environment +process.env.GH_AW_AGENT_OUTPUT = '%s'; + +// Execute the parser script +%s +` + // parseAgentLog parses agent logs and generates a markdown summary func parseAgentLog(runDir string, engine workflow.CodingAgentEngine, verbose bool) error { logsParsingJsLog.Printf("Parsing agent logs in: %s", runDir) + agentLogPath, jsScript, ok, err := parseAgentLogInputs(runDir, engine, verbose) + if err != nil || !ok { + return err + } + + tempDir, err := parseAgentLogTempDir(agentLogPath, jsScript) + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + return parseAgentLogRunNodeScript(runDir, tempDir) +} + +func parseAgentLogInputs(runDir string, engine workflow.CodingAgentEngine, verbose bool) (string, string, bool, error) { // Determine which parser script to use based on the engine if engine == nil { logsParsingJsLog.Print("No engine detected, skipping log parsing") fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No engine detected in %s, skipping log parsing", filepath.Base(runDir)))) - return nil + return "", "", false, nil } // Find the agent log file - use engine.GetLogFileForParsing() to determine location @@ -40,7 +89,7 @@ func parseAgentLog(runDir string, engine workflow.CodingAgentEngine, verbose boo if !found { logsParsingJsLog.Print("No agent log file found") fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No agent logs found in %s, skipping log parsing", filepath.Base(runDir)))) - return nil + return "", "", false, nil } logsParsingJsLog.Printf("Found agent log file: %s", agentLogPath) @@ -48,7 +97,7 @@ func parseAgentLog(runDir string, engine workflow.CodingAgentEngine, verbose boo parserScriptName := engine.GetLogParserScriptId() if parserScriptName == "" { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No log parser available for engine %s in %s, skipping", engine.GetID(), filepath.Base(runDir)))) - return nil + return "", "", false, nil } jsScript := workflow.GetLogParserScript(parserScriptName) @@ -56,22 +105,31 @@ func parseAgentLog(runDir string, engine workflow.CodingAgentEngine, verbose boo if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Failed to get log parser script "+parserScriptName)) } - return nil + return "", "", false, nil } + return agentLogPath, jsScript, true, nil +} +func parseAgentLogTempDir(agentLogPath, jsScript string) (string, error) { // Read the log content logContent, err := os.ReadFile(agentLogPath) if err != nil { - return fmt.Errorf("failed to read agent log file: %w", err) + return "", fmt.Errorf("failed to read agent log file: %w", err) } // Create a temporary directory for running the parser tempDir, err := os.MkdirTemp("", "log_parser") if err != nil { - return fmt.Errorf("failed to create temp dir: %w", err) + return "", fmt.Errorf("failed to create temp dir: %w", err) } - defer os.RemoveAll(tempDir) + if err := parseAgentLogWriteParserFiles(tempDir, logContent, jsScript); err != nil { + _ = os.RemoveAll(tempDir) + return "", err + } + return tempDir, nil +} +func parseAgentLogWriteParserFiles(tempDir string, logContent []byte, jsScript string) error { // Write the log content to a temporary file logFile := filepath.Join(tempDir, "agent.log") if err := os.WriteFile(logFile, logContent, constants.FilePermPublic); err != nil { @@ -96,47 +154,22 @@ func parseAgentLog(runDir string, engine workflow.CodingAgentEngine, verbose boo } } - // Create a Node.js script that mimics the GitHub Actions environment - nodeScript := fmt.Sprintf(` -const fs = require('fs'); - -// Mock @actions/core for the parser -const core = { - summary: { - addRaw: function(content) { - this._content = content; - return this; - }, - write: function() { - console.log(this._content); - }, - _content: '' - }, - setFailed: function(message) { - console.error('FAILED:', message); - process.exit(1); - }, - info: function(message) { - // Silent in CLI mode - } -}; - -// Set global core for the parser scripts -global.core = core; - -// Set up environment -process.env.GH_AW_AGENT_OUTPUT = '%s'; - -// Execute the parser script -%s -`, logFile, jsScript) + nodeScript := parseAgentLogNodeScript(logFile, jsScript) // Write the Node.js script nodeFile := filepath.Join(tempDir, "parser.js") if err := os.WriteFile(nodeFile, []byte(nodeScript), constants.FilePermPublic); err != nil { return fmt.Errorf("failed to write node script: %w", err) } + return nil +} + +func parseAgentLogNodeScript(logFile, jsScript string) string { + // Create a Node.js script that mimics the GitHub Actions environment + return fmt.Sprintf(parseAgentLogNodeScriptTemplate, logFile, jsScript) +} +func parseAgentLogRunNodeScript(runDir, tempDir string) error { // Execute the Node.js script cmd := exec.Command("node", "parser.js") cmd.Dir = tempDir diff --git a/pkg/cli/logs_report.go b/pkg/cli/logs_report.go index cd35d40b5c0..84eb5cd3494 100644 --- a/pkg/cli/logs_report.go +++ b/pkg/cli/logs_report.go @@ -152,280 +152,336 @@ type RunData struct { } // buildLogsData creates structured logs data from processed runs +type buildLogsDataTotals struct { + totalDuration time.Duration + totalAIC float64 + totalTokens int + totalActionMinutes float64 + totalTurns int + totalSteeringEvents int + totalErrors int + totalWarnings int + totalMissingTools int + totalMissingData int + totalSafeItems int + runsWithTemporaryIDChains int + runsWithDelegatedTempTargets int + runsWithMissingTemporaryIDMap int + runsWithInvalidTemporaryIDMap int + totalTemporaryIDMappings int + totalChainedTargets int + totalChainedFollowupActions int + totalClosedTempTargets int + totalGitHubAPICalls int + engineCounts map[string]int +} + +type buildLogsDataRunContext struct { + chainMetrics SafeOutputChainMetrics + gitHubAPICalls int + engineID string + engineName string + awContext *AwContext + awInfo *AwInfo + comparison *AuditComparisonData + ambientContext *AmbientContextMetrics +} + func buildLogsData(processedRuns []ProcessedRun, outputDir string, continuation *ContinuationData) LogsData { reportLog.Printf("Building logs data from %d processed runs", len(processedRuns)) - // Build summary - var totalDuration time.Duration - var totalAIC float64 - var totalTokens int - var totalActionMinutes float64 - var totalTurns int - var totalSteeringEvents int - var totalErrors int - var totalWarnings int - var totalMissingTools int - var totalMissingData int - var totalSafeItems int - var runsWithTemporaryIDChains int - var runsWithDelegatedTempTargets int - var runsWithMissingTemporaryIDMap int - var runsWithInvalidTemporaryIDMap int - var totalTemporaryIDMappings int - var totalChainedTargets int - var totalChainedFollowupActions int - var totalClosedTempTargets int - var totalGitHubAPICalls int // engineCounts tracks the number of runs per engine_id, sourced from aw_info.json. // This is the authoritative engine classification — do not infer engine type from // lock file contents, which contain "copilot" in allowed-domains and source paths // regardless of which engine the workflow uses. - engineCounts := make(map[string]int) + totals := buildLogsDataTotals{engineCounts: make(map[string]int)} // Build runs data // Initialize as empty slice to ensure JSON marshals to [] instead of null runs := make([]RunData, 0, len(processedRuns)) for _, pr := range processedRuns { - run := pr.Run - - if run.Duration > 0 { - totalDuration += run.Duration - } - if pr.TokenUsage != nil { - totalAIC += pr.TokenUsage.TotalAIC - } - totalTokens += run.TokenUsage - totalActionMinutes += run.ActionMinutes - totalTurns += run.Turns - if pr.TokenUsage != nil { - totalSteeringEvents += pr.TokenUsage.TotalSteeringEvents - } - totalErrors += run.ErrorCount - totalWarnings += run.WarningCount - totalMissingTools += run.MissingToolCount - totalMissingData += run.MissingDataCount - totalSafeItems += run.SafeItemsCount - - // Accumulate GitHub API call counts - var gitHubAPICalls int - if pr.GitHubRateLimitUsage != nil { - gitHubAPICalls = pr.GitHubRateLimitUsage.TotalRequestsMade - } - totalGitHubAPICalls += gitHubAPICalls - - chainMetrics := buildSafeOutputChainMetrics(run.LogsPath) - totalTemporaryIDMappings += chainMetrics.TemporaryIDMappings - totalChainedTargets += chainMetrics.ChainedTargetCount - totalChainedFollowupActions += chainMetrics.ChainedFollowupActionCount - totalClosedTempTargets += chainMetrics.ClosedTempTargetCount - if chainMetrics.ChainedTargetCount > 0 { - runsWithTemporaryIDChains++ - } - if chainMetrics.DelegatedTempTargetCount > 0 { - runsWithDelegatedTempTargets++ - } - switch chainMetrics.TemporaryIDMapStatus { - case temporaryIDMapStatusMissing: - runsWithMissingTemporaryIDMap++ - case temporaryIDMapStatusInvalid: - runsWithInvalidTemporaryIDMap++ - } - - // Extract engine ID and aw_context from aw_info.json. - engineID := "" - engineName := "" - var awContext *AwContext - var awInfo *AwInfo - awInfoPath := filepath.Join(run.LogsPath, "aw_info.json") - if info, err := parseAwInfo(awInfoPath, false); err == nil && info != nil { - awInfo = info - engineID = info.EngineID - engineName = info.EngineName - awContext = info.Context - } - if engineName == "" { - engineName = engineID - } - if awContext == nil { - awContext = pr.AwContext - } - // Accumulate engine counts from aw_info.json data (authoritative source). - if engineID != "" { - engineCounts[engineID]++ - } - - comparison := buildAuditComparisonForProcessedRuns(pr, processedRuns) - - var ambientContext *AmbientContextMetrics - if pr.TokenUsage != nil { - ambientContext = pr.TokenUsage.AmbientContext - } - - runData := RunData{ - RunID: run.DatabaseID, - Number: run.Number, - WorkflowName: run.WorkflowName, - WorkflowPath: run.WorkflowPath, - Agent: engineID, - Engine: engineName, - EngineID: engineID, - Status: run.Status, - Conclusion: run.Conclusion, - Classification: deriveRunClassification(comparison), - TokenUsage: run.TokenUsage, - AIC: 0, - AmbientContext: ambientContext, - ActionMinutes: run.ActionMinutes, - Turns: run.Turns, - ErrorCount: run.ErrorCount, - WarningCount: run.WarningCount, - MissingToolCount: run.MissingToolCount, - MissingDataCount: run.MissingDataCount, - SafeItemsCount: run.SafeItemsCount, - ManifestEntryCount: chainMetrics.ManifestEntryCount, - TemporaryIDMapStatus: chainMetrics.TemporaryIDMapStatus, - TemporaryIDMappings: chainMetrics.TemporaryIDMappings, - ChainedTargetCount: chainMetrics.ChainedTargetCount, - ChainedFollowupActionCount: chainMetrics.ChainedFollowupActionCount, - DelegatedTempTargetCount: chainMetrics.DelegatedTempTargetCount, - ClosedTempTargetCount: chainMetrics.ClosedTempTargetCount, - CreatedAt: run.CreatedAt, - StartedAt: run.StartedAt, - UpdatedAt: run.UpdatedAt, - URL: run.URL, - LogsPath: run.LogsPath, - Event: run.Event, - Branch: run.HeadBranch, - HeadSHA: run.HeadSha, - DisplayTitle: run.DisplayTitle, - Comparison: comparison, - TaskDomain: pr.TaskDomain, - BehaviorFingerprint: pr.BehaviorFingerprint, - AgenticAssessments: pr.AgenticAssessments, - AwContext: awContext, - TokenUsageSummary: pr.TokenUsage, - GitHubAPICalls: gitHubAPICalls, - Experiments: extractExperimentData(run.LogsPath), - } - if awInfo != nil { - runData.Repository = awInfo.Repository - if awInfo.Repository != "" { - if parts := strings.SplitN(awInfo.Repository, "/", 2); len(parts) == 2 { - runData.Organization = parts[0] - } - } - runData.Ref = awInfo.Ref - runData.SHA = awInfo.SHA - runData.Actor = awInfo.Actor - runData.RunAttempt = awInfo.RunAttempt - runData.TargetRepo = awInfo.TargetRepo - runData.EventName = awInfo.EventName - // Fall back to inferring the workflow path from the display name when the - // GitHub API returned an empty path (e.g. for scheduled agentic runs). - // This handles both fresh runs and old cached RunSummary entries whose - // run.WorkflowPath was persisted as empty before the fix in - // logs_run_processor.go was applied. - if runData.WorkflowPath == "" && awInfo.WorkflowName != "" { - runData.WorkflowPath = inferWorkflowPathFromDisplayName(awInfo.WorkflowName) - } - } - if run.Duration > 0 { - runData.Duration = timeutil.FormatDuration(run.Duration) - } - if pr.TokenUsage != nil && pr.TokenUsage.TotalAIC > 0 { - runData.AIC = pr.TokenUsage.TotalAIC - } - // Compute average TBT from metrics when available; fall back to wall-time / (turns - 1). - if run.AvgTimeBetweenTurns > 0 { - runData.AvgTimeBetweenTurns = timeutil.FormatDuration(run.AvgTimeBetweenTurns) - } else if run.Turns > 1 && run.Duration > 0 { - runData.AvgTimeBetweenTurns = timeutil.FormatDuration(run.Duration/time.Duration(run.Turns-1)) + " (estimated)" - } - runs = append(runs, runData) - } - - summary := LogsSummary{ - TotalRuns: len(processedRuns), - TotalDuration: timeutil.FormatDuration(totalDuration), - TotalAIC: totalAIC, - TotalTokens: totalTokens, - TotalActionMinutes: totalActionMinutes, - TotalTurns: totalTurns, - TotalSteeringEvents: totalSteeringEvents, - TotalErrors: totalErrors, - TotalWarnings: totalWarnings, - TotalMissingTools: totalMissingTools, - TotalMissingData: totalMissingData, - TotalSafeItems: totalSafeItems, - RunsWithTemporaryIDChains: runsWithTemporaryIDChains, - RunsWithDelegatedTempTargets: runsWithDelegatedTempTargets, - RunsWithMissingTemporaryIDMap: runsWithMissingTemporaryIDMap, - RunsWithInvalidTemporaryIDMap: runsWithInvalidTemporaryIDMap, - TotalTemporaryIDMappings: totalTemporaryIDMappings, - TotalChainedTargets: totalChainedTargets, - TotalChainedFollowupActions: totalChainedFollowupActions, - TotalClosedTempTargets: totalClosedTempTargets, - TotalGitHubAPICalls: totalGitHubAPICalls, - } - if len(engineCounts) > 0 { - summary.EngineCounts = engineCounts + runs = append(runs, buildLogsDataRunData(pr, processedRuns, &totals)) } + summary := buildLogsDataSummary(processedRuns, totals) episodes, edges := buildEpisodeData(runs, processedRuns) + buildLogsDataAddEpisodeCounts(&summary, episodes) + summaries := buildLogsDataBuildSummaries(processedRuns) + + absOutputDir, _ := filepath.Abs(outputDir) + return buildLogsDataResult(summary, runs, episodes, edges, continuation, absOutputDir, summaries) +} + +func buildLogsDataAddEpisodeCounts(summary *LogsSummary, episodes []EpisodeData) { for _, episode := range episodes { summary.TotalEpisodes++ if episode.Confidence == "high" { summary.HighConfidenceEpisodes++ } } +} - // Build tool usage summary - toolUsage := buildToolUsageSummary(processedRuns) +func buildLogsDataRunData(pr ProcessedRun, processedRuns []ProcessedRun, totals *buildLogsDataTotals) RunData { + run := pr.Run + buildLogsDataAccumulateRun(pr, totals) + ctx := buildLogsDataRunContext{ + chainMetrics: buildLogsDataChainMetrics(run, totals), + gitHubAPICalls: buildLogsDataGitHubAPICalls(pr, totals), + comparison: buildAuditComparisonForProcessedRuns(pr, processedRuns), + } + ctx.engineID, ctx.engineName, ctx.awContext, ctx.awInfo = buildLogsDataEngineInfo(run, pr, totals) + if pr.TokenUsage != nil { + ctx.ambientContext = pr.TokenUsage.AmbientContext + } - // Build combined error and warning summary - errorsAndWarnings := buildCombinedErrorsSummary(processedRuns) + runData := buildLogsDataBaseRunData(pr, ctx) + buildLogsDataApplyAwInfo(&runData, ctx.awInfo) + buildLogsDataFinalizeRunData(&runData, pr) + return runData +} - // Build missing tools summary - missingTools := buildMissingToolsSummary(processedRuns) +func buildLogsDataAccumulateRun(pr ProcessedRun, totals *buildLogsDataTotals) { + run := pr.Run + if run.Duration > 0 { + totals.totalDuration += run.Duration + } + if pr.TokenUsage != nil { + totals.totalAIC += pr.TokenUsage.TotalAIC + totals.totalSteeringEvents += pr.TokenUsage.TotalSteeringEvents + } + totals.totalTokens += run.TokenUsage + totals.totalActionMinutes += run.ActionMinutes + totals.totalTurns += run.Turns + totals.totalErrors += run.ErrorCount + totals.totalWarnings += run.WarningCount + totals.totalMissingTools += run.MissingToolCount + totals.totalMissingData += run.MissingDataCount + totals.totalSafeItems += run.SafeItemsCount +} - // Build missing data summary - missingData := buildMissingDataSummary(processedRuns) +func buildLogsDataGitHubAPICalls(pr ProcessedRun, totals *buildLogsDataTotals) int { + var gitHubAPICalls int + if pr.GitHubRateLimitUsage != nil { + gitHubAPICalls = pr.GitHubRateLimitUsage.TotalRequestsMade + } + totals.totalGitHubAPICalls += gitHubAPICalls + return gitHubAPICalls +} - // Build MCP failures summary - mcpFailures := buildMCPFailuresSummary(processedRuns) +func buildLogsDataChainMetrics(run WorkflowRun, totals *buildLogsDataTotals) SafeOutputChainMetrics { + chainMetrics := buildSafeOutputChainMetrics(run.LogsPath) + totals.totalTemporaryIDMappings += chainMetrics.TemporaryIDMappings + totals.totalChainedTargets += chainMetrics.ChainedTargetCount + totals.totalChainedFollowupActions += chainMetrics.ChainedFollowupActionCount + totals.totalClosedTempTargets += chainMetrics.ClosedTempTargetCount + if chainMetrics.ChainedTargetCount > 0 { + totals.runsWithTemporaryIDChains++ + } + if chainMetrics.DelegatedTempTargetCount > 0 { + totals.runsWithDelegatedTempTargets++ + } + switch chainMetrics.TemporaryIDMapStatus { + case temporaryIDMapStatusMissing: + totals.runsWithMissingTemporaryIDMap++ + case temporaryIDMapStatusInvalid: + totals.runsWithInvalidTemporaryIDMap++ + } + return chainMetrics +} - // Build MCP tool usage summary - mcpToolUsage := buildMCPToolUsageSummary(processedRuns) +func buildLogsDataEngineInfo(run WorkflowRun, pr ProcessedRun, totals *buildLogsDataTotals) (string, string, *AwContext, *AwInfo) { + engineID := "" + engineName := "" + var awContext *AwContext + var awInfo *AwInfo + awInfoPath := filepath.Join(run.LogsPath, "aw_info.json") + if info, err := parseAwInfo(awInfoPath, false); err == nil && info != nil { + awInfo = info + engineID = info.EngineID + engineName = info.EngineName + awContext = info.Context + } + if engineName == "" { + engineName = engineID + } + if awContext == nil { + awContext = pr.AwContext + } + if engineID != "" { + totals.engineCounts[engineID]++ + } + return engineID, engineName, awContext, awInfo +} + +func buildLogsDataBaseRunData(pr ProcessedRun, ctx buildLogsDataRunContext) RunData { + run := pr.Run + return RunData{ + RunID: run.DatabaseID, + Number: run.Number, + WorkflowName: run.WorkflowName, + WorkflowPath: run.WorkflowPath, + Agent: ctx.engineID, + Engine: ctx.engineName, + EngineID: ctx.engineID, + Status: run.Status, + Conclusion: run.Conclusion, + Classification: deriveRunClassification(ctx.comparison), + TokenUsage: run.TokenUsage, + AIC: 0, + AmbientContext: ctx.ambientContext, + ActionMinutes: run.ActionMinutes, + Turns: run.Turns, + ErrorCount: run.ErrorCount, + WarningCount: run.WarningCount, + MissingToolCount: run.MissingToolCount, + MissingDataCount: run.MissingDataCount, + SafeItemsCount: run.SafeItemsCount, + ManifestEntryCount: ctx.chainMetrics.ManifestEntryCount, + TemporaryIDMapStatus: ctx.chainMetrics.TemporaryIDMapStatus, + TemporaryIDMappings: ctx.chainMetrics.TemporaryIDMappings, + ChainedTargetCount: ctx.chainMetrics.ChainedTargetCount, + ChainedFollowupActionCount: ctx.chainMetrics.ChainedFollowupActionCount, + DelegatedTempTargetCount: ctx.chainMetrics.DelegatedTempTargetCount, + ClosedTempTargetCount: ctx.chainMetrics.ClosedTempTargetCount, + CreatedAt: run.CreatedAt, + StartedAt: run.StartedAt, + UpdatedAt: run.UpdatedAt, + URL: run.URL, + LogsPath: run.LogsPath, + Event: run.Event, + Branch: run.HeadBranch, + HeadSHA: run.HeadSha, + DisplayTitle: run.DisplayTitle, + Comparison: ctx.comparison, + TaskDomain: pr.TaskDomain, + BehaviorFingerprint: pr.BehaviorFingerprint, + AgenticAssessments: pr.AgenticAssessments, + AwContext: ctx.awContext, + TokenUsageSummary: pr.TokenUsage, + GitHubAPICalls: ctx.gitHubAPICalls, + Experiments: extractExperimentData(run.LogsPath), + } +} + +func buildLogsDataApplyAwInfo(runData *RunData, awInfo *AwInfo) { + if awInfo == nil { + return + } + runData.Repository = awInfo.Repository + if awInfo.Repository != "" { + if parts := strings.SplitN(awInfo.Repository, "/", 2); len(parts) == 2 { + runData.Organization = parts[0] + } + } + runData.Ref = awInfo.Ref + runData.SHA = awInfo.SHA + runData.Actor = awInfo.Actor + runData.RunAttempt = awInfo.RunAttempt + runData.TargetRepo = awInfo.TargetRepo + runData.EventName = awInfo.EventName + // Fall back to inferring the workflow path from the display name when the + // GitHub API returned an empty path (e.g. for scheduled agentic runs). + // This handles both fresh runs and old cached RunSummary entries whose + // run.WorkflowPath was persisted as empty before the fix in + // logs_run_processor.go was applied. + if runData.WorkflowPath == "" && awInfo.WorkflowName != "" { + runData.WorkflowPath = inferWorkflowPathFromDisplayName(awInfo.WorkflowName) + } +} - // Build access log summary - accessLog := buildAccessLogSummary(processedRuns) +func buildLogsDataFinalizeRunData(runData *RunData, pr ProcessedRun) { + run := pr.Run + if run.Duration > 0 { + runData.Duration = timeutil.FormatDuration(run.Duration) + } + if pr.TokenUsage != nil && pr.TokenUsage.TotalAIC > 0 { + runData.AIC = pr.TokenUsage.TotalAIC + } + // Compute average TBT from metrics when available; fall back to wall-time / (turns - 1). + if run.AvgTimeBetweenTurns > 0 { + runData.AvgTimeBetweenTurns = timeutil.FormatDuration(run.AvgTimeBetweenTurns) + } else if run.Turns > 1 && run.Duration > 0 { + runData.AvgTimeBetweenTurns = timeutil.FormatDuration(run.Duration/time.Duration(run.Turns-1)) + " (estimated)" + } +} - // Build firewall log summary - firewallLog := buildFirewallLogSummary(processedRuns) +func buildLogsDataSummary(processedRuns []ProcessedRun, totals buildLogsDataTotals) LogsSummary { + summary := LogsSummary{ + TotalRuns: len(processedRuns), + TotalDuration: timeutil.FormatDuration(totals.totalDuration), + TotalAIC: totals.totalAIC, + TotalTokens: totals.totalTokens, + TotalActionMinutes: totals.totalActionMinutes, + TotalTurns: totals.totalTurns, + TotalSteeringEvents: totals.totalSteeringEvents, + TotalErrors: totals.totalErrors, + TotalWarnings: totals.totalWarnings, + TotalMissingTools: totals.totalMissingTools, + TotalMissingData: totals.totalMissingData, + TotalSafeItems: totals.totalSafeItems, + RunsWithTemporaryIDChains: totals.runsWithTemporaryIDChains, + RunsWithDelegatedTempTargets: totals.runsWithDelegatedTempTargets, + RunsWithMissingTemporaryIDMap: totals.runsWithMissingTemporaryIDMap, + RunsWithInvalidTemporaryIDMap: totals.runsWithInvalidTemporaryIDMap, + TotalTemporaryIDMappings: totals.totalTemporaryIDMappings, + TotalChainedTargets: totals.totalChainedTargets, + TotalChainedFollowupActions: totals.totalChainedFollowupActions, + TotalClosedTempTargets: totals.totalClosedTempTargets, + TotalGitHubAPICalls: totals.totalGitHubAPICalls, + } + if len(totals.engineCounts) > 0 { + summary.EngineCounts = totals.engineCounts + } + return summary +} - // Build redacted domains summary - redactedDomains := buildRedactedDomainsSummary(processedRuns) +type buildLogsDataSummaries struct { + toolUsage []ToolUsageSummary + mcpToolUsage *MCPToolUsageSummary + observability []ObservabilityInsight + errorsAndWarnings []ErrorSummary + missingTools []MissingToolSummary + missingData []MissingDataSummary + mcpFailures []MCPFailureSummary + accessLog *AccessLogSummary + firewallLog *FirewallLogSummary + redactedDomains *RedactedDomainsLogSummary +} +func buildLogsDataBuildSummaries(processedRuns []ProcessedRun) buildLogsDataSummaries { + toolUsage := buildToolUsageSummary(processedRuns) observability := buildLogsObservabilityInsights(processedRuns, toolUsage) observability = append(observability, buildDrain3InsightsMultiRun(processedRuns)...) + return buildLogsDataSummaries{ + toolUsage: toolUsage, + mcpToolUsage: buildMCPToolUsageSummary(processedRuns), + observability: observability, + errorsAndWarnings: buildCombinedErrorsSummary(processedRuns), + missingTools: buildMissingToolsSummary(processedRuns), + missingData: buildMissingDataSummary(processedRuns), + mcpFailures: buildMCPFailuresSummary(processedRuns), + accessLog: buildAccessLogSummary(processedRuns), + firewallLog: buildFirewallLogSummary(processedRuns), + redactedDomains: buildRedactedDomainsSummary(processedRuns), + } +} - absOutputDir, _ := filepath.Abs(outputDir) - +func buildLogsDataResult(summary LogsSummary, runs []RunData, episodes []EpisodeData, edges []EpisodeEdge, continuation *ContinuationData, absOutputDir string, summaries buildLogsDataSummaries) LogsData { return LogsData{ Summary: summary, Runs: runs, Episodes: episodes, Edges: edges, - ToolUsage: toolUsage, - MCPToolUsage: mcpToolUsage, - Observability: observability, - ErrorsAndWarnings: errorsAndWarnings, - MissingTools: missingTools, - MissingData: missingData, - MCPFailures: mcpFailures, - AccessLog: accessLog, - FirewallLog: firewallLog, - RedactedDomains: redactedDomains, + ToolUsage: summaries.toolUsage, + MCPToolUsage: summaries.mcpToolUsage, + Observability: summaries.observability, + ErrorsAndWarnings: summaries.errorsAndWarnings, + MissingTools: summaries.missingTools, + MissingData: summaries.missingData, + MCPFailures: summaries.mcpFailures, + AccessLog: summaries.accessLog, + FirewallLog: summaries.firewallLog, + RedactedDomains: summaries.redactedDomains, Continuation: continuation, LogsLocation: absOutputDir, } diff --git a/pkg/cli/logs_report_mcp.go b/pkg/cli/logs_report_mcp.go index d443c38ba41..e92ad61373b 100644 --- a/pkg/cli/logs_report_mcp.go +++ b/pkg/cli/logs_report_mcp.go @@ -53,15 +53,40 @@ func buildMCPFailuresSummary(processedRuns []ProcessedRun) []MCPFailureSummary { } // buildMCPToolUsageSummary aggregates MCP tool usage data across all runs +type buildMCPToolUsageSummaryAggregation struct { + toolSummaryMap map[string]*MCPToolSummary + serverStatsMap map[string]*MCPServerStats + allToolCalls []MCPToolCall + allFilteredEvents []DifcFilteredEvent +} + func buildMCPToolUsageSummary(processedRuns []ProcessedRun) *MCPToolUsageSummary { reportLog.Printf("Building MCP tool usage summary from %d processed runs", len(processedRuns)) + aggregation := buildMCPToolUsageSummaryAggregate(processedRuns) + + // Return nil if no MCP tool usage data was found + if len(aggregation.toolSummaryMap) == 0 && len(aggregation.serverStatsMap) == 0 && + len(aggregation.allToolCalls) == 0 && len(aggregation.allFilteredEvents) == 0 { + return nil + } - // Maps for aggregating data - toolSummaryMap := make(map[string]*MCPToolSummary) // Key: serverName:toolName - serverStatsMap := make(map[string]*MCPServerStats) // Key: serverName - var allToolCalls []MCPToolCall - var allFilteredEvents []DifcFilteredEvent + summaries, servers := buildMCPToolUsageSummarySortedSlices(aggregation) + reportLog.Printf("Built MCP tool usage summary: %d tool summaries, %d servers, %d total tool calls, %d DIFC filtered events", + len(summaries), len(servers), len(aggregation.allToolCalls), len(aggregation.allFilteredEvents)) + return &MCPToolUsageSummary{ + Summary: summaries, + Servers: servers, + ToolCalls: aggregation.allToolCalls, + FilteredEvents: aggregation.allFilteredEvents, + } +} + +func buildMCPToolUsageSummaryAggregate(processedRuns []ProcessedRun) buildMCPToolUsageSummaryAggregation { + aggregation := buildMCPToolUsageSummaryAggregation{ + toolSummaryMap: make(map[string]*MCPToolSummary), // Key: serverName:toolName + serverStatsMap: make(map[string]*MCPServerStats), // Key: serverName + } // Aggregate data from all runs for _, pr := range processedRuns { if pr.MCPToolUsage == nil { @@ -69,101 +94,91 @@ func buildMCPToolUsageSummary(processedRuns []ProcessedRun) *MCPToolUsageSummary } // Aggregate tool calls - allToolCalls = append(allToolCalls, pr.MCPToolUsage.ToolCalls...) + aggregation.allToolCalls = append(aggregation.allToolCalls, pr.MCPToolUsage.ToolCalls...) // Aggregate DIFC filtered events - allFilteredEvents = append(allFilteredEvents, pr.MCPToolUsage.FilteredEvents...) + aggregation.allFilteredEvents = append(aggregation.allFilteredEvents, pr.MCPToolUsage.FilteredEvents...) // Aggregate tool summaries for _, summary := range pr.MCPToolUsage.Summary { - key := summary.ServerName + ":" + summary.ToolName - - if existing, exists := toolSummaryMap[key]; exists { - // Store previous count before updating - prevCallCount := existing.CallCount - - // Merge with existing summary - existing.CallCount += summary.CallCount - existing.TotalInputSize += summary.TotalInputSize - existing.TotalOutputSize += summary.TotalOutputSize - - // Update max sizes - if summary.MaxInputSize > existing.MaxInputSize { - existing.MaxInputSize = summary.MaxInputSize - } - if summary.MaxOutputSize > existing.MaxOutputSize { - existing.MaxOutputSize = summary.MaxOutputSize - } - - // Update error count - existing.ErrorCount += summary.ErrorCount - - // Recalculate average duration (weighted) - if summary.AvgDuration != "" && existing.CallCount > 0 { - existingDur := parseDurationString(existing.AvgDuration) - newDur := parseDurationString(summary.AvgDuration) - // Weight by call counts using previous count - weightedDur := (existingDur*time.Duration(prevCallCount) + newDur*time.Duration(summary.CallCount)) / time.Duration(existing.CallCount) - existing.AvgDuration = timeutil.FormatDuration(weightedDur) - } - - // Update max duration - if summary.MaxDuration != "" { - maxDur := parseDurationString(summary.MaxDuration) - existingMaxDur := parseDurationString(existing.MaxDuration) - if maxDur > existingMaxDur { - existing.MaxDuration = summary.MaxDuration - } - } - } else { - // Create new summary entry (copy to avoid mutation) - newSummary := summary - toolSummaryMap[key] = &newSummary - } + buildMCPToolUsageSummaryMergeTool(aggregation.toolSummaryMap, summary) } // Aggregate server stats for _, serverStats := range pr.MCPToolUsage.Servers { - if existing, exists := serverStatsMap[serverStats.ServerName]; exists { - // Store previous count before updating - prevRequestCount := existing.RequestCount - - // Merge with existing stats - existing.RequestCount += serverStats.RequestCount - existing.ToolCallCount += serverStats.ToolCallCount - existing.TotalInputSize += serverStats.TotalInputSize - existing.TotalOutputSize += serverStats.TotalOutputSize - existing.ErrorCount += serverStats.ErrorCount - - // Recalculate average duration (weighted) - if serverStats.AvgDuration != "" && existing.RequestCount > 0 { - existingDur := parseDurationString(existing.AvgDuration) - newDur := parseDurationString(serverStats.AvgDuration) - // Weight by request counts using previous count - weightedDur := (existingDur*time.Duration(prevRequestCount) + newDur*time.Duration(serverStats.RequestCount)) / time.Duration(existing.RequestCount) - existing.AvgDuration = timeutil.FormatDuration(weightedDur) - } - } else { - // Create new server stats entry (copy to avoid mutation) - newStats := serverStats - serverStatsMap[serverStats.ServerName] = &newStats - } + buildMCPToolUsageSummaryMergeServer(aggregation.serverStatsMap, serverStats) } } + return aggregation +} - // Return nil if no MCP tool usage data was found - if len(toolSummaryMap) == 0 && len(serverStatsMap) == 0 && len(allToolCalls) == 0 && len(allFilteredEvents) == 0 { - return nil +func buildMCPToolUsageSummaryMergeTool(toolSummaryMap map[string]*MCPToolSummary, summary MCPToolSummary) { + key := summary.ServerName + ":" + summary.ToolName + if existing, exists := toolSummaryMap[key]; exists { + prevCallCount := existing.CallCount + existing.CallCount += summary.CallCount + existing.TotalInputSize += summary.TotalInputSize + existing.TotalOutputSize += summary.TotalOutputSize + if summary.MaxInputSize > existing.MaxInputSize { + existing.MaxInputSize = summary.MaxInputSize + } + if summary.MaxOutputSize > existing.MaxOutputSize { + existing.MaxOutputSize = summary.MaxOutputSize + } + existing.ErrorCount += summary.ErrorCount + buildMCPToolUsageSummaryMergeToolDurations(existing, summary, prevCallCount) + return + } + newSummary := summary + toolSummaryMap[key] = &newSummary +} + +func buildMCPToolUsageSummaryMergeToolDurations(existing *MCPToolSummary, summary MCPToolSummary, prevCallCount int) { + if summary.AvgDuration != "" && existing.CallCount > 0 { + existingDur := parseDurationString(existing.AvgDuration) + newDur := parseDurationString(summary.AvgDuration) + // Weight by call counts using previous count + weightedDur := (existingDur*time.Duration(prevCallCount) + newDur*time.Duration(summary.CallCount)) / time.Duration(existing.CallCount) + existing.AvgDuration = timeutil.FormatDuration(weightedDur) + } + if summary.MaxDuration != "" { + maxDur := parseDurationString(summary.MaxDuration) + existingMaxDur := parseDurationString(existing.MaxDuration) + if maxDur > existingMaxDur { + existing.MaxDuration = summary.MaxDuration + } + } +} + +func buildMCPToolUsageSummaryMergeServer(serverStatsMap map[string]*MCPServerStats, serverStats MCPServerStats) { + if existing, exists := serverStatsMap[serverStats.ServerName]; exists { + prevRequestCount := existing.RequestCount + existing.RequestCount += serverStats.RequestCount + existing.ToolCallCount += serverStats.ToolCallCount + existing.TotalInputSize += serverStats.TotalInputSize + existing.TotalOutputSize += serverStats.TotalOutputSize + existing.ErrorCount += serverStats.ErrorCount + if serverStats.AvgDuration != "" && existing.RequestCount > 0 { + existingDur := parseDurationString(existing.AvgDuration) + newDur := parseDurationString(serverStats.AvgDuration) + weightedDur := (existingDur*time.Duration(prevRequestCount) + newDur*time.Duration(serverStats.RequestCount)) / time.Duration(existing.RequestCount) + existing.AvgDuration = timeutil.FormatDuration(weightedDur) + } + return } + newStats := serverStats + serverStatsMap[serverStats.ServerName] = &newStats +} +func buildMCPToolUsageSummarySortedSlices(aggregation buildMCPToolUsageSummaryAggregation) ([]MCPToolSummary, []MCPServerStats) { // Convert maps to slices var summaries []MCPToolSummary - for _, summary := range toolSummaryMap { + for _, summary := range aggregation.toolSummaryMap { summaries = append(summaries, *summary) } var servers []MCPServerStats - for _, stats := range serverStatsMap { + for _, stats := range aggregation.serverStatsMap { servers = append(servers, *stats) } @@ -180,13 +195,5 @@ func buildMCPToolUsageSummary(processedRuns []ProcessedRun) *MCPToolUsageSummary return cmp.Compare(a.ServerName, b.ServerName) }) - reportLog.Printf("Built MCP tool usage summary: %d tool summaries, %d servers, %d total tool calls, %d DIFC filtered events", - len(summaries), len(servers), len(allToolCalls), len(allFilteredEvents)) - - return &MCPToolUsageSummary{ - Summary: summaries, - Servers: servers, - ToolCalls: allToolCalls, - FilteredEvents: allFilteredEvents, - } + return summaries, servers } diff --git a/pkg/cli/logs_report_tools.go b/pkg/cli/logs_report_tools.go index 22002ed1b97..6e52b380da9 100644 --- a/pkg/cli/logs_report_tools.go +++ b/pkg/cli/logs_report_tools.go @@ -90,29 +90,7 @@ func buildToolUsageSummary(processedRuns []ProcessedRun) []ToolUsageSummary { toolRunTracker[displayKey] = struct { }{} - if existing, exists := toolStats[displayKey]; exists { - existing.TotalCalls += toolCall.CallCount - if toolCall.MaxOutputSize > existing.MaxOutputSize { - existing.MaxOutputSize = toolCall.MaxOutputSize - } - if toolCall.MaxDuration > 0 { - maxDur := timeutil.FormatDuration(toolCall.MaxDuration) - if existing.MaxDuration == "" || toolCall.MaxDuration > parseDurationString(existing.MaxDuration) { - existing.MaxDuration = maxDur - } - } - } else { - info := &ToolUsageSummary{ - Name: displayKey, - TotalCalls: toolCall.CallCount, - MaxOutputSize: toolCall.MaxOutputSize, - Runs: 0, // Will be incremented below - } - if toolCall.MaxDuration > 0 { - info.MaxDuration = timeutil.FormatDuration(toolCall.MaxDuration) - } - toolStats[displayKey] = info - } + buildToolUsageSummaryApplyToolCall(toolStats, displayKey, toolCall) } // Increment run count for tools used in this run @@ -135,3 +113,29 @@ func buildToolUsageSummary(processedRuns []ProcessedRun) []ToolUsageSummary { return result } + +func buildToolUsageSummaryApplyToolCall(toolStats map[string]*ToolUsageSummary, displayKey string, toolCall workflow.ToolCallInfo) { + if existing, exists := toolStats[displayKey]; exists { + existing.TotalCalls += toolCall.CallCount + if toolCall.MaxOutputSize > existing.MaxOutputSize { + existing.MaxOutputSize = toolCall.MaxOutputSize + } + if toolCall.MaxDuration > 0 { + maxDur := timeutil.FormatDuration(toolCall.MaxDuration) + if existing.MaxDuration == "" || toolCall.MaxDuration > parseDurationString(existing.MaxDuration) { + existing.MaxDuration = maxDur + } + } + return + } + info := &ToolUsageSummary{ + Name: displayKey, + TotalCalls: toolCall.CallCount, + MaxOutputSize: toolCall.MaxOutputSize, + Runs: 0, // Will be incremented below + } + if toolCall.MaxDuration > 0 { + info.MaxDuration = timeutil.FormatDuration(toolCall.MaxDuration) + } + toolStats[displayKey] = info +} diff --git a/pkg/cli/logs_safe_output_chains.go b/pkg/cli/logs_safe_output_chains.go index 63b68cbf000..30f165fb9f0 100644 --- a/pkg/cli/logs_safe_output_chains.go +++ b/pkg/cli/logs_safe_output_chains.go @@ -66,19 +66,7 @@ func buildSafeOutputChainMetrics(logsPath string) SafeOutputChainMetrics { closedTargets := make(map[string]struct { }) for _, item := range items { - if item.Repo == "" || item.Number <= 0 { - continue - } - key := safeOutputTargetKey(item.Repo, item.Number) - itemCounts[key]++ - switch item.Type { - case "assign_to_agent", "create_agent_session": - delegatedTargets[key] = struct { - }{} - case "close_issue", "close_pull_request", "close_discussion", "merge_pull_request": - closedTargets[key] = struct { - }{} - } + buildSafeOutputChainMetricsClassifyItem(item, itemCounts, delegatedTargets, closedTargets) } for _, target := range resolvedTargets { @@ -101,6 +89,20 @@ func buildSafeOutputChainMetrics(logsPath string) SafeOutputChainMetrics { return metrics } +func buildSafeOutputChainMetricsClassifyItem(item CreatedItemReport, itemCounts map[string]int, delegatedTargets, closedTargets map[string]struct{}) { + if item.Repo == "" || item.Number <= 0 { + return + } + key := safeOutputTargetKey(item.Repo, item.Number) + itemCounts[key]++ + switch item.Type { + case "assign_to_agent", "create_agent_session": + delegatedTargets[key] = struct{}{} + case "close_issue", "close_pull_request", "close_discussion", "merge_pull_request": + closedTargets[key] = struct{}{} + } +} + func loadResolvedTemporaryIDTargets(logsPath string) (map[string]resolvedTemporaryIDTarget, string) { if logsPath == "" { return nil, "" diff --git a/pkg/cli/logs_usage_activity.go b/pkg/cli/logs_usage_activity.go index ee06e977c82..bea2c212cde 100644 --- a/pkg/cli/logs_usage_activity.go +++ b/pkg/cli/logs_usage_activity.go @@ -105,64 +105,11 @@ func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *Do } if summary.Firewall != nil && result.FirewallAnalysis == nil { - logsUsageActivityLog.Printf("applyUsageActivitySummaryToResult: backfilling firewall analysis (total=%d allowed=%d blocked=%d)", summary.Firewall.TotalRequests, summary.Firewall.AllowedRequests, summary.Firewall.BlockedRequests) - requestsByDomain := maps.Clone(summary.Firewall.RequestsByDomain) - if requestsByDomain == nil { - requestsByDomain = map[string]DomainRequestStats{} - } - allowedSet := map[string]struct{}{} - blockedSet := map[string]struct{}{} - for _, domain := range summary.Firewall.AllowedDomains { - allowedSet[domain] = struct{}{} - } - for _, domain := range summary.Firewall.BlockedDomains { - blockedSet[domain] = struct{}{} - } - for domain, stats := range requestsByDomain { - if stats.Allowed > 0 { - allowedSet[domain] = struct{}{} - } - if stats.Blocked > 0 { - blockedSet[domain] = struct{}{} - } - } - allowedDomains := sliceutil.SortedKeys(allowedSet) - blockedDomains := sliceutil.SortedKeys(blockedSet) - - result.FirewallAnalysis = &FirewallAnalysis{ - AnalysisBase: AnalysisBase{ - DomainBuckets: DomainBuckets{ - AllowedDomains: allowedDomains, - BlockedDomains: blockedDomains, - }, - TotalRequests: summary.Firewall.TotalRequests, - AllowedRequests: summary.Firewall.AllowedRequests, - BlockedRequests: summary.Firewall.BlockedRequests, - }, - RequestsByDomain: requestsByDomain, - } + applyUsageActivitySummaryToResultFirewall(summary, result) } if summary.Gateway != nil && result.MCPToolUsage == nil { - logsUsageActivityLog.Printf("applyUsageActivitySummaryToResult: backfilling MCP tool usage from %d gateway server(s)", len(summary.Gateway.Servers)) - servers := make([]MCPServerStats, 0, len(summary.Gateway.Servers)) - for _, server := range summary.Gateway.Servers { - servers = append(servers, MCPServerStats{ - ServerName: server.ServerName, - // Keep both RequestCount and ToolCallCount aligned because MCPServerStats - // distinguishes overall request volume (RequestCount) from tool-invocation - // volume (ToolCallCount). In usage-aggregate mode we only have per-server - // tool-call counts, so both fields are populated from that single source. - RequestCount: server.ToolCallCount, - ToolCallCount: server.ToolCallCount, - ErrorCount: server.FailedCalls, - }) - } - result.MCPToolUsage = &MCPToolUsageData{ - Summary: []MCPToolSummary{}, - ToolCalls: []MCPToolCall{}, - Servers: servers, - } + applyUsageActivitySummaryToResultGateway(summary, result) } // Backfill safe output item count from usage summary when the safe-outputs-items @@ -173,3 +120,61 @@ func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *Do result.Run.SafeItemsCount = summary.SafeOutputs.TotalItems } } + +func applyUsageActivitySummaryToResultFirewall(summary *usageActivitySummary, result *DownloadResult) { + logsUsageActivityLog.Printf("applyUsageActivitySummaryToResult: backfilling firewall analysis (total=%d allowed=%d blocked=%d)", summary.Firewall.TotalRequests, summary.Firewall.AllowedRequests, summary.Firewall.BlockedRequests) + requestsByDomain := maps.Clone(summary.Firewall.RequestsByDomain) + if requestsByDomain == nil { + requestsByDomain = map[string]DomainRequestStats{} + } + allowedSet := map[string]struct{}{} + blockedSet := map[string]struct{}{} + for _, domain := range summary.Firewall.AllowedDomains { + allowedSet[domain] = struct{}{} + } + for _, domain := range summary.Firewall.BlockedDomains { + blockedSet[domain] = struct{}{} + } + for domain, stats := range requestsByDomain { + if stats.Allowed > 0 { + allowedSet[domain] = struct{}{} + } + if stats.Blocked > 0 { + blockedSet[domain] = struct{}{} + } + } + result.FirewallAnalysis = &FirewallAnalysis{ + AnalysisBase: AnalysisBase{ + DomainBuckets: DomainBuckets{ + AllowedDomains: sliceutil.SortedKeys(allowedSet), + BlockedDomains: sliceutil.SortedKeys(blockedSet), + }, + TotalRequests: summary.Firewall.TotalRequests, + AllowedRequests: summary.Firewall.AllowedRequests, + BlockedRequests: summary.Firewall.BlockedRequests, + }, + RequestsByDomain: requestsByDomain, + } +} + +func applyUsageActivitySummaryToResultGateway(summary *usageActivitySummary, result *DownloadResult) { + logsUsageActivityLog.Printf("applyUsageActivitySummaryToResult: backfilling MCP tool usage from %d gateway server(s)", len(summary.Gateway.Servers)) + servers := make([]MCPServerStats, 0, len(summary.Gateway.Servers)) + for _, server := range summary.Gateway.Servers { + servers = append(servers, MCPServerStats{ + ServerName: server.ServerName, + // Keep both RequestCount and ToolCallCount aligned because MCPServerStats + // distinguishes overall request volume (RequestCount) from tool-invocation + // volume (ToolCallCount). In usage-aggregate mode we only have per-server + // tool-call counts, so both fields are populated from that single source. + RequestCount: server.ToolCallCount, + ToolCallCount: server.ToolCallCount, + ErrorCount: server.FailedCalls, + }) + } + result.MCPToolUsage = &MCPToolUsageData{ + Summary: []MCPToolSummary{}, + ToolCalls: []MCPToolCall{}, + Servers: servers, + } +} diff --git a/pkg/cli/logs_utils.go b/pkg/cli/logs_utils.go index 8a6263af462..012889d1554 100644 --- a/pkg/cli/logs_utils.go +++ b/pkg/cli/logs_utils.go @@ -46,39 +46,9 @@ func getAgenticWorkflowNames(verbose bool) ([]string, error) { logsUtilsLog.Printf("Found %d .lock.yml file(s) in %s", len(files), workflowsDir) for _, file := range files { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Reading workflow file: "+file)) - } - - content, err := os.ReadFile(file) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read %s: %v", file, err))) - } - continue - } - - // Extract the workflow name using simple string parsing - lines := strings.SplitSeq(string(content), "\n") - for line := range lines { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "name:") { - // Parse the name field - parts := strings.SplitN(trimmed, ":", 2) - if len(parts) == 2 { - name := strings.TrimSpace(parts[1]) - // Remove quotes if present - name = strings.Trim(name, `"'`) - if name != "" { - workflowNames = append(workflowNames, name) - logsUtilsLog.Printf("Discovered workflow name: %s (from %s)", name, file) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found agentic workflow: "+name)) - } - break - } - } - } + name, ok := getAgenticWorkflowNamesFromFile(file, verbose) + if ok { + workflowNames = append(workflowNames, name) } } @@ -89,6 +59,43 @@ func getAgenticWorkflowNames(verbose bool) ([]string, error) { return workflowNames, nil } +func getAgenticWorkflowNamesFromFile(file string, verbose bool) (string, bool) { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Reading workflow file: "+file)) + } + content, err := os.ReadFile(file) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read %s: %v", file, err))) + } + return "", false + } + + // Extract the workflow name using simple string parsing + lines := strings.SplitSeq(string(content), "\n") + for line := range lines { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "name:") { + continue + } + // Parse the name field + parts := strings.SplitN(trimmed, ":", 2) + if len(parts) != 2 { + continue + } + name := strings.Trim(strings.TrimSpace(parts[1]), `"'`) + if name == "" { + continue + } + logsUtilsLog.Printf("Discovered workflow name: %s (from %s)", name, file) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found agentic workflow: "+name)) + } + return name, true + } + return "", false +} + // findAgentOutputFile searches for a file named agent_output.json within the logDir tree. // Returns the first path found (depth-first) and a boolean indicating success. func findAgentOutputFile(logDir string) (string, bool) { @@ -130,145 +137,155 @@ func findAgentLogFile(logDir string, engine workflow.CodingAgentEngine) (string, // If the engine specifies a log file that isn't the default agent-stdio.log, // look in the agent_output artifact directory or flattened location if logFileForParsing != "" && logFileForParsing != defaultAgentStdioLogPath { - // Check for agent_output directory (artifact, before flattening) - agentOutputDir := filepath.Join(logDir, "agent_output") - if fileutil.DirExists(agentOutputDir) { - // Find the first file in this directory - var foundFile string - if walkErr := filepath.Walk(agentOutputDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - logsUtilsLog.Printf("walk error at %s: %v", path, err) - return nil - } - if info == nil { - return nil - } - if !info.IsDir() && foundFile == "" { - foundFile = path - return errWalkStop - } - return nil - }); walkErr != nil && !errors.Is(walkErr, errWalkStop) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", agentOutputDir, walkErr))) - } - if foundFile != "" { - return foundFile, true - } + if found, ok := findAgentLogFileNonDefault(logDir, logFileForParsing); ok { + return found, true } + } + + return findAgentLogFileDefault(logDir) +} + +func findAgentLogFileNonDefault(logDir, logFileForParsing string) (string, bool) { + if found, ok := findAgentLogFileAgentOutputDir(logDir); ok { + return found, true + } + if found, ok := findAgentLogFileFlattened(logDir, logFileForParsing); ok { + return found, true + } + return findAgentLogFileRecursive(logDir) +} - // Check for flattened location (after flattening) - // The engine's log path is absolute (e.g., /tmp/gh-aw/sandbox/agent/logs/ for a - // directory, or /tmp/gh-aw/pi-streaming.jsonl for a direct file). - // After flattening, it's at logDir/. - // Strip /tmp/gh-aw/ prefix to get the relative path - const tmpGhAwPrefix = "/tmp/gh-aw/" - if after, ok := strings.CutPrefix(logFileForParsing, tmpGhAwPrefix); ok { - relPath := after - flattenedPath := filepath.Join(logDir, relPath) - logsUtilsLog.Printf("Checking flattened location for logs: %s", flattenedPath) - // Case 1: the engine log path points to a specific file (e.g. pi-streaming.jsonl) - if fileutil.FileExists(flattenedPath) { - logsUtilsLog.Printf("Found engine log file at flattened path: %s", flattenedPath) - return flattenedPath, true - } - // Case 2: the engine log path points to a directory — walk it for known log formats - if fileutil.DirExists(flattenedPath) { - // Prefer events.jsonl (structured Copilot session format) over debug .log files. - // Walk the full tree: stop immediately when events.jsonl is found (preferred), - // but keep walking after a .log match in case events.jsonl appears later. - var foundEventsJsonl, foundLogFile string - if walkErr := filepath.Walk(flattenedPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - logsUtilsLog.Printf("walk error at %s: %v", path, err) - return nil - } - if info == nil { - return nil - } - if !info.IsDir() { - if info.Name() == "events.jsonl" && foundEventsJsonl == "" { - foundEventsJsonl = path - logsUtilsLog.Printf("Found events.jsonl file: %s", path) - return errWalkStop - } else if strings.HasSuffix(info.Name(), ".log") && foundLogFile == "" { - foundLogFile = path - logsUtilsLog.Printf("Found session log file: %s", path) - } - } - return nil - }); walkErr != nil && !errors.Is(walkErr, errWalkStop) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", flattenedPath, walkErr))) - } - if foundEventsJsonl != "" { - return foundEventsJsonl, true - } - if foundLogFile != "" { - return foundLogFile, true - } - } - // Case 3: fall back to searching logDir for a file whose base name matches - // the engine's log file name (handles unusual artifact layouts) - targetBase := filepath.Base(logFileForParsing) - logsUtilsLog.Printf("Searching logDir %s for engine log file by base name: %s", logDir, targetBase) - var foundByBase string - if walkErr := filepath.Walk(logDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - logsUtilsLog.Printf("walk error at %s: %v", path, err) - return nil - } - if info == nil { - return nil - } - if !info.IsDir() && info.Name() == targetBase && foundByBase == "" { - foundByBase = path - return errWalkStop - } - return nil - }); walkErr != nil && !errors.Is(walkErr, errWalkStop) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", logDir, walkErr))) - } - if foundByBase != "" { - logsUtilsLog.Printf("Found engine log file by base name: %s", foundByBase) - return foundByBase, true - } +func findAgentLogFileAgentOutputDir(logDir string) (string, bool) { + // Check for agent_output directory (artifact, before flattening) + agentOutputDir := filepath.Join(logDir, "agent_output") + if !fileutil.DirExists(agentOutputDir) { + return "", false + } + // Find the first file in this directory + var foundFile string + if walkErr := filepath.Walk(agentOutputDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + logsUtilsLog.Printf("walk error at %s: %v", path, err) + return nil + } + if info != nil && !info.IsDir() && foundFile == "" { + foundFile = path + return errWalkStop } + return nil + }); walkErr != nil && !errors.Is(walkErr, errWalkStop) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", agentOutputDir, walkErr))) + } + return foundFile, foundFile != "" +} - // Fallback: search recursively in logDir for events.jsonl, session*.log or process*.log files - // This handles cases where the artifact structure is different than expected - // Note: Copilot changed from session-*.log to process-*.log naming convention - logsUtilsLog.Printf("Searching recursively in %s for events.jsonl, session*.log or process*.log files", logDir) - var foundEventsJsonl, foundLogFile string - if walkErr := filepath.Walk(logDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - logsUtilsLog.Printf("walk error at %s: %v", path, err) - return nil - } - if info == nil { - return nil - } - fileName := info.Name() - if !info.IsDir() { - if fileName == "events.jsonl" && foundEventsJsonl == "" { - foundEventsJsonl = path - logsUtilsLog.Printf("Found events.jsonl via recursive search: %s", path) - return errWalkStop - } else if (strings.HasPrefix(fileName, "session") || strings.HasPrefix(fileName, "process")) && strings.HasSuffix(fileName, ".log") && foundLogFile == "" { - foundLogFile = path - logsUtilsLog.Printf("Found Copilot log file via recursive search: %s", path) - } - } +func findAgentLogFileFlattened(logDir, logFileForParsing string) (string, bool) { + // Check for flattened location (after flattening). Strip /tmp/gh-aw/ prefix to + // get the relative path under logDir. + const tmpGhAwPrefix = "/tmp/gh-aw/" + after, ok := strings.CutPrefix(logFileForParsing, tmpGhAwPrefix) + if !ok { + return "", false + } + flattenedPath := filepath.Join(logDir, after) + logsUtilsLog.Printf("Checking flattened location for logs: %s", flattenedPath) + if fileutil.FileExists(flattenedPath) { + logsUtilsLog.Printf("Found engine log file at flattened path: %s", flattenedPath) + return flattenedPath, true + } + if fileutil.DirExists(flattenedPath) { + if found, ok := findAgentLogFileFlattenedDir(flattenedPath); ok { + return found, true + } + } + return findAgentLogFileByBase(logDir, logFileForParsing) +} + +func findAgentLogFileFlattenedDir(flattenedPath string) (string, bool) { + // Prefer events.jsonl (structured Copilot session format) over debug .log files. + var foundEventsJsonl, foundLogFile string + if walkErr := filepath.Walk(flattenedPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + logsUtilsLog.Printf("walk error at %s: %v", path, err) return nil - }); walkErr != nil && !errors.Is(walkErr, errWalkStop) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", logDir, walkErr))) } - if foundEventsJsonl != "" { - return foundEventsJsonl, true + if info == nil || info.IsDir() { + return nil } - if foundLogFile != "" { - return foundLogFile, true + if info.Name() == "events.jsonl" && foundEventsJsonl == "" { + foundEventsJsonl = path + logsUtilsLog.Printf("Found events.jsonl file: %s", path) + return errWalkStop + } else if strings.HasSuffix(info.Name(), ".log") && foundLogFile == "" { + foundLogFile = path + logsUtilsLog.Printf("Found session log file: %s", path) } + return nil + }); walkErr != nil && !errors.Is(walkErr, errWalkStop) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", flattenedPath, walkErr))) } + if foundEventsJsonl != "" { + return foundEventsJsonl, true + } + return foundLogFile, foundLogFile != "" +} + +func findAgentLogFileByBase(logDir, logFileForParsing string) (string, bool) { + targetBase := filepath.Base(logFileForParsing) + logsUtilsLog.Printf("Searching logDir %s for engine log file by base name: %s", logDir, targetBase) + var foundByBase string + if walkErr := filepath.Walk(logDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + logsUtilsLog.Printf("walk error at %s: %v", path, err) + return nil + } + if info != nil && !info.IsDir() && info.Name() == targetBase && foundByBase == "" { + foundByBase = path + return errWalkStop + } + return nil + }); walkErr != nil && !errors.Is(walkErr, errWalkStop) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", logDir, walkErr))) + } + if foundByBase != "" { + logsUtilsLog.Printf("Found engine log file by base name: %s", foundByBase) + return foundByBase, true + } + return "", false +} + +func findAgentLogFileRecursive(logDir string) (string, bool) { + // Fallback: search recursively in logDir for events.jsonl, session*.log or process*.log files. + logsUtilsLog.Printf("Searching recursively in %s for events.jsonl, session*.log or process*.log files", logDir) + var foundEventsJsonl, foundLogFile string + if walkErr := filepath.Walk(logDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + logsUtilsLog.Printf("walk error at %s: %v", path, err) + return nil + } + if info == nil || info.IsDir() { + return nil + } + fileName := info.Name() + if fileName == "events.jsonl" && foundEventsJsonl == "" { + foundEventsJsonl = path + logsUtilsLog.Printf("Found events.jsonl via recursive search: %s", path) + return errWalkStop + } else if (strings.HasPrefix(fileName, "session") || strings.HasPrefix(fileName, "process")) && strings.HasSuffix(fileName, ".log") && foundLogFile == "" { + foundLogFile = path + logsUtilsLog.Printf("Found Copilot log file via recursive search: %s", path) + } + return nil + }); walkErr != nil && !errors.Is(walkErr, errWalkStop) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", logDir, walkErr))) + } + if foundEventsJsonl != "" { + return foundEventsJsonl, true + } + return foundLogFile, foundLogFile != "" +} +func findAgentLogFileDefault(logDir string) (string, bool) { // Default to agent-stdio.log agentStdioLog := filepath.Join(logDir, "agent-stdio.log") if fileutil.FileExists(agentStdioLog) { diff --git a/pkg/cli/mcp_add.go b/pkg/cli/mcp_add.go index 7ab25c55577..9066a9b8bb5 100644 --- a/pkg/cli/mcp_add.go +++ b/pkg/cli/mcp_add.go @@ -22,93 +22,126 @@ var mcpAddLog = logger.New("cli:mcp_add") func AddMCPTool(ctx context.Context, workflowFile string, mcpServerID string, registryURL string, transportType string, customToolID string, verbose bool) error { mcpAddLog.Printf("Adding MCP tool: serverID=%s, registryURL=%s, transport=%s", mcpServerID, registryURL, transportType) + workflowPath, err := addMCPToolResolveWorkflow(workflowFile, mcpServerID, verbose) + if err != nil { + return err + } + + // Create registry client + registryClient := NewMCPRegistryClient(registryURL) + selectedServer, err := addMCPToolSelectServer(ctx, registryClient, mcpServerID, verbose) + if err != nil { + return err + } + + // Determine tool ID (use custom if provided, otherwise use cleaned server name) + toolID := addMCPToolToolID(selectedServer, customToolID, verbose) + if err := addMCPToolValidateWorkflow(workflowPath, toolID); err != nil { + return err + } + + // Create MCP tool configuration based on server info and preferences + mcpConfig, err := createMCPToolConfig(selectedServer, transportType, registryClient.registryURL, verbose) + if err != nil { + return fmt.Errorf("failed to create MCP tool configuration: %w", err) + } + + // Add the tool to the workflow + if err := addToolToWorkflow(workflowPath, toolID, mcpConfig, verbose); err != nil { + return fmt.Errorf("failed to add tool to workflow: %w", err) + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Added MCP tool '%s' to workflow %s", toolID, console.ToRelativePath(workflowPath)))) + + // Check for required secrets and provide CLI commands if missing + if err := checkAndSuggestSecrets(mcpConfig, verbose); err != nil { + // Don't fail the command if secret checking fails, just log a warning + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not check repository secrets: %v", err))) + } + } + + addMCPToolCompile(workflowPath, verbose) + return nil +} + +func addMCPToolResolveWorkflow(workflowFile, mcpServerID string, verbose bool) (string, error) { // Resolve the workflow file path workflowPath, err := ResolveWorkflowPath(workflowFile) if err != nil { mcpAddLog.Printf("Failed to resolve workflow path: %v", err) - return err + return "", err } mcpAddLog.Printf("Resolved workflow path: %s", workflowPath) - if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Adding MCP tool '%s' to workflow: %s", mcpServerID, console.ToRelativePath(workflowPath)))) } + return workflowPath, nil +} - // Create registry client - registryClient := NewMCPRegistryClient(registryURL) - +func addMCPToolSelectServer(ctx context.Context, registryClient *MCPRegistryClient, mcpServerID string, verbose bool) (*MCPRegistryServerForProcessing, error) { // Search for the MCP server in the registry if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Searching for MCP server '%s' in registry: %s", mcpServerID, registryClient.registryURL))) } - mcpAddLog.Printf("Searching MCP registry for server: %s", mcpServerID) servers, err := registryClient.SearchServers(ctx, mcpServerID) if err != nil { mcpAddLog.Printf("MCP registry search failed: %v", err) - return fmt.Errorf("failed to search MCP registry: %w", err) + return nil, fmt.Errorf("failed to search MCP registry: %w", err) } mcpAddLog.Printf("Found %d matching servers in registry", len(servers)) - if len(servers) == 0 { - return fmt.Errorf("no MCP servers found matching '%s'", mcpServerID) + return nil, fmt.Errorf("no MCP servers found matching '%s'", mcpServerID) + } + if selectedServer := addMCPToolSelectServerMatch(servers, mcpServerID); selectedServer != nil { + return selectedServer, nil + } + selectedServer := &servers[0] + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No exact match for '%s', using closest match: %s", mcpServerID, selectedServer.Name))) } + return selectedServer, nil +} - // Find exact match by name first, then by partial match - var selectedServer *MCPRegistryServerForProcessing +func addMCPToolSelectServerMatch(servers []MCPRegistryServerForProcessing, mcpServerID string) *MCPRegistryServerForProcessing { for i, server := range servers { // Prioritize name matches over ID matches if server.Name == mcpServerID { - selectedServer = &servers[i] - break - } - } - - // If no name match, try partial match - if selectedServer == nil { - for i, server := range servers { - if strings.Contains(strings.ToLower(server.Name), strings.ToLower(mcpServerID)) { - selectedServer = &servers[i] - break - } + return &servers[i] } } - - // If still no exact match, use the first result if it looks like a partial match - if selectedServer == nil && len(servers) > 0 { - selectedServer = &servers[0] - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("No exact match for '%s', using closest match: %s", mcpServerID, selectedServer.Name))) + for i, server := range servers { + if strings.Contains(strings.ToLower(server.Name), strings.ToLower(mcpServerID)) { + return &servers[i] } } + return nil +} - if selectedServer == nil { - return fmt.Errorf("no MCP servers found matching '%s'", mcpServerID) - } - - // Determine tool ID (use custom if provided, otherwise use cleaned server name) +func addMCPToolToolID(selectedServer *MCPRegistryServerForProcessing, customToolID string, verbose bool) string { toolID := stringutil.SanitizeToolID(selectedServer.Name) if customToolID != "" { toolID = customToolID } - if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Selected server: %s (Transport: %s)", selectedServer.Name, selectedServer.Transport))) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Will add as tool ID: "+toolID)) } + return toolID +} +func addMCPToolValidateWorkflow(workflowPath, toolID string) error { // Read the workflow file content, err := os.ReadFile(workflowPath) if err != nil { return fmt.Errorf("failed to read workflow file: %w", err) } - // Parse the workflow file workflowData, err := parser.ExtractFrontmatterFromContent(string(content)) if err != nil { return fmt.Errorf("failed to parse workflow file: %w", err) } - // Check if tool already exists if workflowData.Frontmatter["tools"] != nil { if tools, ok := workflowData.Frontmatter["tools"].(map[string]any); ok { @@ -117,33 +150,14 @@ func AddMCPTool(ctx context.Context, workflowFile string, mcpServerID string, re } } } + return nil +} - // Create MCP tool configuration based on server info and preferences - mcpConfig, err := createMCPToolConfig(selectedServer, transportType, registryClient.registryURL, verbose) - if err != nil { - return fmt.Errorf("failed to create MCP tool configuration: %w", err) - } - - // Add the tool to the workflow - if err := addToolToWorkflow(workflowPath, toolID, mcpConfig, verbose); err != nil { - return fmt.Errorf("failed to add tool to workflow: %w", err) - } - - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Added MCP tool '%s' to workflow %s", toolID, console.ToRelativePath(workflowPath)))) - - // Check for required secrets and provide CLI commands if missing - if err := checkAndSuggestSecrets(mcpConfig, verbose); err != nil { - // Don't fail the command if secret checking fails, just log a warning - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not check repository secrets: %v", err))) - } - } - +func addMCPToolCompile(workflowPath string, verbose bool) { // Compile the workflow if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Compiling workflow...")) } - mcpAddLog.Print("Compiling workflow after adding MCP tool") compiler := workflow.NewCompiler( workflow.WithVerbose(verbose), @@ -159,8 +173,6 @@ func AddMCPTool(ctx context.Context, workflowFile string, mcpServerID string, re mcpAddLog.Print("Workflow compiled successfully") fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Workflow compiled successfully")) } - - return nil } // createMCPToolConfig creates the MCP tool configuration based on registry server info @@ -187,100 +199,92 @@ func createMCPToolConfig(server *MCPRegistryServerForProcessing, preferredTransp "registry": fmt.Sprintf("%s/servers/%s", registryURL, server.Name), } + if err := createMCPToolConfigTransport(mcpSection, server, transport); err != nil { + return nil, err + } + config["mcp"] = mcpSection + return config, nil +} + +func createMCPToolConfigTransport(mcpSection map[string]any, server *MCPRegistryServerForProcessing, transport string) error { switch transport { case "stdio": - // Handle container field (simplified Docker run) - if server.Config != nil { - if container, hasContainer := server.Config["container"]; hasContainer { - if containerStr, ok := container.(string); ok { - mcpSection["container"] = containerStr - - // Add environment variables for Docker container - if env, hasEnv := server.Config["env"]; hasEnv { - mcpSection["env"] = convertToGitHubActionsEnv(env, server.EnvironmentVariables) - } - } - } else { - // Handle regular command and args - // Use runtime_hint for command if available, otherwise fall back to Command - if server.RuntimeHint != "" { - mcpSection["command"] = server.RuntimeHint - } else if server.Command != "" { - mcpSection["command"] = server.Command - } - - // Combine runtime_arguments and package arguments for args - var allArgs []string - allArgs = append(allArgs, server.RuntimeArguments...) - allArgs = append(allArgs, server.Args...) - if len(allArgs) > 0 { - mcpSection["args"] = allArgs - } + createMCPToolConfigStdio(mcpSection, server) + return nil + case "http": + return createMCPToolConfigHTTP(mcpSection, server) + case "docker": + return createMCPToolConfigDocker(mcpSection, server) + default: + return fmt.Errorf("unsupported transport type: %s", transport) + } +} - // Add environment variables if present +func createMCPToolConfigStdio(mcpSection map[string]any, server *MCPRegistryServerForProcessing) { + if server.Config != nil { + if container, hasContainer := server.Config["container"]; hasContainer { + if containerStr, ok := container.(string); ok { + mcpSection["container"] = containerStr if env, hasEnv := server.Config["env"]; hasEnv { mcpSection["env"] = convertToGitHubActionsEnv(env, server.EnvironmentVariables) } } - } else { - // Handle command and args when no config - // Use runtime_hint for command if available, otherwise fall back to Command - if server.RuntimeHint != "" { - mcpSection["command"] = server.RuntimeHint - } else if server.Command != "" { - mcpSection["command"] = server.Command - } - - // Combine runtime_arguments and package arguments for args - var allArgs []string - allArgs = append(allArgs, server.RuntimeArguments...) - allArgs = append(allArgs, server.Args...) - if len(allArgs) > 0 { - mcpSection["args"] = allArgs - } + return } - - case "http": - // For HTTP transport, we need a URL - if server.Config != nil { - if url, hasURL := server.Config["url"]; hasURL { - mcpSection["url"] = url - } else { - return nil, errors.New("HTTP transport requires URL configuration") - } - - // Add headers if present - if headers, hasHeaders := server.Config["headers"]; hasHeaders { - mcpSection["headers"] = headers - } - } else { - return nil, errors.New("HTTP transport requires configuration") + createMCPToolConfigCommand(mcpSection, server) + if env, hasEnv := server.Config["env"]; hasEnv { + mcpSection["env"] = convertToGitHubActionsEnv(env, server.EnvironmentVariables) } + return + } + createMCPToolConfigCommand(mcpSection, server) +} - case "docker": - // For Docker transport, use container configuration - if server.Config != nil { - if container, hasContainer := server.Config["container"]; hasContainer { - mcpSection["container"] = container - } else { - return nil, errors.New("docker transport requires container configuration") - } - - // Add environment variables if present - if env, hasEnv := server.Config["env"]; hasEnv { - mcpSection["env"] = convertToGitHubActionsEnv(env, server.EnvironmentVariables) - } - } else { - return nil, errors.New("docker transport requires configuration") - } +func createMCPToolConfigCommand(mcpSection map[string]any, server *MCPRegistryServerForProcessing) { + // Use runtime_hint for command if available, otherwise fall back to Command + if server.RuntimeHint != "" { + mcpSection["command"] = server.RuntimeHint + } else if server.Command != "" { + mcpSection["command"] = server.Command + } - default: - return nil, fmt.Errorf("unsupported transport type: %s", transport) + // Combine runtime_arguments and package arguments for args + var allArgs []string + allArgs = append(allArgs, server.RuntimeArguments...) + allArgs = append(allArgs, server.Args...) + if len(allArgs) > 0 { + mcpSection["args"] = allArgs } +} - config["mcp"] = mcpSection +func createMCPToolConfigHTTP(mcpSection map[string]any, server *MCPRegistryServerForProcessing) error { + if server.Config == nil { + return errors.New("HTTP transport requires configuration") + } + if url, hasURL := server.Config["url"]; hasURL { + mcpSection["url"] = url + } else { + return errors.New("HTTP transport requires URL configuration") + } + if headers, hasHeaders := server.Config["headers"]; hasHeaders { + mcpSection["headers"] = headers + } + return nil +} - return config, nil +func createMCPToolConfigDocker(mcpSection map[string]any, server *MCPRegistryServerForProcessing) error { + if server.Config == nil { + return errors.New("docker transport requires configuration") + } + if container, hasContainer := server.Config["container"]; hasContainer { + mcpSection["container"] = container + } else { + return errors.New("docker transport requires container configuration") + } + if env, hasEnv := server.Config["env"]; hasEnv { + mcpSection["env"] = convertToGitHubActionsEnv(env, server.EnvironmentVariables) + } + return nil } // addToolToWorkflow adds a tool configuration to the workflow file diff --git a/pkg/cli/mcp_inspect.go b/pkg/cli/mcp_inspect.go index 3294a99b7ae..041cd6f1fff 100644 --- a/pkg/cli/mcp_inspect.go +++ b/pkg/cli/mcp_inspect.go @@ -31,54 +31,25 @@ func InspectWorkflowMCP(ctx context.Context, workflowFile string, serverFilter s mcpInspectLog.Printf("Inspecting workflow MCP: workflow=%s, serverFilter=%s, toolFilter=%s", workflowFile, serverFilter, toolFilter) - workflowsDir := getWorkflowsDir() - // If no workflow file specified, show available workflow files with MCP configs if workflowFile == "" { - return listWorkflowsWithMCP(workflowsDir, verbose) + return listWorkflowsWithMCP(getWorkflowsDir(), verbose) } - // Resolve the workflow file path - workflowPath, err := ResolveWorkflowPath(workflowFile) + workflowPath, err := inspectWorkflowMCPResolvePath(workflowFile) if err != nil { return err } - - // Convert to absolute path if needed - if !filepath.IsAbs(workflowPath) { - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - workflowPath = filepath.Join(cwd, workflowPath) - } - if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Inspecting MCP servers in: "+workflowPath)) } - // Use the compiler to parse the workflow file - // This automatically handles imports, merging, and validation - compiler := workflow.NewCompiler( - workflow.WithVerbose(verbose), - ) - workflowData, err := compiler.ParseWorkflowFile(workflowPath) + workflowData, err := inspectWorkflowMCPParse(workflowPath, verbose) if err != nil { - // Handle shared workflow error separately (not a fatal error for inspection) - if errors.As(err, new(*workflow.SharedWorkflowError)) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Cannot inspect shared/imported workflows directly - they must be imported by a main workflow")) - return nil - } - - // Handle redirect-only workflow error separately (not a fatal error for inspection) - if errors.As(err, new(*workflow.RedirectOnlyWorkflowError)) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Cannot inspect redirect-only workflows directly - run 'gh aw update' to follow the redirect and get the full workflow")) - return nil - } - - errMsg := fmt.Sprintf("failed to parse workflow file: %v", err) - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) - return fmt.Errorf("failed to parse workflow file: %w", err) + return err + } + if workflowData == nil { + return nil } mcpInspectLog.Printf("Workflow parsed: name=%s, has_mcp_scripts=%t", @@ -88,67 +59,17 @@ func InspectWorkflowMCP(ctx context.Context, workflowFile string, serverFilter s fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Workflow parsed successfully")) } - // Build frontmatter map from WorkflowData for MCP extraction - // This includes all merged imports and tools - frontmatterForMCP := buildFrontmatterFromWorkflowData(workflowData) - - // Extract MCP configurations from the merged frontmatter - mcpConfigs, err := parser.ExtractMCPConfigurations(frontmatterForMCP, serverFilter) + mcpConfigs, err := inspectWorkflowMCPConfigs(workflowData, serverFilter) if err != nil { - errMsg := fmt.Sprintf("failed to extract MCP configurations: %v", err) - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) - return fmt.Errorf("failed to extract MCP configurations: %w", err) + return err } - mcpInspectLog.Printf("Extracted %d MCP configs (server_filter=%q)", len(mcpConfigs), serverFilter) - - // Filter out safe-outputs MCP servers for inspection - mcpConfigs = filterOutSafeOutputs(mcpConfigs) - mcpInspectLog.Printf("After filtering safe-outputs: %d MCP configs remain", len(mcpConfigs)) - // Start mcp-scripts server if present - var mcpScriptsServerCmd *exec.Cmd - var mcpScriptsTmpDir string - if workflowData != nil && workflowData.MCPScripts != nil && len(workflowData.MCPScripts.Tools) > 0 { - // Start mcp-scripts server and add it to the list of MCP configs - config, serverCmd, tmpDir, err := startMCPScriptsServer(ctx, workflowData.MCPScripts, verbose) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to start mcp-scripts server: %v", err))) - } - mcpInspectLog.Printf("Failed to start mcp-scripts server: %v", err) - } else { - mcpScriptsServerCmd = serverCmd - mcpScriptsTmpDir = tmpDir - // Add mcp-scripts config to the list of MCP servers to inspect - mcpConfigs = append(mcpConfigs, *config) - mcpInspectLog.Print("MCP Scripts server started successfully") - } - } + scriptsServer, scriptsTmpDir, mcpConfigs := inspectWorkflowMCPStartScripts(ctx, workflowData, mcpConfigs, verbose) // Cleanup mcp-scripts server when done - if mcpScriptsServerCmd != nil { - defer func() { - if mcpScriptsServerCmd.Process != nil { - // Try graceful shutdown first - if err := mcpScriptsServerCmd.Process.Signal(os.Interrupt); err != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to send interrupt signal: %v", err))) - } - // Wait a moment for graceful shutdown (respects context cancellation) - select { - case <-time.After(mcpScriptsServerShutdownDelay): - case <-ctx.Done(): - } - // Attempt force kill (may fail if process already exited gracefully, which is fine) - _ = mcpScriptsServerCmd.Process.Kill() - } - // Cleanup temporary directory - if mcpScriptsTmpDir != "" { - if err := os.RemoveAll(mcpScriptsTmpDir); err != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to cleanup temporary directory: %v", err))) - } - } - }() + if scriptsServer != nil { + defer inspectWorkflowMCPCleanupScripts(ctx, scriptsServer, scriptsTmpDir, verbose) } if len(mcpConfigs) == 0 { @@ -161,6 +82,99 @@ func InspectWorkflowMCP(ctx context.Context, workflowFile string, serverFilter s } // Inspect each MCP server + inspectWorkflowMCPPrintHeader(workflowPath, workflowData, mcpConfigs, toolFilter) + inspectWorkflowMCPServers(mcpConfigs, toolFilter, verbose, useActionsSecrets) + return nil +} + +func inspectWorkflowMCPResolvePath(workflowFile string) (string, error) { + // Resolve the workflow file path + workflowPath, err := ResolveWorkflowPath(workflowFile) + if err != nil { + return "", err + } + // Convert to absolute path if needed + if !filepath.IsAbs(workflowPath) { + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current directory: %w", err) + } + workflowPath = filepath.Join(cwd, workflowPath) + } + return workflowPath, nil +} + +func inspectWorkflowMCPParse(workflowPath string, verbose bool) (*workflow.WorkflowData, error) { + // Use the compiler to parse the workflow file. + compiler := workflow.NewCompiler(workflow.WithVerbose(verbose)) + workflowData, err := compiler.ParseWorkflowFile(workflowPath) + if err == nil { + return workflowData, nil + } + if errors.As(err, new(*workflow.SharedWorkflowError)) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Cannot inspect shared/imported workflows directly - they must be imported by a main workflow")) + return nil, nil + } + if errors.As(err, new(*workflow.RedirectOnlyWorkflowError)) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Cannot inspect redirect-only workflows directly - run 'gh aw update' to follow the redirect and get the full workflow")) + return nil, nil + } + errMsg := fmt.Sprintf("failed to parse workflow file: %v", err) + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) + return nil, fmt.Errorf("failed to parse workflow file: %w", err) +} + +func inspectWorkflowMCPConfigs(workflowData *workflow.WorkflowData, serverFilter string) ([]parser.RegistryMCPServerConfig, error) { + // Build frontmatter map from WorkflowData for MCP extraction + frontmatterForMCP := buildFrontmatterFromWorkflowData(workflowData) + mcpConfigs, err := parser.ExtractMCPConfigurations(frontmatterForMCP, serverFilter) + if err != nil { + errMsg := fmt.Sprintf("failed to extract MCP configurations: %v", err) + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) + return nil, fmt.Errorf("failed to extract MCP configurations: %w", err) + } + mcpInspectLog.Printf("Extracted %d MCP configs (server_filter=%q)", len(mcpConfigs), serverFilter) + mcpConfigs = filterOutSafeOutputs(mcpConfigs) + mcpInspectLog.Printf("After filtering safe-outputs: %d MCP configs remain", len(mcpConfigs)) + return mcpConfigs, nil +} + +func inspectWorkflowMCPStartScripts(ctx context.Context, workflowData *workflow.WorkflowData, mcpConfigs []parser.RegistryMCPServerConfig, verbose bool) (*exec.Cmd, string, []parser.RegistryMCPServerConfig) { + if workflowData == nil || workflowData.MCPScripts == nil || len(workflowData.MCPScripts.Tools) == 0 { + return nil, "", mcpConfigs + } + config, serverCmd, tmpDir, err := startMCPScriptsServer(ctx, workflowData.MCPScripts, verbose) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to start mcp-scripts server: %v", err))) + } + mcpInspectLog.Printf("Failed to start mcp-scripts server: %v", err) + return nil, "", mcpConfigs + } + mcpConfigs = append(mcpConfigs, *config) + mcpInspectLog.Print("MCP Scripts server started successfully") + return serverCmd, tmpDir, mcpConfigs +} + +func inspectWorkflowMCPCleanupScripts(ctx context.Context, mcpScriptsServerCmd *exec.Cmd, mcpScriptsTmpDir string, verbose bool) { + if mcpScriptsServerCmd.Process != nil { + if err := mcpScriptsServerCmd.Process.Signal(os.Interrupt); err != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to send interrupt signal: %v", err))) + } + select { + case <-time.After(mcpScriptsServerShutdownDelay): + case <-ctx.Done(): + } + _ = mcpScriptsServerCmd.Process.Kill() + } + if mcpScriptsTmpDir != "" { + if err := os.RemoveAll(mcpScriptsTmpDir); err != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to cleanup temporary directory: %v", err))) + } + } +} + +func inspectWorkflowMCPPrintHeader(workflowPath string, workflowData *workflow.WorkflowData, mcpConfigs []parser.RegistryMCPServerConfig, toolFilter string) { if toolFilter != "" { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d MCP server(s), looking for tool '%s'", len(mcpConfigs), toolFilter))) } else { @@ -174,7 +188,9 @@ func InspectWorkflowMCP(ctx context.Context, workflowFile string, serverFilter s } } fmt.Fprintln(os.Stderr) +} +func inspectWorkflowMCPServers(mcpConfigs []parser.RegistryMCPServerConfig, toolFilter string, verbose bool, useActionsSecrets bool) { for i, config := range mcpConfigs { if i > 0 { fmt.Fprintln(os.Stderr) @@ -186,8 +202,6 @@ func InspectWorkflowMCP(ctx context.Context, workflowFile string, serverFilter s })) } } - - return nil } func renderMCPInspectionTree(workflowPath string, workflowData *workflow.WorkflowData, mcpConfigs []parser.RegistryMCPServerConfig) string { @@ -272,35 +286,7 @@ The command will: `, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - var workflowFile string - if len(args) > 0 { - workflowFile = args[0] - } - - verbose, _ := cmd.Flags().GetBool("verbose") - // Check for verbose flag from parent commands (root and mcp) - if cmd.Parent() != nil { - if parentVerbose, _ := cmd.Parent().PersistentFlags().GetBool("verbose"); parentVerbose { - verbose = true - } - if cmd.Parent().Parent() != nil { - if rootVerbose, _ := cmd.Parent().Parent().PersistentFlags().GetBool("verbose"); rootVerbose { - verbose = true - } - } - } - - // Validate that tool flag requires server flag - if toolFilter != "" && serverFilter == "" { - return errors.New("--tool flag requires --server flag to be specified") - } - - // Handle spawn inspector flag - if spawnInspector { - return spawnMCPInspector(cmd.Context(), workflowFile, serverFilter, verbose) - } - - return InspectWorkflowMCP(cmd.Context(), workflowFile, serverFilter, toolFilter, verbose, checkSecrets) + return newMCPInspectSubcommandRun(cmd, args, serverFilter, toolFilter, spawnInspector, checkSecrets) }, } @@ -315,6 +301,43 @@ The command will: return cmd } +func newMCPInspectSubcommandRun(cmd *cobra.Command, args []string, serverFilter, toolFilter string, spawnInspector, checkSecrets bool) error { + var workflowFile string + if len(args) > 0 { + workflowFile = args[0] + } + + verbose, _ := cmd.Flags().GetBool("verbose") + verbose = newMCPInspectSubcommandVerbose(cmd, verbose) + + // Validate that tool flag requires server flag + if toolFilter != "" && serverFilter == "" { + return errors.New("--tool flag requires --server flag to be specified") + } + + // Handle spawn inspector flag + if spawnInspector { + return spawnMCPInspector(cmd.Context(), workflowFile, serverFilter, verbose) + } + + return InspectWorkflowMCP(cmd.Context(), workflowFile, serverFilter, toolFilter, verbose, checkSecrets) +} + +func newMCPInspectSubcommandVerbose(cmd *cobra.Command, verbose bool) bool { + // Check for verbose flag from parent commands (root and mcp) + if cmd.Parent() != nil { + if parentVerbose, _ := cmd.Parent().PersistentFlags().GetBool("verbose"); parentVerbose { + verbose = true + } + if cmd.Parent().Parent() != nil { + if rootVerbose, _ := cmd.Parent().Parent().PersistentFlags().GetBool("verbose"); rootVerbose { + verbose = true + } + } + } + return verbose +} + // buildFrontmatterFromWorkflowData constructs a fully resolved frontmatter map from WorkflowData. // It uses RawFrontmatter as the base (for safe-outputs, mcp-scripts, on, etc.) and overlays the // fully merged tools and mcp-servers so that configurations from imported workflows are included. diff --git a/pkg/cli/mcp_inspect_inspector.go b/pkg/cli/mcp_inspect_inspector.go index f30c47eea6f..5f2cb65f89b 100644 --- a/pkg/cli/mcp_inspect_inspector.go +++ b/pkg/cli/mcp_inspect_inspector.go @@ -41,198 +41,201 @@ func spawnMCPInspector(ctx context.Context, workflowFile string, serverFilter st return fmt.Errorf("npx not found. Please install Node.js and npm to use the MCP inspector: %w", err) } - var mcpConfigs []parser.RegistryMCPServerConfig - var serverProcesses []*exec.Cmd - g, gctx := errgroup.WithContext(ctx) + mcpConfigs, serverProcesses, err := spawnMCPInspectorWorkflow(gctx, workflowFile, serverFilter, verbose, g) + if err != nil { + return err + } + if len(mcpConfigs) == 0 && workflowFile != "" { + return nil + } - // If workflow file is specified, extract MCP configurations and start servers - if workflowFile != "" { - // Resolve the workflow file path (supports shared workflows) - workflowPath, err := ResolveWorkflowPath(workflowFile) - if err != nil { - return err - } + // Set up cleanup function for stdio servers + defer spawnMCPInspectorCleanup(serverProcesses, verbose, g) - // Convert to absolute path if needed - if !filepath.IsAbs(workflowPath) { - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - workflowPath = filepath.Join(cwd, workflowPath) - } + return spawnMCPInspectorRun(gctx, serverProcesses) +} - // Use the compiler to parse the workflow file - // This automatically handles imports, merging, and validation - compiler := workflow.NewCompiler( - workflow.WithVerbose(verbose), - ) - workflowData, err := compiler.ParseWorkflowFile(workflowPath) +func spawnMCPInspectorWorkflow(ctx context.Context, workflowFile, serverFilter string, verbose bool, g *errgroup.Group) ([]parser.RegistryMCPServerConfig, []*exec.Cmd, error) { + if workflowFile == "" { + return nil, nil, nil + } + mcpConfigs, err := spawnMCPInspectorLoadConfigs(workflowFile, serverFilter, verbose) + if err != nil { + return nil, nil, err + } + if len(mcpConfigs) == 0 { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("No MCP servers found in workflow")) + return mcpConfigs, nil, nil + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d MCP server(s) in workflow:", len(mcpConfigs)))) + for _, config := range mcpConfigs { + fmt.Fprintf(os.Stderr, " • %s (%s)\n", config.Name, config.Type) + } + fmt.Fprintln(os.Stderr) + serverProcesses := spawnMCPInspectorStartStdioServers(ctx, mcpConfigs, verbose, g) + spawnMCPInspectorPrintConfigDetails(mcpConfigs) + return mcpConfigs, serverProcesses, nil +} + +func spawnMCPInspectorLoadConfigs(workflowFile, serverFilter string, verbose bool) ([]parser.RegistryMCPServerConfig, error) { + workflowPath, err := ResolveWorkflowPath(workflowFile) + if err != nil { + return nil, err + } + if !filepath.IsAbs(workflowPath) { + cwd, err := os.Getwd() if err != nil { - return err + return nil, fmt.Errorf("failed to get current directory: %w", err) } + workflowPath = filepath.Join(cwd, workflowPath) + } + compiler := workflow.NewCompiler(workflow.WithVerbose(verbose)) + workflowData, err := compiler.ParseWorkflowFile(workflowPath) + if err != nil { + return nil, err + } + frontmatterForMCP := buildFrontmatterFromWorkflowData(workflowData) + mcpConfigs, err := parser.ExtractMCPConfigurations(frontmatterForMCP, serverFilter) + if err != nil { + mcpInspectorLog.Printf("Failed to extract MCP configurations: %v", err) + return nil, err + } + mcpInspectorLog.Printf("Extracted %d MCP server configurations from workflow", len(mcpConfigs)) + return mcpConfigs, nil +} - // Build frontmatter map from WorkflowData for MCP extraction - // This includes all merged imports and tools - frontmatterForMCP := buildFrontmatterFromWorkflowData(workflowData) - - // Extract MCP configurations from the merged frontmatter - mcpConfigs, err = parser.ExtractMCPConfigurations(frontmatterForMCP, serverFilter) - if err != nil { - mcpInspectorLog.Printf("Failed to extract MCP configurations: %v", err) - return err +func spawnMCPInspectorStartStdioServers(ctx context.Context, mcpConfigs []parser.RegistryMCPServerConfig, verbose bool, g *errgroup.Group) []*exec.Cmd { + stdioServers := []parser.RegistryMCPServerConfig{} + for _, config := range mcpConfigs { + if config.Type == "stdio" { + stdioServers = append(stdioServers, config) } + } + if len(stdioServers) == 0 { + return nil + } + mcpInspectorLog.Printf("Starting %d stdio MCP servers", len(stdioServers)) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Starting stdio MCP servers...")) + serverProcesses := make([]*exec.Cmd, 0, len(stdioServers)) + for _, config := range stdioServers { + if cmd := spawnMCPInspectorStartServer(ctx, config, verbose, g); cmd != nil { + serverProcesses = append(serverProcesses, cmd) + } + } + time.Sleep(mcpStdioServerStartupDelay) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("All stdio servers started successfully")) + return serverProcesses +} - mcpInspectorLog.Printf("Extracted %d MCP server configurations from workflow", len(mcpConfigs)) +func spawnMCPInspectorStartServer(ctx context.Context, config parser.RegistryMCPServerConfig, verbose bool, g *errgroup.Group) *exec.Cmd { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Starting server: "+config.Name)) + } + cmd := spawnMCPInspectorCommand(ctx, config) + if cmd == nil { + return nil + } + if err := cmd.Start(); err != nil { + mcpInspectorLog.Printf("Failed to start MCP server %s: %v", config.Name, err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to start server %s: %v", config.Name, err))) + return nil + } + mcpInspectorLog.Printf("Started MCP server %s (PID: %d, type: %s)", config.Name, cmd.Process.Pid, config.Type) + spawnMCPInspectorMonitorServer(g, cmd, config.Name, verbose) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Started server: %s (PID: %d)", config.Name, cmd.Process.Pid))) + } + return cmd +} - if len(mcpConfigs) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d MCP server(s) in workflow:", len(mcpConfigs)))) - for _, config := range mcpConfigs { - fmt.Fprintf(os.Stderr, " • %s (%s)\n", config.Name, config.Type) - } - fmt.Fprintln(os.Stderr) +func spawnMCPInspectorCommand(ctx context.Context, config parser.RegistryMCPServerConfig) *exec.Cmd { + var cmd *exec.Cmd + if config.Container != "" { + args := append([]string{"run", "--rm", "-i"}, config.Args...) + cmd = mcpInspectorCommandContext(ctx, "docker", args...) + } else { + if config.Command == "" { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping server %s: no command specified", config.Name))) + return nil + } + if _, err := mcpInspectorLookPath(config.Command); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping server %s: command not found: %s", config.Name, config.Command))) + return nil + } + cmd = mcpInspectorCommandContext(ctx, config.Command, config.Args...) + } + cmd.Env = os.Environ() + for key, value := range config.Env { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, os.ExpandEnv(value))) + } + return cmd +} - // Start stdio MCP servers in the background - stdioServers := []parser.RegistryMCPServerConfig{} - for _, config := range mcpConfigs { - if config.Type == "stdio" { - stdioServers = append(stdioServers, config) - } +func spawnMCPInspectorMonitorServer(g *errgroup.Group, cmd *exec.Cmd, name string, verbose bool) { + g.Go(func() error { + defer func() { + if r := recover(); r != nil { + mcpInspectorLog.Printf("Recovered panic while waiting for MCP server %s: %v", name, r) } - - if len(stdioServers) > 0 { - mcpInspectorLog.Printf("Starting %d stdio MCP servers", len(stdioServers)) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Starting stdio MCP servers...")) - - for _, config := range stdioServers { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Starting server: "+config.Name)) - } - - // Create the command for the MCP server - var cmd *exec.Cmd - if config.Container != "" { - // Docker container mode - args := append([]string{"run", "--rm", "-i"}, config.Args...) - cmd = mcpInspectorCommandContext(gctx, "docker", args...) - } else { - // Direct command mode - if config.Command == "" { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping server %s: no command specified", config.Name))) - continue - } - // Validate the command exists before executing - if _, err := mcpInspectorLookPath(config.Command); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping server %s: command not found: %s", config.Name, config.Command))) - continue - } - // #nosec G204 -- config.Command is validated via mcpInspectorLookPath above; - // mcpInspectorCommandContext passes separate args (not shell execution), which prevents shell injection. - cmd = mcpInspectorCommandContext(gctx, config.Command, config.Args...) - } - - // Set environment variables - cmd.Env = os.Environ() - for key, value := range config.Env { - // Resolve environment variable references - resolvedValue := os.ExpandEnv(value) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, resolvedValue)) - } - - // Start the server process - if err := cmd.Start(); err != nil { - mcpInspectorLog.Printf("Failed to start MCP server %s: %v", config.Name, err) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to start server %s: %v", config.Name, err))) - continue - } - - mcpInspectorLog.Printf("Started MCP server %s (PID: %d, type: %s)", config.Name, cmd.Process.Pid, config.Type) - serverProcesses = append(serverProcesses, cmd) - - // Monitor the process in the background using errgroup for structured concurrency. - capturedCmd := cmd - capturedName := config.Name - g.Go(func() error { - defer func() { - if r := recover(); r != nil { - mcpInspectorLog.Printf("Recovered panic while waiting for MCP server %s: %v", capturedName, r) - } - mcpInspectorMonitorDone(capturedName) - }() - - // Background MCP servers are tolerant of exit errors: a server - // crashing should not abort the inspector session. Log the event - // and return nil so other monitors and the errgroup itself are - // unaffected. - if err := capturedCmd.Wait(); err != nil { - mcpInspectorLog.Printf("MCP server %s exited with error: %v", capturedName, err) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Server %s exited with error: %v", capturedName, err))) - } - } - return nil - }) - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Started server: %s (PID: %d)", config.Name, cmd.Process.Pid))) - } - } - - // Give servers a moment to start up - time.Sleep(mcpStdioServerStartupDelay) - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("All stdio servers started successfully")) + mcpInspectorMonitorDone(name) + }() + if err := cmd.Wait(); err != nil { + mcpInspectorLog.Printf("MCP server %s exited with error: %v", name, err) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Server %s exited with error: %v", name, err))) } + } + return nil + }) +} - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Configuration details for MCP inspector:")) - for _, config := range mcpConfigs { - fmt.Fprintf(os.Stderr, "\n📡 %s (%s):\n", config.Name, config.Type) - switch config.Type { - case "stdio": - if config.Container != "" { - fmt.Fprintf(os.Stderr, " Container: %s\n", config.Container) - } else { - fmt.Fprintf(os.Stderr, " Command: %s\n", config.Command) - if len(config.Args) > 0 { - fmt.Fprintf(os.Stderr, " Args: %s\n", strings.Join(config.Args, " ")) - } - } - case "http": - fmt.Fprintf(os.Stderr, " URL: %s\n", config.URL) - } - if len(config.Env) > 0 { - fmt.Fprintf(os.Stderr, " Environment Variables: %v\n", config.Env) +func spawnMCPInspectorPrintConfigDetails(mcpConfigs []parser.RegistryMCPServerConfig) { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Configuration details for MCP inspector:")) + for _, config := range mcpConfigs { + fmt.Fprintf(os.Stderr, "\n📡 %s (%s):\n", config.Name, config.Type) + switch config.Type { + case "stdio": + if config.Container != "" { + fmt.Fprintf(os.Stderr, " Container: %s\n", config.Container) + } else { + fmt.Fprintf(os.Stderr, " Command: %s\n", config.Command) + if len(config.Args) > 0 { + fmt.Fprintf(os.Stderr, " Args: %s\n", strings.Join(config.Args, " ")) } } - fmt.Fprintln(os.Stderr) - } else { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("No MCP servers found in workflow")) - return nil + case "http": + fmt.Fprintf(os.Stderr, " URL: %s\n", config.URL) + } + if len(config.Env) > 0 { + fmt.Fprintf(os.Stderr, " Environment Variables: %v\n", config.Env) } } + fmt.Fprintln(os.Stderr) +} - // Set up cleanup function for stdio servers - defer func() { - if len(serverProcesses) > 0 { - mcpInspectorLog.Printf("Cleaning up %d MCP server processes", len(serverProcesses)) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Cleaning up MCP servers...")) - for i, cmd := range serverProcesses { - if cmd.Process != nil { - if err := cmd.Process.Kill(); err != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to kill server process %d: %v", cmd.Process.Pid, err))) - } - } - // Give each process a chance to clean up - if i < len(serverProcesses)-1 { - time.Sleep(mcpProcessCleanupDelay) - } - } - if err := g.Wait(); err != nil { - mcpInspectorLog.Printf("Error from MCP server monitor goroutine: %v", err) +func spawnMCPInspectorCleanup(serverProcesses []*exec.Cmd, verbose bool, g *errgroup.Group) { + if len(serverProcesses) == 0 { + return + } + mcpInspectorLog.Printf("Cleaning up %d MCP server processes", len(serverProcesses)) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Cleaning up MCP servers...")) + for i, cmd := range serverProcesses { + if cmd.Process != nil { + if err := cmd.Process.Kill(); err != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to kill server process %d: %v", cmd.Process.Pid, err))) } } - }() + if i < len(serverProcesses)-1 { + time.Sleep(mcpProcessCleanupDelay) + } + } + if err := g.Wait(); err != nil { + mcpInspectorLog.Printf("Error from MCP server monitor goroutine: %v", err) + } +} +func spawnMCPInspectorRun(ctx context.Context, serverProcesses []*exec.Cmd) error { mcpInspectorLog.Print("Launching @modelcontextprotocol/inspector") fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Launching @modelcontextprotocol/inspector...")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Visit http://localhost:5173 after the inspector starts")) @@ -240,12 +243,10 @@ func spawnMCPInspector(ctx context.Context, workflowFile string, serverFilter st fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("%d stdio MCP server(s) are running in the background", len(serverProcesses)))) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Configure them in the inspector using the details shown above")) } - - cmd := mcpInspectorCommandContext(gctx, "npx", "@modelcontextprotocol/inspector") + cmd := mcpInspectorCommandContext(ctx, "npx", "@modelcontextprotocol/inspector") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin - err := cmd.Run() if err != nil { mcpInspectorLog.Printf("MCP inspector exited with error: %v", err) diff --git a/pkg/cli/mcp_inspect_mcp.go b/pkg/cli/mcp_inspect_mcp.go index 730b8bf8603..96019397310 100644 --- a/pkg/cli/mcp_inspect_mcp.go +++ b/pkg/cli/mcp_inspect_mcp.go @@ -138,6 +138,35 @@ func connectStdioMCPServer(ctx context.Context, config parser.RegistryMCPServerC fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Starting stdio MCP server: %s %s", config.Command, strings.Join(config.Args, " ")))) } + cmd, err := connectStdioMCPServerCommand(ctx, config) + if err != nil { + return nil, err + } + + // Create MCP client and connect + client := mcp.NewClient(mcpInspectClientImplementation(), &mcp.ClientOptions{ + Logger: logger.NewSlogLoggerWithHandler(mcpInspectServerLog), + }) + transport := &mcp.CommandTransport{Command: cmd} + + // Create a timeout context for connection + connectCtx, cancel := context.WithTimeout(ctx, MCPConnectTimeout) + defer cancel() + + session, err := client.Connect(connectCtx, transport, nil) + if err != nil { + return nil, fmt.Errorf("failed to connect to MCP server: %w", err) + } + defer session.Close() + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Successfully connected to MCP server")) + } + + return connectStdioMCPServerInfo(ctx, session, config, verbose), nil +} + +func connectStdioMCPServerCommand(ctx context.Context, config parser.RegistryMCPServerConfig) (*exec.Cmd, error) { // Validate the command exists if config.Command != "" { if _, err := exec.LookPath(config.Command); err != nil { @@ -153,7 +182,7 @@ func connectStdioMCPServer(ctx context.Context, config parser.RegistryMCPServerC cmd = exec.CommandContext(ctx, "docker", args...) } else { // Direct command mode - // #nosec G204 -- config.Command is validated via exec.LookPath above (line 138); + // #nosec G204 -- config.Command is validated via exec.LookPath above; // exec.Command with separate args (not shell execution) prevents shell injection. cmd = exec.CommandContext(ctx, config.Command, config.Args...) } @@ -166,26 +195,10 @@ func connectStdioMCPServer(ctx context.Context, config parser.RegistryMCPServerC cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, resolvedValue)) } - // Create MCP client and connect - client := mcp.NewClient(mcpInspectClientImplementation(), &mcp.ClientOptions{ - Logger: logger.NewSlogLoggerWithHandler(mcpInspectServerLog), - }) - transport := &mcp.CommandTransport{Command: cmd} - - // Create a timeout context for connection - connectCtx, cancel := context.WithTimeout(ctx, MCPConnectTimeout) - defer cancel() - - session, err := client.Connect(connectCtx, transport, nil) - if err != nil { - return nil, fmt.Errorf("failed to connect to MCP server: %w", err) - } - defer session.Close() - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Successfully connected to MCP server")) - } + return cmd, nil +} +func connectStdioMCPServerInfo(ctx context.Context, session *mcp.ClientSession, config parser.RegistryMCPServerConfig, verbose bool) *parser.MCPServerInfo { // Query server capabilities info := &parser.MCPServerInfo{ Config: config, @@ -225,7 +238,7 @@ func connectStdioMCPServer(ctx context.Context, config parser.RegistryMCPServerC // so we'll keep an empty list or try to infer from resources info.Roots = extractRootsFromResources(info.Resources) - return info, nil + return info } // connectHTTPMCPServer connects to an HTTP-based MCP server using the Go SDK @@ -234,6 +247,26 @@ func connectHTTPMCPServer(ctx context.Context, config parser.RegistryMCPServerCo fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Connecting to HTTP MCP server: "+config.URL)) } + client, transport := connectHTTPMCPServerClient(config) + + // Create a timeout context for connection + connectCtx, cancel := context.WithTimeout(ctx, MCPConnectTimeout) + defer cancel() + + session, err := client.Connect(connectCtx, transport, nil) + if err != nil { + return nil, fmt.Errorf("failed to connect to HTTP MCP server: %w", err) + } + defer session.Close() + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Successfully connected to HTTP MCP server")) + } + + return connectHTTPMCPServerInfo(ctx, session, config, verbose), nil +} + +func connectHTTPMCPServerClient(config parser.RegistryMCPServerConfig) (*mcp.Client, *mcp.StreamableClientTransport) { // Create MCP client with logger for better debugging client := mcp.NewClient(mcpInspectClientImplementation(), &mcp.ClientOptions{ Logger: logger.NewSlogLoggerWithHandler(mcpInspectServerLog), @@ -263,20 +296,10 @@ func connectHTTPMCPServer(ctx context.Context, config parser.RegistryMCPServerCo } } - // Create a timeout context for connection - connectCtx, cancel := context.WithTimeout(ctx, MCPConnectTimeout) - defer cancel() - - session, err := client.Connect(connectCtx, transport, nil) - if err != nil { - return nil, fmt.Errorf("failed to connect to HTTP MCP server: %w", err) - } - defer session.Close() - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Successfully connected to HTTP MCP server")) - } + return client, transport +} +func connectHTTPMCPServerInfo(ctx context.Context, session *mcp.ClientSession, config parser.RegistryMCPServerConfig, verbose bool) *parser.MCPServerInfo { // Query server capabilities info := &parser.MCPServerInfo{ Config: config, @@ -315,7 +338,7 @@ func connectHTTPMCPServer(ctx context.Context, config parser.RegistryMCPServerCo // Extract root URIs from resources (simple heuristic) info.Roots = extractRootsFromResources(info.Resources) - return info, nil + return info } // extractRootsFromResources infers root URIs from a list of resources by extracting @@ -357,104 +380,97 @@ func mcpInspectClientImplementation() *mcp.Implementation { // displayServerCapabilities shows the server's tools, resources, and roots in formatted tables func displayServerCapabilities(info *parser.MCPServerInfo, toolFilter string) { mcpInspectServerLog.Printf("Displaying server capabilities: tools=%d, resources=%d, toolFilter=%q", len(info.Tools), len(info.Resources), toolFilter) - // Display tools with allowed/not allowed status - if len(info.Tools) > 0 { - // If a specific tool is requested, show detailed information - if toolFilter != "" { - displayDetailedToolInfo(info, toolFilter) - } else { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("🛠️ Tool Access Status")) - - // Configure options for inspect command - // Use a slightly shorter truncation length than list-tools for better fit - opts := MCPToolTableOptions{ - TruncateLength: 50, - ShowSummary: true, - ShowVerboseHint: false, - } + displayServerCapabilitiesTools(info, toolFilter) + displayServerCapabilitiesResources(info, toolFilter) + displayServerCapabilitiesRoots(info, toolFilter) - // Render the table using the shared helper and print to stdout (structured data) - fmt.Fprint(os.Stdout, renderMCPToolTable(info, opts)) - - // Add helpful hint about how to allow tools in workflow frontmatter - displayToolAllowanceHint(info) - } + fmt.Fprintln(os.Stderr) +} - } else { +func displayServerCapabilitiesTools(info *parser.MCPServerInfo, toolFilter string) { + // Display tools with allowed/not allowed status + if len(info.Tools) == 0 { if toolFilter != "" { fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatWarningMessage(fmt.Sprintf("Tool '%s' not found", toolFilter))) } else { fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatWarningMessage("No tools available")) } + return } - // Display resources (skip if showing specific tool details) - if toolFilter == "" && len(info.Resources) > 0 { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("📚 Available Resources")) - - headers := []string{"URI", "Name", "Description", "MIME Type"} - rows := make([][]string, 0, len(info.Resources)) - - for _, resource := range info.Resources { - description := stringutil.Truncate(resource.Description, 40) - - mimeType := resource.MIMEType - if mimeType == "" { - mimeType = "N/A" - } + // If a specific tool is requested, show detailed information + if toolFilter != "" { + displayDetailedToolInfo(info, toolFilter) + return + } - rows = append(rows, []string{resource.URI, resource.Name, description, mimeType}) - } + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("🛠️ Tool Access Status")) + opts := MCPToolTableOptions{ + TruncateLength: 50, + ShowSummary: true, + ShowVerboseHint: false, + } + fmt.Fprint(os.Stdout, renderMCPToolTable(info, opts)) + displayToolAllowanceHint(info) +} - fmt.Fprint(os.Stdout, console.RenderTable(console.TableConfig{ - Headers: headers, - Rows: rows, - })) - } else if toolFilter == "" { +func displayServerCapabilitiesResources(info *parser.MCPServerInfo, toolFilter string) { + // Display resources (skip if showing specific tool details) + if toolFilter != "" { + return + } + if len(info.Resources) == 0 { fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatWarningMessage("No resources available")) + return } - // Display roots (skip if showing specific tool details) - if toolFilter == "" && len(info.Roots) > 0 { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("🌳 Available Roots")) - - headers := []string{"URI", "Name"} - rows := make([][]string, 0, len(info.Roots)) + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("📚 Available Resources")) + headers := []string{"URI", "Name", "Description", "MIME Type"} + rows := make([][]string, 0, len(info.Resources)) - for _, root := range info.Roots { - rows = append(rows, []string{root.URI, root.Name}) + for _, resource := range info.Resources { + description := stringutil.Truncate(resource.Description, 40) + mimeType := resource.MIMEType + if mimeType == "" { + mimeType = "N/A" } + rows = append(rows, []string{resource.URI, resource.Name, description, mimeType}) + } + + fmt.Fprint(os.Stdout, console.RenderTable(console.TableConfig{ + Headers: headers, + Rows: rows, + })) +} - fmt.Fprint(os.Stdout, console.RenderTable(console.TableConfig{ - Headers: headers, - Rows: rows, - })) - } else if toolFilter == "" { +func displayServerCapabilitiesRoots(info *parser.MCPServerInfo, toolFilter string) { + // Display roots (skip if showing specific tool details) + if toolFilter != "" { + return + } + if len(info.Roots) == 0 { fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatWarningMessage("No roots available")) + return } - fmt.Fprintln(os.Stderr) + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("🌳 Available Roots")) + headers := []string{"URI", "Name"} + rows := make([][]string, 0, len(info.Roots)) + for _, root := range info.Roots { + rows = append(rows, []string{root.URI, root.Name}) + } + fmt.Fprint(os.Stdout, console.RenderTable(console.TableConfig{ + Headers: headers, + Rows: rows, + })) } // displayDetailedToolInfo shows detailed information about a specific tool func displayDetailedToolInfo(info *parser.MCPServerInfo, toolName string) { // Find the specific tool - var foundTool *mcp.Tool - for _, tool := range info.Tools { - if tool.Name == toolName { - foundTool = tool - break - } - } - + foundTool := displayDetailedToolInfoFindTool(info, toolName) if foundTool == nil { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatWarningMessage(fmt.Sprintf("Tool '%s' not found", toolName))) - fmt.Fprintf(os.Stderr, "Available tools: ") - toolNames := make([]string, len(info.Tools)) - for i, tool := range info.Tools { - toolNames[i] = tool.Name - } - fmt.Fprintf(os.Stderr, "%s\n", strings.Join(toolNames, ", ")) + displayDetailedToolInfoNotFound(info, toolName) return } @@ -480,78 +496,123 @@ func displayDetailedToolInfo(info *parser.MCPServerInfo, toolName string) { fmt.Fprintf(os.Stderr, "📝 **Description:** %s\n", foundTool.Description) // Display allowance status + displayDetailedToolInfoStatus(isAllowed) + + // Display annotations if available + displayDetailedToolInfoAnnotations(foundTool) + + // Display input schema + displayDetailedToolInfoSchema("📥 Input Schema", "📥 No input schema defined", foundTool.InputSchema) + + // Display output schema + displayDetailedToolInfoSchema("📤 Output Schema", "📤 No output schema defined", foundTool.OutputSchema) + + fmt.Fprintln(os.Stderr) +} + +func displayDetailedToolInfoFindTool(info *parser.MCPServerInfo, toolName string) *mcp.Tool { + for _, tool := range info.Tools { + if tool.Name == toolName { + return tool + } + } + return nil +} + +func displayDetailedToolInfoNotFound(info *parser.MCPServerInfo, toolName string) { + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatWarningMessage(fmt.Sprintf("Tool '%s' not found", toolName))) + fmt.Fprintf(os.Stderr, "Available tools: ") + toolNames := make([]string, len(info.Tools)) + for i, tool := range info.Tools { + toolNames[i] = tool.Name + } + fmt.Fprintf(os.Stderr, "%s\n", strings.Join(toolNames, ", ")) +} + +func displayDetailedToolInfoStatus(isAllowed bool) { if isAllowed { fmt.Fprintf(os.Stderr, "✅ **Status:** Allowed\n") } else { fmt.Fprintf(os.Stderr, "🚫 **Status:** Not allowed (add to 'allowed' list in workflow frontmatter)\n") } +} - // Display annotations if available - if foundTool.Annotations != nil { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("⚙️ Tool Attributes")) - - if foundTool.Annotations.ReadOnlyHint { - fmt.Fprintf(os.Stderr, "🔒 **Read-only:** This tool does not modify its environment\n") - } else { - fmt.Fprintf(os.Stderr, "🔓 **Modifies environment:** This tool can make changes\n") - } - - if foundTool.Annotations.IdempotentHint { - fmt.Fprintf(os.Stderr, "🔄 **Idempotent:** Calling with same arguments has no additional effect\n") - } +func displayDetailedToolInfoAnnotations(foundTool *mcp.Tool) { + if foundTool.Annotations == nil { + return + } + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("⚙️ Tool Attributes")) - if foundTool.Annotations.DestructiveHint != nil { - if *foundTool.Annotations.DestructiveHint { - fmt.Fprintf(os.Stderr, "⚠️ **Destructive:** May perform destructive updates\n") - } else { - fmt.Fprintf(os.Stderr, "➕ **Additive:** Performs only additive updates\n") - } - } + if foundTool.Annotations.ReadOnlyHint { + fmt.Fprintf(os.Stderr, "🔒 **Read-only:** This tool does not modify its environment\n") + } else { + fmt.Fprintf(os.Stderr, "🔓 **Modifies environment:** This tool can make changes\n") + } - if foundTool.Annotations.OpenWorldHint != nil { - if *foundTool.Annotations.OpenWorldHint { - fmt.Fprintf(os.Stderr, "🌐 **Open world:** Interacts with external entities\n") - } else { - fmt.Fprintf(os.Stderr, "🏠 **Closed world:** Domain of interaction is closed\n") - } - } + if foundTool.Annotations.IdempotentHint { + fmt.Fprintf(os.Stderr, "🔄 **Idempotent:** Calling with same arguments has no additional effect\n") } + displayDetailedToolInfoDestructiveHint(foundTool) + displayDetailedToolInfoOpenWorldHint(foundTool) +} - // Display input schema - if foundTool.InputSchema != nil { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("📥 Input Schema")) - if schemaJSON, err := json.MarshalIndent(foundTool.InputSchema, "", " "); err == nil { - fmt.Fprintf(os.Stderr, "```json\n%s\n```\n", string(schemaJSON)) - } else { - fmt.Fprintf(os.Stderr, "Error displaying input schema: %v\n", err) - } +func displayDetailedToolInfoDestructiveHint(foundTool *mcp.Tool) { + if foundTool.Annotations.DestructiveHint == nil { + return + } + if *foundTool.Annotations.DestructiveHint { + fmt.Fprintf(os.Stderr, "⚠️ **Destructive:** May perform destructive updates\n") } else { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("📥 No input schema defined")) + fmt.Fprintf(os.Stderr, "➕ **Additive:** Performs only additive updates\n") } +} - // Display output schema - if foundTool.OutputSchema != nil { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader("📤 Output Schema")) - if schemaJSON, err := json.MarshalIndent(foundTool.OutputSchema, "", " "); err == nil { - fmt.Fprintf(os.Stderr, "```json\n%s\n```\n", string(schemaJSON)) - } else { - fmt.Fprintf(os.Stderr, "Error displaying output schema: %v\n", err) - } +func displayDetailedToolInfoOpenWorldHint(foundTool *mcp.Tool) { + if foundTool.Annotations.OpenWorldHint == nil { + return + } + if *foundTool.Annotations.OpenWorldHint { + fmt.Fprintf(os.Stderr, "🌐 **Open world:** Interacts with external entities\n") } else { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("📤 No output schema defined")) + fmt.Fprintf(os.Stderr, "🏠 **Closed world:** Domain of interaction is closed\n") } +} - fmt.Fprintln(os.Stderr) +func displayDetailedToolInfoSchema(header string, missing string, schema any) { + if schema == nil { + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage(missing)) + return + } + + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSectionHeader(header)) + if schemaJSON, err := json.MarshalIndent(schema, "", " "); err == nil { + fmt.Fprintf(os.Stderr, "```json\n%s\n```\n", string(schemaJSON)) + } else { + fmt.Fprintf(os.Stderr, "Error displaying input schema: %v\n", err) + } } // displayToolAllowanceHint shows helpful information about how to allow tools in workflow frontmatter func displayToolAllowanceHint(info *parser.MCPServerInfo) { + blockedTools := displayToolAllowanceHintBlockedTools(info) + + if len(blockedTools) > 0 { + displayToolAllowanceHintBlocked(info, blockedTools) + } else if len(info.Config.Allowed) == 0 { + displayToolAllowanceHintDefaultAllowed(info) + } else { + // All tools are explicitly allowed + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSuccessMessage("✅ All available tools are explicitly allowed in your workflow")) + } + + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("📖 For more information, see: https://github.com/github/gh-aw/blob/main/docs/tools.md")) +} + +func displayToolAllowanceHintBlockedTools(info *parser.MCPServerInfo) []string { // Create a map for quick lookup of allowed tools - allowedMap := make(map[string]struct { - }) + allowedMap := make(map[string]struct{}) for _, allowed := range info.Config.Allowed { - allowedMap[allowed] = struct { - }{} + allowedMap[allowed] = struct{}{} } // Count blocked tools and collect their names @@ -561,58 +622,52 @@ func displayToolAllowanceHint(info *parser.MCPServerInfo) { blockedTools = append(blockedTools, tool.Name) } } + return blockedTools +} - if len(blockedTools) > 0 { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("💡 To allow blocked tools, add them to your workflow frontmatter:")) - - // Show the frontmatter syntax example - fmt.Fprintf(os.Stderr, "\n") - fmt.Fprintf(os.Stderr, "```yaml\n") - fmt.Fprintf(os.Stderr, "tools:\n") - fmt.Fprintf(os.Stderr, " %s:\n", info.Config.Name) - fmt.Fprintf(os.Stderr, " allowed:\n") - - // Add currently allowed tools first (if any) - for _, allowed := range info.Config.Allowed { - fmt.Fprintf(os.Stderr, " - %s\n", allowed) - } +func displayToolAllowanceHintBlocked(info *parser.MCPServerInfo, blockedTools []string) { + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("💡 To allow blocked tools, add them to your workflow frontmatter:")) + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "```yaml\n") + fmt.Fprintf(os.Stderr, "tools:\n") + fmt.Fprintf(os.Stderr, " %s:\n", info.Config.Name) + fmt.Fprintf(os.Stderr, " allowed:\n") - // Show first few blocked tools as examples (limit to 3 for readability) - exampleCount := min(len(blockedTools), 3) + for _, allowed := range info.Config.Allowed { + fmt.Fprintf(os.Stderr, " - %s\n", allowed) + } - for i := range exampleCount { - fmt.Fprintf(os.Stderr, " - %s\n", blockedTools[i]) - } + exampleCount := min(len(blockedTools), 3) + for i := range exampleCount { + fmt.Fprintf(os.Stderr, " - %s\n", blockedTools[i]) + } - if len(blockedTools) > 3 { - fmt.Fprintf(os.Stderr, " # ... and %d more tools\n", len(blockedTools)-3) - } + if len(blockedTools) > 3 { + fmt.Fprintf(os.Stderr, " # ... and %d more tools\n", len(blockedTools)-3) + } + fmt.Fprintf(os.Stderr, "```\n") - fmt.Fprintf(os.Stderr, "```\n") + if len(blockedTools) > 3 { + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("📋 All blocked tools: "+strings.Join(blockedTools, ", "))) + } +} - if len(blockedTools) > 3 { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("📋 All blocked tools: "+strings.Join(blockedTools, ", "))) - } - } else if len(info.Config.Allowed) == 0 { - // No explicit allowed list - all tools are allowed by default - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("💡 All tools are currently allowed (no 'allowed' list specified)")) - if len(info.Tools) > 0 { - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("To restrict tools, add an 'allowed' list to your workflow frontmatter:")) - fmt.Fprintf(os.Stderr, "\n") - fmt.Fprintf(os.Stderr, "```yaml\n") - fmt.Fprintf(os.Stderr, "tools:\n") - fmt.Fprintf(os.Stderr, " %s:\n", info.Config.Name) - fmt.Fprintf(os.Stderr, " allowed:\n") - fmt.Fprintf(os.Stderr, " - %s # Allow only specific tools\n", info.Tools[0].Name) - if len(info.Tools) > 1 { - fmt.Fprintf(os.Stderr, " - %s\n", info.Tools[1].Name) - } - fmt.Fprintf(os.Stderr, "```\n") - } - } else { - // All tools are explicitly allowed - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatSuccessMessage("✅ All available tools are explicitly allowed in your workflow")) +func displayToolAllowanceHintDefaultAllowed(info *parser.MCPServerInfo) { + // No explicit allowed list - all tools are allowed by default + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("💡 All tools are currently allowed (no 'allowed' list specified)")) + if len(info.Tools) == 0 { + return } - fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("📖 For more information, see: https://github.com/github/gh-aw/blob/main/docs/tools.md")) + fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage("To restrict tools, add an 'allowed' list to your workflow frontmatter:")) + fmt.Fprintf(os.Stderr, "\n") + fmt.Fprintf(os.Stderr, "```yaml\n") + fmt.Fprintf(os.Stderr, "tools:\n") + fmt.Fprintf(os.Stderr, " %s:\n", info.Config.Name) + fmt.Fprintf(os.Stderr, " allowed:\n") + fmt.Fprintf(os.Stderr, " - %s # Allow only specific tools\n", info.Tools[0].Name) + if len(info.Tools) > 1 { + fmt.Fprintf(os.Stderr, " - %s\n", info.Tools[1].Name) + } + fmt.Fprintf(os.Stderr, "```\n") } diff --git a/pkg/cli/mcp_inspect_mcp_scripts_files.go b/pkg/cli/mcp_inspect_mcp_scripts_files.go index 04c3b6d0846..f2601e97689 100644 --- a/pkg/cli/mcp_inspect_mcp_scripts_files.go +++ b/pkg/cli/mcp_inspect_mcp_scripts_files.go @@ -15,6 +15,27 @@ import ( func writeMCPScriptsFiles(dir string, mcpScriptsConfig *workflow.MCPScriptsConfig, verbose bool) error { mcpInspectLog.Printf("Writing mcp-scripts files to: %s", dir) + if err := writeMCPScriptsFilesLogsDir(dir); err != nil { + return err + } + if err := writeMCPScriptsFilesJavaScript(dir, verbose); err != nil { + return err + } + if err := writeMCPScriptsFilesToolsConfig(dir, mcpScriptsConfig, verbose); err != nil { + return err + } + if err := writeMCPScriptsFilesServerScript(dir, mcpScriptsConfig, verbose); err != nil { + return err + } + if err := writeMCPScriptsFilesToolHandlers(dir, mcpScriptsConfig, verbose); err != nil { + return err + } + + mcpInspectLog.Printf("Successfully wrote all mcp-scripts files") + return nil +} + +func writeMCPScriptsFilesLogsDir(dir string) error { // Create logs directory logsDir := filepath.Join(dir, "logs") if err := os.MkdirAll(logsDir, constants.DirPermPublic); err != nil { @@ -22,7 +43,10 @@ func writeMCPScriptsFiles(dir string, mcpScriptsConfig *workflow.MCPScriptsConfi fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) return fmt.Errorf("failed to create logs directory: %w", err) } + return nil +} +func writeMCPScriptsFilesJavaScript(dir string, verbose bool) error { // Write JavaScript dependencies that are needed jsFiles := []struct { name string @@ -40,41 +64,26 @@ func writeMCPScriptsFiles(dir string, mcpScriptsConfig *workflow.MCPScriptsConfi } for _, jsFile := range jsFiles { - filePath := filepath.Join(dir, jsFile.name) - if err := os.WriteFile(filePath, []byte(jsFile.content), constants.FilePermPublic); err != nil { - errMsg := fmt.Sprintf("failed to write %s: %v", jsFile.name, err) - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) - return fmt.Errorf("failed to write %s: %w", jsFile.name, err) - } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Wrote "+jsFile.name)) + if err := writeMCPScriptsFilesFile(dir, jsFile.name, jsFile.content, constants.FilePermPublic, verbose); err != nil { + return err } } + return nil +} +func writeMCPScriptsFilesToolsConfig(dir string, mcpScriptsConfig *workflow.MCPScriptsConfig, verbose bool) error { // Generate and write tools.json toolsJSON := workflow.GenerateMCPScriptsToolsConfig(mcpScriptsConfig) - toolsPath := filepath.Join(dir, "tools.json") - if err := os.WriteFile(toolsPath, []byte(toolsJSON), constants.FilePermPublic); err != nil { - errMsg := fmt.Sprintf("failed to write tools.json: %v", err) - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) - return fmt.Errorf("failed to write tools.json: %w", err) - } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Wrote tools.json")) - } + return writeMCPScriptsFilesFile(dir, "tools.json", toolsJSON, constants.FilePermPublic, verbose) +} +func writeMCPScriptsFilesServerScript(dir string, mcpScriptsConfig *workflow.MCPScriptsConfig, verbose bool) error { // Generate and write mcp-server.cjs entry point mcpServerScript := workflow.GenerateMCPScriptsMCPServerScript(mcpScriptsConfig) - mcpServerPath := filepath.Join(dir, "mcp-server.cjs") - if err := os.WriteFile(mcpServerPath, []byte(mcpServerScript), constants.FilePermExecutable); err != nil { - errMsg := fmt.Sprintf("failed to write mcp-server.cjs: %v", err) - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) - return fmt.Errorf("failed to write mcp-server.cjs: %w", err) - } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Wrote mcp-server.cjs")) - } + return writeMCPScriptsFilesFile(dir, "mcp-server.cjs", mcpServerScript, constants.FilePermExecutable, verbose) +} +func writeMCPScriptsFilesToolHandlers(dir string, mcpScriptsConfig *workflow.MCPScriptsConfig, verbose bool) error { // Generate and write tool handler files for toolName, toolConfig := range mcpScriptsConfig.Tools { var content string @@ -107,7 +116,18 @@ func writeMCPScriptsFiles(dir string, mcpScriptsConfig *workflow.MCPScriptsConfi fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Wrote tool handler: %s%s", toolName, extension))) } } + return nil +} - mcpInspectLog.Printf("Successfully wrote all mcp-scripts files") +func writeMCPScriptsFilesFile(dir string, name string, content string, mode os.FileMode, verbose bool) error { + filePath := filepath.Join(dir, name) + if err := os.WriteFile(filePath, []byte(content), mode); err != nil { + errMsg := fmt.Sprintf("failed to write %s: %v", name, err) + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) + return fmt.Errorf("failed to write %s: %w", name, err) + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Wrote "+name)) + } return nil } diff --git a/pkg/cli/mcp_inspect_mcp_scripts_server.go b/pkg/cli/mcp_inspect_mcp_scripts_server.go index 4b13854ebbd..a99e7ad854c 100644 --- a/pkg/cli/mcp_inspect_mcp_scripts_server.go +++ b/pkg/cli/mcp_inspect_mcp_scripts_server.go @@ -131,11 +131,46 @@ func startMCPScriptsHTTPServer(dir string, port int, verbose bool) (*exec.Cmd, e func startMCPScriptsServer(ctx context.Context, mcpScriptsConfig *workflow.MCPScriptsConfig, verbose bool) (*parser.RegistryMCPServerConfig, *exec.Cmd, string, error) { mcpInspectLog.Printf("Starting mcp-scripts server with %d tools", len(mcpScriptsConfig.Tools)) + tmpDir, err := startMCPScriptsServerPrepareFiles(mcpScriptsConfig, verbose) + if err != nil { + return nil, nil, "", err + } + + serverCmd, port, err := startMCPScriptsServerLaunch(tmpDir, verbose) + if err != nil { + return nil, nil, "", err + } + + // Wait for the server to start up + if err := waitForServerReady(ctx, port, 5*time.Second, verbose); err != nil { + startMCPScriptsServerCleanupServer(serverCmd, tmpDir, verbose) + return nil, nil, "", err + } + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("MCP Scripts HTTP server started successfully")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Server running on: http://localhost:%d", port))) + } + + // Create MCP server config for the mcp-scripts server + config := &parser.RegistryMCPServerConfig{ + BaseMCPServerConfig: types.BaseMCPServerConfig{ + Type: "http", + URL: fmt.Sprintf("http://localhost:%d", port), + Env: make(map[string]string), + }, + Name: "mcpscripts", + } + + return config, serverCmd, tmpDir, nil +} + +func startMCPScriptsServerPrepareFiles(mcpScriptsConfig *workflow.MCPScriptsConfig, verbose bool) (string, error) { // Check if node is available if _, err := exec.LookPath("node"); err != nil { errMsg := "node not found. Please install Node.js to run the mcp-scripts MCP server" fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) - return nil, nil, "", fmt.Errorf("node not found. Please install Node.js to run the mcp-scripts MCP server: %w", err) + return "", fmt.Errorf("node not found. Please install Node.js to run the mcp-scripts MCP server: %w", err) } if verbose { @@ -147,7 +182,7 @@ func startMCPScriptsServer(ctx context.Context, mcpScriptsConfig *workflow.MCPSc if err != nil { errMsg := fmt.Sprintf("failed to create temporary directory: %v", err) fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) - return nil, nil, "", fmt.Errorf("failed to create temporary directory: %w", err) + return "", fmt.Errorf("failed to create temporary directory: %w", err) } if verbose { @@ -156,70 +191,49 @@ func startMCPScriptsServer(ctx context.Context, mcpScriptsConfig *workflow.MCPSc } } - // Write mcp-scripts files to temporary directory if err := writeMCPScriptsFiles(tmpDir, mcpScriptsConfig, verbose); err != nil { - // Clean up temporary directory on error - if err := os.RemoveAll(tmpDir); err != nil && verbose { - mcpInspectLog.Printf("Warning: failed to clean up temporary directory %s: %v", tmpDir, err) - } + startMCPScriptsServerCleanupDir(tmpDir, verbose) errMsg := fmt.Sprintf("failed to write mcp-scripts files: %v", err) fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) - return nil, nil, "", fmt.Errorf("failed to write mcp-scripts files: %w", err) + return "", fmt.Errorf("failed to write mcp-scripts files: %w", err) } + return tmpDir, nil +} +func startMCPScriptsServerLaunch(tmpDir string, verbose bool) (*exec.Cmd, int, error) { // Find an available port for the HTTP server port := findAvailablePort(mcpScriptsStartPort, verbose) if port == 0 { - if err := os.RemoveAll(tmpDir); err != nil && verbose { - mcpInspectLog.Printf("Warning: failed to clean up temporary directory %s: %v", tmpDir, err) - } + startMCPScriptsServerCleanupDir(tmpDir, verbose) errMsg := "failed to find an available port for the HTTP server" fmt.Fprintln(os.Stderr, console.FormatErrorMessage(errMsg)) - return nil, nil, "", errors.New("failed to find an available port for the HTTP server") + return nil, 0, errors.New("failed to find an available port for the HTTP server") } if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Using port %d for mcp-scripts HTTP server", port))) } - // Start the HTTP server serverCmd, err := startMCPScriptsHTTPServer(tmpDir, port, verbose) if err != nil { - // Clean up temporary directory on error - if rmErr := os.RemoveAll(tmpDir); rmErr != nil && verbose { - mcpInspectLog.Printf("Warning: failed to clean up temporary directory %s: %v", tmpDir, rmErr) - } - return nil, nil, "", fmt.Errorf("failed to start mcp-scripts HTTP server: %w", err) + startMCPScriptsServerCleanupDir(tmpDir, verbose) + return nil, 0, fmt.Errorf("failed to start mcp-scripts HTTP server: %w", err) } + return serverCmd, port, nil +} - // Wait for the server to start up - if err := waitForServerReady(ctx, port, 5*time.Second, verbose); err != nil { - if serverCmd.Process != nil { - // Kill the process and log warning if it fails - if err := serverCmd.Process.Kill(); err != nil && verbose { - mcpInspectLog.Printf("Warning: failed to kill server process %d: %v", serverCmd.Process.Pid, err) - } - } - if err := os.RemoveAll(tmpDir); err != nil && verbose { - mcpInspectLog.Printf("Warning: failed to clean up temporary directory %s: %v", tmpDir, err) +func startMCPScriptsServerCleanupServer(serverCmd *exec.Cmd, tmpDir string, verbose bool) { + if serverCmd.Process != nil { + // Kill the process and log warning if it fails + if err := serverCmd.Process.Kill(); err != nil && verbose { + mcpInspectLog.Printf("Warning: failed to kill server process %d: %v", serverCmd.Process.Pid, err) } - return nil, nil, "", err - } - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("MCP Scripts HTTP server started successfully")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Server running on: http://localhost:%d", port))) } + startMCPScriptsServerCleanupDir(tmpDir, verbose) +} - // Create MCP server config for the mcp-scripts server - config := &parser.RegistryMCPServerConfig{ - BaseMCPServerConfig: types.BaseMCPServerConfig{ - Type: "http", - URL: fmt.Sprintf("http://localhost:%d", port), - Env: make(map[string]string), - }, - Name: "mcpscripts", +func startMCPScriptsServerCleanupDir(tmpDir string, verbose bool) { + if err := os.RemoveAll(tmpDir); err != nil && verbose { + mcpInspectLog.Printf("Warning: failed to clean up temporary directory %s: %v", tmpDir, err) } - - return config, serverCmd, tmpDir, nil } diff --git a/pkg/cli/mcp_list.go b/pkg/cli/mcp_list.go index 631db772316..7c59c758bc4 100644 --- a/pkg/cli/mcp_list.go +++ b/pkg/cli/mcp_list.go @@ -19,28 +19,29 @@ import ( var mcpListLog = logger.New("cli:mcp_list") +type mcpWorkflowData struct { + name string + serverCount int + serverNames []string +} + // ListWorkflowMCP lists MCP servers defined in a workflow func ListWorkflowMCP(workflowFile string, verbose bool) error { mcpListLog.Printf("Listing MCP servers: workflow=%s, verbose=%t", workflowFile, verbose) // Determine the workflow directory and file workflowsDir := constants.GetWorkflowDir() - var workflowPath string - - if workflowFile != "" { - // Resolve the workflow file path - var err error - workflowPath, err = ResolveWorkflowPath(workflowFile) - if err != nil { - mcpListLog.Printf("Failed to resolve workflow path: %v", err) - return err - } - mcpListLog.Printf("Resolved workflow path: %s", workflowPath) - } else { + + if workflowFile == "" { // No specific workflow file provided, list all workflows with MCP servers mcpListLog.Print("No workflow file specified, listing all workflows with MCP servers") return listWorkflowsWithMCPServers(workflowsDir, verbose) } + workflowPath, err := listWorkflowMCPWorkflowPath(workflowFile) + if err != nil { + return err + } + // Parse the specific workflow file and extract MCP configurations frontmatter, mcpConfigs, err := loadWorkflowMCPConfigs(workflowPath, "") if err != nil { @@ -54,78 +55,84 @@ func ListWorkflowMCP(workflowFile string, verbose bool) error { return nil } - // Check if workflow has network access configured - hasNetworkAccess := checkNetworkAccess(frontmatter.Frontmatter) + listWorkflowMCPDisplay(workflowPath, mcpConfigs, checkNetworkAccess(frontmatter.Frontmatter), verbose) - // Display the MCP servers - if verbose { - // Create detailed table for verbose mode - headers := []string{"Server Name", "Type", "Status", "Tools Count", "Network Access", "Command/URL"} - rows := make([][]string, 0, len(mcpConfigs)) - - for _, config := range mcpConfigs { - status := determineConfigStatus(config) - toolsCount := formatToolsCount(config.Allowed) - networkAccess := formatNetworkAccess(hasNetworkAccess) - - commandOrURL := "" - if config.Command != "" { - commandOrURL = config.Command - } else if config.URL != "" { - commandOrURL = config.URL - } else if config.Container != "" { - commandOrURL = config.Container - } - // Truncate if too long - commandOrURL = stringutil.Truncate(commandOrURL, 40) - - rows = append(rows, []string{ - config.Name, - config.Type, - status, - toolsCount, - networkAccess, - commandOrURL, - }) - } + if !verbose { + fmt.Fprintf(os.Stderr, "\nRun 'gh aw mcp list %s --verbose' for detailed information\n", workflowFile) + } - tableConfig := console.TableConfig{ - Title: "MCP servers in " + filepath.Base(workflowPath), - Headers: headers, - Rows: rows, - } - fmt.Fprint(os.Stderr, console.RenderTable(tableConfig)) + return nil +} + +func listWorkflowMCPWorkflowPath(workflowFile string) (string, error) { + // Resolve the workflow file path + workflowPath, err := ResolveWorkflowPath(workflowFile) + if err != nil { + mcpListLog.Printf("Failed to resolve workflow path: %v", err) + return "", err + } + mcpListLog.Printf("Resolved workflow path: %s", workflowPath) + return workflowPath, nil +} + +func listWorkflowMCPDisplay(workflowPath string, mcpConfigs []parser.RegistryMCPServerConfig, hasNetworkAccess bool, verbose bool) { + if verbose { + listWorkflowMCPDisplayVerbose(workflowPath, mcpConfigs, hasNetworkAccess) } else { - // Simple table for basic mode - headers := []string{"Server Name", "Status", "Tools Count", "Network Access"} - rows := make([][]string, 0, len(mcpConfigs)) - - for _, config := range mcpConfigs { - status := determineConfigStatus(config) - toolsCount := formatToolsCount(config.Allowed) - networkAccess := formatNetworkAccess(hasNetworkAccess) - - rows = append(rows, []string{ - config.Name, - status, - toolsCount, - networkAccess, - }) - } + listWorkflowMCPDisplayBasic(workflowPath, mcpConfigs, hasNetworkAccess) + } +} - tableConfig := console.TableConfig{ - Title: "MCP servers in " + filepath.Base(workflowPath), - Headers: headers, - Rows: rows, - } - fmt.Fprint(os.Stderr, console.RenderTable(tableConfig)) +func listWorkflowMCPDisplayVerbose(workflowPath string, mcpConfigs []parser.RegistryMCPServerConfig, hasNetworkAccess bool) { + headers := []string{"Server Name", "Type", "Status", "Tools Count", "Network Access", "Command/URL"} + rows := make([][]string, 0, len(mcpConfigs)) + for _, config := range mcpConfigs { + rows = append(rows, []string{ + config.Name, + config.Type, + determineConfigStatus(config), + formatToolsCount(config.Allowed), + formatNetworkAccess(hasNetworkAccess), + listWorkflowMCPCommandOrURL(config), + }) } - if !verbose { - fmt.Fprintf(os.Stderr, "\nRun 'gh aw mcp list %s --verbose' for detailed information\n", workflowFile) + fmt.Fprint(os.Stderr, console.RenderTable(console.TableConfig{ + Title: "MCP servers in " + filepath.Base(workflowPath), + Headers: headers, + Rows: rows, + })) +} + +func listWorkflowMCPDisplayBasic(workflowPath string, mcpConfigs []parser.RegistryMCPServerConfig, hasNetworkAccess bool) { + headers := []string{"Server Name", "Status", "Tools Count", "Network Access"} + rows := make([][]string, 0, len(mcpConfigs)) + for _, config := range mcpConfigs { + rows = append(rows, []string{ + config.Name, + determineConfigStatus(config), + formatToolsCount(config.Allowed), + formatNetworkAccess(hasNetworkAccess), + }) } - return nil + fmt.Fprint(os.Stderr, console.RenderTable(console.TableConfig{ + Title: "MCP servers in " + filepath.Base(workflowPath), + Headers: headers, + Rows: rows, + })) +} + +func listWorkflowMCPCommandOrURL(config parser.RegistryMCPServerConfig) string { + commandOrURL := "" + if config.Command != "" { + commandOrURL = config.Command + } else if config.URL != "" { + commandOrURL = config.URL + } else if config.Container != "" { + commandOrURL = config.Container + } + return stringutil.Truncate(commandOrURL, 40) } // listWorkflowsWithMCPServers shows available workflow files that contain MCP configurations @@ -137,28 +144,7 @@ func listWorkflowsWithMCPServers(workflowsDir string, verbose bool) error { return err } - var workflowData []struct { - name string - serverCount int - serverNames []string - } - var totalMCPCount int - - for _, result := range results { - serverNames := sliceutil.Map(result.MCPConfigs, func(config parser.RegistryMCPServerConfig) string { return config.Name }) - - workflowData = append(workflowData, struct { - name string - serverCount int - serverNames []string - }{ - name: result.BaseName, - serverCount: len(result.MCPConfigs), - serverNames: serverNames, - }) - totalMCPCount += len(result.MCPConfigs) - } - + workflowData := listWorkflowsWithMCPServersData(results) if len(workflowData) == 0 { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No workflows with MCP servers found")) return nil @@ -175,62 +161,71 @@ func listWorkflowsWithMCPServers(workflowsDir string, verbose bool) error { // If interactive selection failed or was cancelled, fall back to table display mcpListLog.Printf("Interactive selection failed or cancelled, showing table: %v", err) - // Display results in table format - if verbose { - // Detailed table with server names - headers := []string{"Workflow", "Server Count", "MCP Servers"} - rows := make([][]string, 0, len(workflowData)) - - for _, workflow := range workflowData { - serverList := strings.Join(workflow.serverNames, ", ") - // Truncate if too long - serverList = stringutil.Truncate(serverList, 50) - - rows = append(rows, []string{ - workflow.name, - strconv.Itoa(workflow.serverCount), - serverList, - }) - } + listWorkflowsWithMCPServersDisplay(workflowData, verbose) + fmt.Fprintf(os.Stderr, "Run 'gh aw mcp list ' to list MCP servers in a specific workflow\n") - tableConfig := console.TableConfig{ - Headers: headers, - Rows: rows, - } - fmt.Fprint(os.Stderr, console.RenderTable(tableConfig)) - } else { - // Simple table with just workflow names and counts - headers := []string{"Workflow", "Server Count"} - rows := make([][]string, 0, len(workflowData)) - - for _, workflow := range workflowData { - rows = append(rows, []string{ - workflow.name, - strconv.Itoa(workflow.serverCount), - }) - } + return nil +} - tableConfig := console.TableConfig{ - Headers: headers, - Rows: rows, - } - fmt.Fprint(os.Stderr, console.RenderTable(tableConfig)) +func listWorkflowsWithMCPServersData(results []WorkflowMCPMetadata) []mcpWorkflowData { + var workflowData []mcpWorkflowData + for _, result := range results { + serverNames := sliceutil.Map(result.MCPConfigs, func(config parser.RegistryMCPServerConfig) string { return config.Name }) + workflowData = append(workflowData, mcpWorkflowData{ + name: result.BaseName, + serverCount: len(result.MCPConfigs), + serverNames: serverNames, + }) + } + return workflowData +} + +func listWorkflowsWithMCPServersDisplay(workflowData []mcpWorkflowData, verbose bool) { + if verbose { + listWorkflowsWithMCPServersDisplayVerbose(workflowData) + } else { + listWorkflowsWithMCPServersDisplayBasic(workflowData) } if !verbose { fmt.Fprintf(os.Stderr, "\nRun 'gh aw mcp list --verbose' for detailed information\n") } - fmt.Fprintf(os.Stderr, "Run 'gh aw mcp list ' to list MCP servers in a specific workflow\n") +} - return nil +func listWorkflowsWithMCPServersDisplayVerbose(workflowData []mcpWorkflowData) { + headers := []string{"Workflow", "Server Count", "MCP Servers"} + rows := make([][]string, 0, len(workflowData)) + for _, workflow := range workflowData { + serverList := stringutil.Truncate(strings.Join(workflow.serverNames, ", "), 50) + rows = append(rows, []string{ + workflow.name, + strconv.Itoa(workflow.serverCount), + serverList, + }) + } + fmt.Fprint(os.Stderr, console.RenderTable(console.TableConfig{ + Headers: headers, + Rows: rows, + })) +} + +func listWorkflowsWithMCPServersDisplayBasic(workflowData []mcpWorkflowData) { + headers := []string{"Workflow", "Server Count"} + rows := make([][]string, 0, len(workflowData)) + for _, workflow := range workflowData { + rows = append(rows, []string{ + workflow.name, + strconv.Itoa(workflow.serverCount), + }) + } + fmt.Fprint(os.Stderr, console.RenderTable(console.TableConfig{ + Headers: headers, + Rows: rows, + })) } // showInteractiveMCPWorkflowSelection displays an interactive list of workflows with MCP servers -func showInteractiveMCPWorkflowSelection(workflows []struct { - name string - serverCount int - serverNames []string -}, verbose bool) (string, error) { +func showInteractiveMCPWorkflowSelection(workflows []mcpWorkflowData, verbose bool) (string, error) { mcpListLog.Printf("Showing interactive MCP workflow selection: workflows=%d", len(workflows)) // Convert workflow data to ListItems diff --git a/pkg/cli/mcp_list_tools.go b/pkg/cli/mcp_list_tools.go index 78dc1a6d7f9..724c25550b0 100644 --- a/pkg/cli/mcp_list_tools.go +++ b/pkg/cli/mcp_list_tools.go @@ -24,29 +24,19 @@ const ( // ListToolsForMCP lists available tools for a specific MCP server func ListToolsForMCP(workflowFile string, mcpServerName string, verbose bool) error { mcpListToolsLog.Printf("Listing tools for MCP server: %s, workflow: %s", mcpServerName, workflowFile) - workflowsDir := getWorkflowsDir() // If no workflow file specified, search for workflows containing the MCP server if workflowFile == "" { + workflowsDir := getWorkflowsDir() mcpListToolsLog.Printf("No workflow file specified, searching in: %s", workflowsDir) return findWorkflowsWithMCPServer(workflowsDir, mcpServerName, verbose) } - // Resolve the workflow file path - workflowPath, err := ResolveWorkflowPath(workflowFile) + workflowPath, err := listToolsForMCPWorkflowPath(workflowFile) if err != nil { return err } - // Convert to absolute path if needed - if !filepath.IsAbs(workflowPath) { - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - workflowPath = filepath.Join(cwd, workflowPath) - } - if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessageStderr(fmt.Sprintf("Looking for MCP server '%s' in: %s", mcpServerName, workflowPath))) } @@ -59,24 +49,9 @@ func ListToolsForMCP(workflowFile string, mcpServerName string, verbose bool) er mcpListToolsLog.Printf("Found %d MCP configs in workflow, searching for server: %s", len(mcpConfigs), mcpServerName) - // Find the specific MCP server - var targetConfig *parser.RegistryMCPServerConfig - for _, config := range mcpConfigs { - if strings.EqualFold(config.Name, mcpServerName) { - targetConfig = &config - break - } - } - + targetConfig := listToolsForMCPFindConfig(mcpConfigs, mcpServerName) if targetConfig == nil { - mcpListToolsLog.Printf("MCP server %q not found in workflow %q", mcpServerName, filepath.Base(workflowPath)) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("MCP server '%s' not found in workflow '%s'", mcpServerName, filepath.Base(workflowPath)))) - - // Show available servers - if len(mcpConfigs) > 0 { - serverNames := sliceutil.Map(mcpConfigs, func(config parser.RegistryMCPServerConfig) string { return config.Name }) - fmt.Fprintln(os.Stderr, console.FormatInfoMessageStderr("Available MCP servers: "+strings.Join(serverNames, ", "))) - } + listToolsForMCPDisplayMissingServer(workflowPath, mcpServerName, mcpConfigs) return nil } @@ -104,6 +79,45 @@ func ListToolsForMCP(workflowFile string, mcpServerName string, verbose bool) er return nil } +func listToolsForMCPWorkflowPath(workflowFile string) (string, error) { + // Resolve the workflow file path + workflowPath, err := ResolveWorkflowPath(workflowFile) + if err != nil { + return "", err + } + + // Convert to absolute path if needed + if !filepath.IsAbs(workflowPath) { + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current directory: %w", err) + } + workflowPath = filepath.Join(cwd, workflowPath) + } + + return workflowPath, nil +} + +func listToolsForMCPFindConfig(mcpConfigs []parser.RegistryMCPServerConfig, mcpServerName string) *parser.RegistryMCPServerConfig { + for _, config := range mcpConfigs { + if strings.EqualFold(config.Name, mcpServerName) { + return &config + } + } + return nil +} + +func listToolsForMCPDisplayMissingServer(workflowPath string, mcpServerName string, mcpConfigs []parser.RegistryMCPServerConfig) { + mcpListToolsLog.Printf("MCP server %q not found in workflow %q", mcpServerName, filepath.Base(workflowPath)) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("MCP server '%s' not found in workflow '%s'", mcpServerName, filepath.Base(workflowPath)))) + + // Show available servers + if len(mcpConfigs) > 0 { + serverNames := sliceutil.Map(mcpConfigs, func(config parser.RegistryMCPServerConfig) string { return config.Name }) + fmt.Fprintln(os.Stderr, console.FormatInfoMessageStderr("Available MCP servers: "+strings.Join(serverNames, ", "))) + } +} + // findWorkflowsWithMCPServer searches for workflows containing a specific MCP server func findWorkflowsWithMCPServer(workflowsDir string, mcpServerName string, verbose bool) error { // Scan workflows for MCP configurations, filtering by server name @@ -177,22 +191,7 @@ func NewMCPListToolsSubcommand() *cobra.Command { cmd := &cobra.Command{ Use: "list-tools [workflow]", Short: "List available tools for a specific MCP server, or find workflows using it", - Long: `List available tools for a specific MCP server, or find workflows using it. - -When no workflow argument is provided, this command searches workflows in .github/workflows -for references to the specified MCP server and returns matching workflow IDs. -When a workflow is provided, it connects to the specified MCP server and displays all -available tools. It reuses the same infrastructure as 'mcp inspect' to establish -connections and query server capabilities. - -The workflow-id-or-file can be: -- A workflow ID (basename without .md extension, e.g., "weekly-research") -- A file path (e.g., "weekly-research.md" or ".github/workflows/weekly-research.md") - -The command will: -- Parse the workflow to find the specified MCP server configuration -- Connect to the MCP server using the same logic as 'mcp inspect' -- Display available tools with their descriptions and allowance status`, + Long: newMCPListToolsSubcommandLong(), Example: ` gh aw mcp list-tools --server github # Search for workflows containing the 'github' MCP server gh aw mcp list-tools weekly-research --server github # List tools for 'github' server in weekly-research.md gh aw mcp list-tools issue-triage --server safe-outputs # List tools for 'safe-outputs' server in issue-triage.md @@ -232,3 +231,22 @@ The command will: return cmd } + +func newMCPListToolsSubcommandLong() string { + return `List available tools for a specific MCP server, or find workflows using it. + +When no workflow argument is provided, this command searches workflows in .github/workflows +for references to the specified MCP server and returns matching workflow IDs. +When a workflow is provided, it connects to the specified MCP server and displays all +available tools. It reuses the same infrastructure as 'mcp inspect' to establish +connections and query server capabilities. + +The workflow-id-or-file can be: +- A workflow ID (basename without .md extension, e.g., "weekly-research") +- A file path (e.g., "weekly-research.md" or ".github/workflows/weekly-research.md") + +The command will: +- Parse the workflow to find the specified MCP server configuration +- Connect to the MCP server using the same logic as 'mcp inspect' +- Display available tools with their descriptions and allowance status` +} diff --git a/pkg/cli/mcp_logs_guardrail.go b/pkg/cli/mcp_logs_guardrail.go index eb2c91b9888..16fb30c4439 100644 --- a/pkg/cli/mcp_logs_guardrail.go +++ b/pkg/cli/mcp_logs_guardrail.go @@ -41,6 +41,32 @@ type MCPLogsGuardrailResponse struct { // The cache directory is kept separate from the artifact download directory so // these summary files are never included in artifact uploads. func buildLogsFileResponse(outputStr string) string { + if errResponse := buildLogsFileResponseEnsureCacheDir(); errResponse != "" { + return errResponse + } + + // Use SHA256 of content as filename for content-addressed deduplication. + filePath := buildLogsFileResponsePath(outputStr) + + if errResponse := buildLogsFileResponseEnsureCacheFile(filePath, outputStr); errResponse != "" { + return errResponse + } + + response := MCPLogsGuardrailResponse{ + Message: fmt.Sprintf("Logs data has been written to '%s'. Use the file_path to read the full data.", filePath), + FilePath: filePath, + } + + responseJSON, err := json.MarshalIndent(response, "", " ") + if err != nil { + mcpLogsGuardrailLog.Printf("Failed to marshal logs file response: %v", err) + return fmt.Sprintf(`{"message":"Logs data written to file","file_path":%q}`, filePath) + } + + return string(responseJSON) +} + +func buildLogsFileResponseEnsureCacheDir() string { // Verify or create the cache directory. Use Lstat to detect symlinks and // refuse to follow them, hardening against symlink-based directory attacks. if info, err := os.Lstat(mcpLogsCacheDir); err == nil { @@ -63,12 +89,16 @@ func buildLogsFileResponse(outputStr string) string { mcpLogsGuardrailLog.Printf("Failed to set logs cache directory permissions: %v", chmodErr) return buildLogsFileErrorResponse(fmt.Sprintf("failed to set logs cache directory permissions: %v", chmodErr)) } + return "" +} - // Use SHA256 of content as filename for content-addressed deduplication. +func buildLogsFileResponsePath(outputStr string) string { sum := sha256.Sum256([]byte(outputStr)) fileName := hex.EncodeToString(sum[:]) + ".json" - filePath := filepath.Join(mcpLogsCacheDir, fileName) + return filepath.Join(mcpLogsCacheDir, fileName) +} +func buildLogsFileResponseEnsureCacheFile(filePath string, outputStr string) string { // Skip writing if a file with identical content already exists. if fileInfo, err := os.Lstat(filePath); err == nil { if fileInfo.Mode()&os.ModeSymlink != 0 { @@ -83,24 +113,7 @@ func buildLogsFileResponse(outputStr string) string { } mcpLogsGuardrailLog.Printf("Logs data already cached at: %s", filePath) } else if os.IsNotExist(err) { - // Write with O_EXCL to avoid following symlinks or races. - writeErr := func() (err error) { - f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, constants.FilePermPublic) - if err != nil { - return fmt.Errorf("failed to create logs cache file: %w", err) - } - defer func() { - if closeErr := f.Close(); closeErr != nil && err == nil { - err = fmt.Errorf("failed to write logs data to file: %w", closeErr) - } - }() - - if _, err = f.WriteString(outputStr); err != nil { - return fmt.Errorf("failed to write logs data to file: %w", err) - } - return nil - }() - if writeErr != nil { + if writeErr := buildLogsFileResponseWriteCacheFile(filePath, outputStr); writeErr != nil { mcpLogsGuardrailLog.Printf("Failed to populate logs cache file %s: %v", filePath, writeErr) _ = os.Remove(filePath) return buildLogsFileErrorResponse(writeErr.Error()) @@ -116,18 +129,25 @@ func buildLogsFileResponse(outputStr string) string { return buildLogsFileErrorResponse(fmt.Sprintf("failed to access logs cache file: %v", err)) } - response := MCPLogsGuardrailResponse{ - Message: fmt.Sprintf("Logs data has been written to '%s'. Use the file_path to read the full data.", filePath), - FilePath: filePath, - } + return "" +} - responseJSON, err := json.MarshalIndent(response, "", " ") +func buildLogsFileResponseWriteCacheFile(filePath string, outputStr string) (err error) { + // Write with O_EXCL to avoid following symlinks or races. + f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, constants.FilePermPublic) if err != nil { - mcpLogsGuardrailLog.Printf("Failed to marshal logs file response: %v", err) - return fmt.Sprintf(`{"message":"Logs data written to file","file_path":%q}`, filePath) + return fmt.Errorf("failed to create logs cache file: %w", err) } + defer func() { + if closeErr := f.Close(); closeErr != nil && err == nil { + err = fmt.Errorf("failed to write logs data to file: %w", closeErr) + } + }() - return string(responseJSON) + if _, err = f.WriteString(outputStr); err != nil { + return fmt.Errorf("failed to write logs data to file: %w", err) + } + return nil } // buildLogsFileErrorResponse returns a JSON error response when file writing fails. diff --git a/pkg/cli/mcp_permissions.go b/pkg/cli/mcp_permissions.go index a1b3dfbe0bc..d90f414a43b 100644 --- a/pkg/cli/mcp_permissions.go +++ b/pkg/cli/mcp_permissions.go @@ -74,31 +74,17 @@ func checkActorPermission(ctx context.Context, actor string, validateActor bool, // If validation is enabled but no actor is specified, deny access if actor == "" { - mcpLog.Printf("Tool %s: access denied (no actor specified, validation enabled)", toolName) - return newMCPError(jsonrpc.CodeInvalidRequest, "permission denied: insufficient role", map[string]any{ - "error": "GITHUB_ACTOR environment variable not set", - "tool": toolName, - "reason": "This tool requires at least write access to the repository. Set GITHUB_ACTOR environment variable to enable access.", - }) + return checkActorPermissionMissingActor(toolName) } // Get repository using cached lookup repo, err := getRepository(ctx) if err != nil { - mcpLog.Printf("Tool %s: failed to get repository context, denying access: %v", toolName, err) - return newMCPError(jsonrpc.CodeInternalError, "permission check failed", map[string]any{ - "error": err.Error(), - "tool": toolName, - "reason": "Could not determine repository context to verify permissions. Ensure you are running from within a git repository with gh authenticated.", - }) + return checkActorPermissionRepositoryError(toolName, err) } if repo == "" { - mcpLog.Printf("Tool %s: no repository context, denying access", toolName) - return newMCPError(jsonrpc.CodeInvalidRequest, "permission check failed", map[string]any{ - "tool": toolName, - "reason": "No repository context available. Run from within a git repository.", - }) + return checkActorPermissionMissingRepository(toolName) } // Query actor's role in the repository with caching @@ -107,30 +93,64 @@ func checkActorPermission(ctx context.Context, actor string, validateActor bool, permission, err := queryActorRole(ctx, actor, repo) if err != nil { - mcpLog.Printf("Tool %s: failed to query actor role, denying access: %v", toolName, err) - return newMCPError(jsonrpc.CodeInternalError, "permission denied: unable to verify repository access", map[string]any{ - "error": err.Error(), - "tool": toolName, - "actor": actor, - "repository": repo, - "reason": "Failed to query actor's repository permissions from GitHub API.", - }) + return checkActorPermissionQueryError(toolName, actor, repo, err) } // Check if the actor has write+ access if !hasWriteAccess(permission) { - mcpLog.Printf("Tool %s: access denied for actor %s (permission: %s, requires: write+)", toolName, actor, permission) - return newMCPError(jsonrpc.CodeInvalidRequest, "permission denied: insufficient role", map[string]any{ - "error": "insufficient repository permissions", - "tool": toolName, - "actor": actor, - "repository": repo, - "role": permission, - "required": "write, maintain, or admin", - "reason": fmt.Sprintf("Actor %s has %s access to %s. This tool requires at least write access.", actor, permission, repo), - }) + return checkActorPermissionInsufficientRole(toolName, actor, repo, permission) } mcpLog.Printf("Tool %s: access allowed for actor %s (permission: %s)", toolName, actor, permission) return nil } + +func checkActorPermissionMissingActor(toolName string) error { + mcpLog.Printf("Tool %s: access denied (no actor specified, validation enabled)", toolName) + return newMCPError(jsonrpc.CodeInvalidRequest, "permission denied: insufficient role", map[string]any{ + "error": "GITHUB_ACTOR environment variable not set", + "tool": toolName, + "reason": "This tool requires at least write access to the repository. Set GITHUB_ACTOR environment variable to enable access.", + }) +} + +func checkActorPermissionRepositoryError(toolName string, err error) error { + mcpLog.Printf("Tool %s: failed to get repository context, denying access: %v", toolName, err) + return newMCPError(jsonrpc.CodeInternalError, "permission check failed", map[string]any{ + "error": err.Error(), + "tool": toolName, + "reason": "Could not determine repository context to verify permissions. Ensure you are running from within a git repository with gh authenticated.", + }) +} + +func checkActorPermissionMissingRepository(toolName string) error { + mcpLog.Printf("Tool %s: no repository context, denying access", toolName) + return newMCPError(jsonrpc.CodeInvalidRequest, "permission check failed", map[string]any{ + "tool": toolName, + "reason": "No repository context available. Run from within a git repository.", + }) +} + +func checkActorPermissionQueryError(toolName string, actor string, repo string, err error) error { + mcpLog.Printf("Tool %s: failed to query actor role, denying access: %v", toolName, err) + return newMCPError(jsonrpc.CodeInternalError, "permission denied: unable to verify repository access", map[string]any{ + "error": err.Error(), + "tool": toolName, + "actor": actor, + "repository": repo, + "reason": "Failed to query actor's repository permissions from GitHub API.", + }) +} + +func checkActorPermissionInsufficientRole(toolName string, actor string, repo string, permission string) error { + mcpLog.Printf("Tool %s: access denied for actor %s (permission: %s, requires: write+)", toolName, actor, permission) + return newMCPError(jsonrpc.CodeInvalidRequest, "permission denied: insufficient role", map[string]any{ + "error": "insufficient repository permissions", + "tool": toolName, + "actor": actor, + "repository": repo, + "role": permission, + "required": "write, maintain, or admin", + "reason": fmt.Sprintf("Actor %s has %s access to %s. This tool requires at least write access.", actor, permission, repo), + }) +} diff --git a/pkg/cli/mcp_registry.go b/pkg/cli/mcp_registry.go index 7b144e0e7b5..8b56082adef 100644 --- a/pkg/cli/mcp_registry.go +++ b/pkg/cli/mcp_registry.go @@ -69,13 +69,30 @@ func (c *MCPRegistryClient) createRegistryRequest(ctx context.Context, method, u func (c *MCPRegistryClient) SearchServers(ctx context.Context, query string) ([]MCPRegistryServerForProcessing, error) { mcpRegistryLog.Printf("Searching MCP servers: query=%q", query) - // Always use servers endpoint for listing all servers searchURL := c.registryURL + "/servers" + response, err := c.searchServersFetchResponse(ctx, searchURL) + if err != nil { + return nil, err + } + + // Convert servers to flattened format and filter by status + mcpRegistryLog.Printf("Processing %d servers from registry", len(response.Servers)) + servers := searchServersProcessResponse(response) + + if query != "" { + return searchServersFilterByQuery(servers, query), nil + } + if err := c.searchServersValidateMinimumCount(servers); err != nil { + return nil, err + } + return servers, nil +} +func (c *MCPRegistryClient) searchServersFetchResponse(ctx context.Context, searchURL string) (ServerListResponse, error) { // Create HTTP request with proper headers req, err := c.createRegistryRequest(ctx, "GET", searchURL) if err != nil { - return nil, fmt.Errorf("failed to create registry request: %w", err) + return ServerListResponse{}, fmt.Errorf("failed to create registry request: %w", err) } // Make HTTP request with spinner @@ -86,175 +103,176 @@ func (c *MCPRegistryClient) SearchServers(ctx context.Context, query string) ([] if err != nil { spinner.Stop() - return nil, fmt.Errorf("failed to search MCP registry: %w", err) + return ServerListResponse{}, fmt.Errorf("failed to search MCP registry: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { spinner.Stop() - body, _ := io.ReadAll(resp.Body) - // Provide more helpful error messages for common HTTP status codes - switch resp.StatusCode { - case http.StatusForbidden: - return nil, fmt.Errorf("MCP registry access forbidden (403): %s\nThis may be due to network or firewall restrictions", string(body)) - case http.StatusUnauthorized: - return nil, fmt.Errorf("MCP registry access unauthorized (401): %s\nAuthentication may be required", string(body)) - case http.StatusNotFound: - return nil, fmt.Errorf("MCP registry endpoint not found (404): %s\nPlease verify the registry URL is correct", string(body)) - case http.StatusTooManyRequests: - return nil, fmt.Errorf("MCP registry rate limit exceeded (429): %s\nPlease try again later", string(body)) - default: - return nil, fmt.Errorf("MCP registry returned status %d: %s", resp.StatusCode, string(body)) - } + return ServerListResponse{}, searchServersStatusError(resp) } // Parse response body, err := io.ReadAll(resp.Body) if err != nil { spinner.Stop() - return nil, fmt.Errorf("failed to read registry response: %w", err) + return ServerListResponse{}, fmt.Errorf("failed to read registry response: %w", err) } var response ServerListResponse if err := json.Unmarshal(body, &response); err != nil { spinner.Stop() - return nil, fmt.Errorf("failed to parse registry response: %w", err) + return ServerListResponse{}, fmt.Errorf("failed to parse registry response: %w", err) } // Stop spinner with success message spinner.StopWithMessage(fmt.Sprintf("✓ Fetched %d servers from registry", len(response.Servers))) + return response, nil +} - // Convert servers to flattened format and filter by status - mcpRegistryLog.Printf("Processing %d servers from registry", len(response.Servers)) +func searchServersStatusError(resp *http.Response) error { + body, _ := io.ReadAll(resp.Body) + // Provide more helpful error messages for common HTTP status codes + switch resp.StatusCode { + case http.StatusForbidden: + return fmt.Errorf("MCP registry access forbidden (403): %s\nThis may be due to network or firewall restrictions", string(body)) + case http.StatusUnauthorized: + return fmt.Errorf("MCP registry access unauthorized (401): %s\nAuthentication may be required", string(body)) + case http.StatusNotFound: + return fmt.Errorf("MCP registry endpoint not found (404): %s\nPlease verify the registry URL is correct", string(body)) + case http.StatusTooManyRequests: + return fmt.Errorf("MCP registry rate limit exceeded (429): %s\nPlease try again later", string(body)) + default: + return fmt.Errorf("MCP registry returned status %d: %s", resp.StatusCode, string(body)) + } +} + +func searchServersProcessResponse(response ServerListResponse) []MCPRegistryServerForProcessing { servers := make([]MCPRegistryServerForProcessing, 0, len(response.Servers)) for _, serverResp := range response.Servers { - server := serverResp.Server - // Only include active servers (check in _meta) - if meta, ok := serverResp.Meta["io.modelcontextprotocol.registry/official"].(map[string]any); ok { - if status, ok := meta["status"].(string); ok && status != StatusActive { - continue - } + if !searchServersIsActive(serverResp) { + continue } - processedServer := MCPRegistryServerForProcessing{ - Name: server.Name, - Description: server.Description, + servers = append(servers, searchServersProcessServer(serverResp)) + } + return servers +} + +func searchServersIsActive(serverResp ServerResponse) bool { + if meta, ok := serverResp.Meta["io.modelcontextprotocol.registry/official"].(map[string]any); ok { + if status, ok := meta["status"].(string); ok && status != StatusActive { + return false } + } + return true +} + +func searchServersProcessServer(serverResp ServerResponse) MCPRegistryServerForProcessing { + server := serverResp.Server + processedServer := MCPRegistryServerForProcessing{ + Name: server.Name, + Description: server.Description, + } + + // Set repository URL if available + if server.Repository != nil && server.Repository.URL != "" { + processedServer.Repository = server.Repository.URL + } + + if len(server.Packages) > 0 { + searchServersProcessPackage(&processedServer, server.Packages[0]) + } else if len(server.Remotes) > 0 { + searchServersProcessRemote(&processedServer, server.Remotes[0]) + } else { + processedServer.Transport = "stdio" // default fallback + } + return processedServer +} + +func searchServersProcessPackage(processedServer *MCPRegistryServerForProcessing, pkg MCPPackage) { + if pkg.Transport != nil { + processedServer.Transport = pkg.Transport.Type + } + if processedServer.Transport == "" { + processedServer.Transport = "stdio" // default fallback + } - // Set repository URL if available - if server.Repository != nil && server.Repository.URL != "" { - processedServer.Repository = server.Repository.URL + processedServer.Command = pkg.Identifier + processedServer.RuntimeHint = pkg.RuntimeHint + processedServer.RuntimeArguments = searchServersPositionalArguments(pkg.RuntimeArguments) + processedServer.Args = searchServersPositionalArguments(pkg.PackageArguments) + + if len(pkg.EnvironmentVariables) > 0 { + processedServer.Config = make(map[string]any) + envVars := make(map[string]any) + for _, envVar := range pkg.EnvironmentVariables { + envVars[envVar.Name] = searchServersVariableValue(envVar) } + processedServer.Config["env"] = envVars + processedServer.EnvironmentVariables = pkg.EnvironmentVariables + } +} + +func searchServersProcessRemote(processedServer *MCPRegistryServerForProcessing, remote Remote) { + processedServer.Transport = remote.Type + processedServer.Config = map[string]any{ + "url": remote.URL, + } - // Extract transport and config from first package if available - if len(server.Packages) > 0 { - pkg := server.Packages[0] - - // Use transport type from package - if pkg.Transport != nil { - processedServer.Transport = pkg.Transport.Type - } - if processedServer.Transport == "" { - processedServer.Transport = "stdio" // default fallback - } - - // Set command from package identifier - processedServer.Command = pkg.Identifier - - // Set runtime hint (used for the actual command execution) - processedServer.RuntimeHint = pkg.RuntimeHint - - // Extract runtime arguments - var runtimeArgs []string - for _, arg := range pkg.RuntimeArguments { - if arg.Type == ArgumentTypePositional && arg.Value != "" { - runtimeArgs = append(runtimeArgs, arg.Value) - } - } - processedServer.RuntimeArguments = runtimeArgs - - // Extract string values from package arguments as command args - var args []string - for _, arg := range pkg.PackageArguments { - if arg.Type == ArgumentTypePositional && arg.Value != "" { - args = append(args, arg.Value) - } - } - processedServer.Args = args - - // Convert environment variables to config - if len(pkg.EnvironmentVariables) > 0 { - processedServer.Config = make(map[string]any) - envVars := make(map[string]any) - - for _, envVar := range pkg.EnvironmentVariables { - // Use name as key, and create a placeholder value for secrets - if envVar.IsSecret { - envVars[envVar.Name] = fmt.Sprintf("${%s}", envVar.Name) - } else if envVar.Default != "" { - envVars[envVar.Name] = envVar.Default - } else { - envVars[envVar.Name] = fmt.Sprintf("${%s}", envVar.Name) - } - } - processedServer.Config["env"] = envVars - - // Preserve environment variable metadata for proper GitHub Actions conversion - processedServer.EnvironmentVariables = pkg.EnvironmentVariables - } - } else if len(server.Remotes) > 0 { - // Handle remote servers - remote := server.Remotes[0] - processedServer.Transport = remote.Type - processedServer.Config = map[string]any{ - "url": remote.URL, - } - - // Add headers if present - if len(remote.Headers) > 0 { - headers := make(map[string]any) - for _, header := range remote.Headers { - if header.IsSecret { - headers[header.Name] = fmt.Sprintf("${%s}", header.Name) - } else if header.Default != "" { - headers[header.Name] = header.Default - } else { - headers[header.Name] = fmt.Sprintf("${%s}", header.Name) - } - } - processedServer.Config["headers"] = headers - } - } else { - processedServer.Transport = "stdio" // default fallback + // Add headers if present + if len(remote.Headers) > 0 { + headers := make(map[string]any) + for _, header := range remote.Headers { + headers[header.Name] = searchServersVariableValue(header) } + processedServer.Config["headers"] = headers + } +} - servers = append(servers, processedServer) +func searchServersVariableValue(variable EnvironmentVariable) any { + if variable.IsSecret { + return fmt.Sprintf("${%s}", variable.Name) } + if variable.Default != "" { + return variable.Default + } + return fmt.Sprintf("${%s}", variable.Name) +} - // Apply local filtering if query is provided - if query != "" { - var filteredServers []MCPRegistryServerForProcessing - queryLower := strings.ToLower(query) - - for _, server := range servers { - // Check if query matches name or description (case-insensitive) - if strings.Contains(strings.ToLower(server.Name), queryLower) || - strings.Contains(strings.ToLower(server.Description), queryLower) { - filteredServers = append(filteredServers, server) - } +func searchServersPositionalArguments(arguments []Argument) []string { + var args []string + for _, arg := range arguments { + if arg.Type == ArgumentTypePositional && arg.Value != "" { + args = append(args, arg.Value) } + } + return args +} - mcpRegistryLog.Printf("Filtered to %d servers matching query", len(filteredServers)) - return filteredServers, nil +func searchServersFilterByQuery(servers []MCPRegistryServerForProcessing, query string) []MCPRegistryServerForProcessing { + var filteredServers []MCPRegistryServerForProcessing + queryLower := strings.ToLower(query) + + for _, server := range servers { + // Check if query matches name or description (case-insensitive) + if strings.Contains(strings.ToLower(server.Name), queryLower) || + strings.Contains(strings.ToLower(server.Description), queryLower) { + filteredServers = append(filteredServers, server) + } } + mcpRegistryLog.Printf("Filtered to %d servers matching query", len(filteredServers)) + return filteredServers +} + +func (c *MCPRegistryClient) searchServersValidateMinimumCount(servers []MCPRegistryServerForProcessing) error { // Validate minimum server count for production registry // Note: This validation helps detect issues with the registry API, but we make it more lenient // to accommodate potential changes in the registry size if strings.Contains(c.registryURL, "api.mcp.github.com") && len(servers) < 10 { - return nil, fmt.Errorf("registry validation failed: expected at least 10 servers from production registry, got %d\nThis may indicate an issue with the registry API or access restrictions", len(servers)) + return fmt.Errorf("registry validation failed: expected at least 10 servers from production registry, got %d\nThis may indicate an issue with the registry API or access restrictions", len(servers)) } - - return servers, nil + return nil } diff --git a/pkg/cli/mcp_registry_list.go b/pkg/cli/mcp_registry_list.go index 47d2b132a1f..5409acd89b1 100644 --- a/pkg/cli/mcp_registry_list.go +++ b/pkg/cli/mcp_registry_list.go @@ -18,6 +18,26 @@ func listAvailableServers(ctx context.Context, registryURL string, verbose bool) // Create registry client registryClient := NewMCPRegistryClient(registryURL) + servers, err := listAvailableServersFetch(ctx, registryClient, verbose) + if err != nil { + return err + } + + mcpRegistryListLog.Printf("Retrieved %d servers from registry", len(servers)) + listAvailableServersVerbose(servers, verbose) + + if len(servers) == 0 { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("No MCP servers found in the registry")) + return nil + } + + listAvailableServersRender(registryClient.registryURL, servers) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Usage: gh aw mcp add ")) + + return nil +} + +func listAvailableServersFetch(ctx context.Context, registryClient *MCPRegistryClient, verbose bool) ([]MCPRegistryServerForProcessing, error) { // Search for all servers (empty query) if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Fetching available MCP servers from registry: "+registryClient.registryURL)) @@ -26,25 +46,24 @@ func listAvailableServers(ctx context.Context, registryURL string, verbose bool) servers, err := registryClient.SearchServers(ctx, "") if err != nil { mcpRegistryListLog.Printf("Failed to fetch MCP servers: %v", err) - return fmt.Errorf("failed to fetch MCP servers: %w", err) + return nil, fmt.Errorf("failed to fetch MCP servers: %w", err) } + return servers, nil +} - mcpRegistryListLog.Printf("Retrieved %d servers from registry", len(servers)) - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Retrieved %d servers from registry", len(servers)))) - if len(servers) > 0 { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("First server example - Name: '%s', Description: '%s'", - servers[0].Name, servers[0].Description))) - } +func listAvailableServersVerbose(servers []MCPRegistryServerForProcessing, verbose bool) { + if !verbose { + return } - if len(servers) == 0 { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("No MCP servers found in the registry")) - return nil + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Retrieved %d servers from registry", len(servers)))) + if len(servers) > 0 { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("First server example - Name: '%s', Description: '%s'", + servers[0].Name, servers[0].Description))) } +} - // Prepare table data +func listAvailableServersRender(registryURL string, servers []MCPRegistryServerForProcessing) { headers := []string{"Name", "Description"} rows := make([][]string, 0, len(servers)) @@ -69,7 +88,7 @@ func listAvailableServers(ctx context.Context, registryURL string, verbose bool) // Create and render table tableConfig := console.TableConfig{ - Title: "MCP registry: " + registryClient.registryURL, + Title: "MCP registry: " + registryURL, Headers: headers, Rows: rows, ShowTotal: true, @@ -77,7 +96,4 @@ func listAvailableServers(ctx context.Context, registryURL string, verbose bool) } fmt.Fprint(os.Stderr, console.RenderTable(tableConfig)) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Usage: gh aw mcp add ")) - - return nil } diff --git a/pkg/cli/mcp_secrets.go b/pkg/cli/mcp_secrets.go index bc6cb5af743..2866d70ea41 100644 --- a/pkg/cli/mcp_secrets.go +++ b/pkg/cli/mcp_secrets.go @@ -16,9 +16,41 @@ var mcpSecretsLog = logger.New("cli:mcp_secrets") func checkAndSuggestSecrets(toolConfig map[string]any, verbose bool) error { mcpSecretsLog.Print("Checking and suggesting secrets for MCP tool configuration") + requiredSecrets := checkAndSuggestSecretsRequired(toolConfig) + if len(requiredSecrets) == 0 { + mcpSecretsLog.Print("No required secrets found in tool configuration") + return nil + } + mcpSecretsLog.Printf("Found %d required secrets in configuration", len(requiredSecrets)) + + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Checking repository secrets...")) + } + + missingSecrets, err := checkAndSuggestSecretsMissing(requiredSecrets, verbose) + if err != nil { + return err + } + + // Suggest CLI commands for missing secrets + if len(missingSecrets) > 0 { + mcpSecretsLog.Printf("Found %d missing secrets", len(missingSecrets)) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("The following secrets are required but not found in the repository:")) + for _, secretName := range missingSecrets { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("To add %s secret:", secretName))) + fmt.Fprintln(os.Stderr, console.FormatCommandMessage("gh secret set "+secretName)) + } + } else if verbose { + mcpSecretsLog.Print("All required secrets are available in repository") + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("All required secrets are available in the repository")) + } + + return nil +} + +func checkAndSuggestSecretsRequired(toolConfig map[string]any) []string { // Extract environment variables from the tool config var requiredSecrets []string - if mcpSection, ok := toolConfig["mcp"].(map[string]any); ok { if env, hasEnv := mcpSection["env"].(map[string]string); hasEnv { for _, value := range env { @@ -30,17 +62,10 @@ func checkAndSuggestSecrets(toolConfig map[string]any, verbose bool) error { } } } + return requiredSecrets +} - if len(requiredSecrets) == 0 { - mcpSecretsLog.Print("No required secrets found in tool configuration") - return nil - } - mcpSecretsLog.Printf("Found %d required secrets in configuration", len(requiredSecrets)) - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Checking repository secrets...")) - } - +func checkAndSuggestSecretsMissing(requiredSecrets []string, verbose bool) ([]string, error) { // Check each secret using GitHub CLI var missingSecrets []string for _, secretName := range requiredSecrets { @@ -51,28 +76,14 @@ func checkAndSuggestSecrets(toolConfig map[string]any, verbose bool) error { if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Repository secrets check skipped (insufficient permissions)")) } - return nil + return nil, nil } - return err + return nil, err } if !exists { missingSecrets = append(missingSecrets, secretName) } } - - // Suggest CLI commands for missing secrets - if len(missingSecrets) > 0 { - mcpSecretsLog.Printf("Found %d missing secrets", len(missingSecrets)) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("The following secrets are required but not found in the repository:")) - for _, secretName := range missingSecrets { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("To add %s secret:", secretName))) - fmt.Fprintln(os.Stderr, console.FormatCommandMessage("gh secret set "+secretName)) - } - } else if verbose { - mcpSecretsLog.Print("All required secrets are available in repository") - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("All required secrets are available in the repository")) - } - - return nil + return missingSecrets, nil } diff --git a/pkg/cli/mcp_server.go b/pkg/cli/mcp_server.go index ff1409cc31d..ee5bca093c9 100644 --- a/pkg/cli/mcp_server.go +++ b/pkg/cli/mcp_server.go @@ -17,8 +17,23 @@ type execCmdFunc func(ctx context.Context, args ...string) *exec.Cmd // createMCPServer creates and configures the MCP server with all tools func createMCPServer(cmdPath string, actor string, validateActor bool, manifestCacheFile string, env []string) *mcp.Server { - // Helper function to execute command with proper path - execCmd := func(ctx context.Context, args ...string) *exec.Cmd { + execCmd := createMCPServerExecCmd(cmdPath, env) + createMCPServerLogActor(actor, validateActor) + server := createMCPServerInstance() + + if !createMCPServerRegisterTools(server, execCmd, actor, validateActor, manifestCacheFile) { + return server + } + + // Add receiving middleware to transform raw JSON-schema "additional properties" + // validation errors into helpful messages with "Did you mean?" suggestions. + server.AddReceivingMiddleware(argumentValidationMiddleware(mcpToolParams())) + + return server +} + +func createMCPServerExecCmd(cmdPath string, env []string) execCmdFunc { + return func(ctx context.Context, args ...string) *exec.Cmd { var cmd *exec.Cmd if cmdPath != "" { // Use custom command path @@ -32,25 +47,25 @@ func createMCPServer(cmdPath string, actor string, validateActor bool, manifestC } return cmd } +} +func createMCPServerLogActor(actor string, validateActor bool) { // Log actor and validation settings - if validateActor { - if actor != "" { - mcpLog.Printf("Actor validation enabled: actor=%s (logs/audit tools will check permissions)", actor) - } else { - mcpLog.Print("Actor validation enabled: no actor specified (logs/audit tools will deny access)") - } + if validateActor && actor != "" { + mcpLog.Printf("Actor validation enabled: actor=%s (logs/audit tools will check permissions)", actor) + } else if validateActor { + mcpLog.Print("Actor validation enabled: no actor specified (logs/audit tools will deny access)") + } else if actor != "" { + mcpLog.Printf("Actor validation disabled: actor=%s (logs/audit tools will allow access)", actor) } else { - if actor != "" { - mcpLog.Printf("Actor validation disabled: actor=%s (logs/audit tools will allow access)", actor) - } else { - mcpLog.Print("Actor validation disabled: no actor specified (logs/audit tools will allow access)") - } + mcpLog.Print("Actor validation disabled: no actor specified (logs/audit tools will allow access)") } +} +func createMCPServerInstance() *mcp.Server { // Create MCP server with capabilities and logging // Note: Schema caching is automatic in go-sdk v1.3.0+ (eliminates repeated reflection overhead) - server := mcp.NewServer(&mcp.Implementation{ + return mcp.NewServer(&mcp.Implementation{ Name: "gh-aw", Version: GetVersion(), }, &mcp.ServerOptions{ @@ -61,25 +76,27 @@ func createMCPServer(cmdPath string, actor string, validateActor bool, manifestC }, Logger: logger.NewSlogLoggerWithHandler(mcpLog), }) +} +func createMCPServerRegisterTools(server *mcp.Server, execCmd execCmdFunc, actor string, validateActor bool, manifestCacheFile string) bool { // Register read-only tools registerStatusTool(server) if err := registerCompileTool(server, execCmd, manifestCacheFile); err != nil { - return server + return false } // Register privileged tools (require write+ access) if err := registerLogsTool(server, execCmd, actor, validateActor); err != nil { - return server + return false } if err := registerAuditTool(server, execCmd, actor, validateActor); err != nil { - return server + return false } if err := registerAuditDiffTool(server, execCmd, actor, validateActor); err != nil { - return server + return false } // Register remaining read-only tools @@ -90,10 +107,5 @@ func createMCPServer(cmdPath string, actor string, validateActor bool, manifestC registerAddTool(server, execCmd) registerUpdateTool(server, execCmd) registerFixTool(server, execCmd) - - // Add receiving middleware to transform raw JSON-schema "additional properties" - // validation errors into helpful messages with "Did you mean?" suggestions. - server.AddReceivingMiddleware(argumentValidationMiddleware(mcpToolParams())) - - return server + return true } diff --git a/pkg/cli/mcp_server_command.go b/pkg/cli/mcp_server_command.go index d712aa38aba..644b15620a6 100644 --- a/pkg/cli/mcp_server_command.go +++ b/pkg/cli/mcp_server_command.go @@ -73,7 +73,31 @@ func runMCPServer(ctx context.Context, port int, cmdPath string, validateActor b // Get actor from environment variable actor := os.Getenv("GITHUB_ACTOR") //nolint:osgetenvlibrary + runMCPServerLogStartup(actor, port, validateActor) + cmdPath = runMCPServerCommandPath(cmdPath) + runMCPServerLogWorkingDirectory() + runMCPServerValidateConfiguration(ctx, cmdPath, mcpServerEnv) + + manifestCacheFile, cleanupManifestCache := runMCPServerManifestCache() + if cleanupManifestCache != nil { + defer cleanupManifestCache() + } + + // Create the server configuration + server := createMCPServer(cmdPath, actor, validateActor, manifestCacheFile, mcpServerEnv) + + if port > 0 { + // Run HTTP server with SSE transport + return runHTTPServer(server, port) + } + + // Run stdio transport + mcpLog.Print("MCP server ready on stdio") + return server.Run(ctx, &mcp.StdioTransport{}) +} + +func runMCPServerLogStartup(actor string, port int, validateActor bool) { if validateActor { mcpLog.Printf("Actor validation enabled (--validate-actor flag)") } @@ -89,63 +113,59 @@ func runMCPServer(ctx context.Context, port int, cmdPath string, validateActor b } else { mcpLog.Print("Starting MCP server with stdio transport") } +} +func runMCPServerCommandPath(cmdPath string) string { // Determine, log, and validate the binary path only if --cmd flag is not provided // When --cmd is provided, the user explicitly specified the binary path to use - if cmdPath == "" { - // Attempt to detect the binary path and assign it to cmdPath - // This ensures createMCPServer receives the actual binary path instead of falling back to "gh aw" - detectedPath, err := logAndValidateBinaryPath() - if err == nil && detectedPath != "" { - cmdPath = detectedPath - mcpLog.Printf("Using detected binary path: %s", cmdPath) - } + if cmdPath != "" { + return cmdPath } + detectedPath, err := logAndValidateBinaryPath() + if err == nil && detectedPath != "" { + mcpLog.Printf("Using detected binary path: %s", detectedPath) + return detectedPath + } + return cmdPath +} + +func runMCPServerLogWorkingDirectory() { // Log current working directory if cwd, err := os.Getwd(); err == nil { mcpLog.Printf("Current working directory: %s", cwd) } else { mcpLog.Printf("WARNING: Failed to get current working directory: %v", err) } +} - // Validate that the CLI and secrets are properly configured - // Note: Validation failures are logged as warnings but don't prevent server startup - // This allows the server to start in test environments or non-repository directories +func runMCPServerValidateConfiguration(ctx context.Context, cmdPath string, mcpServerEnv []string) { + // Validate that the CLI and secrets are properly configured. + // Validation failures are logged as warnings but don't prevent server startup. if err := validateMCPServerConfiguration(ctx, cmdPath, mcpServerEnv); err != nil { mcpLog.Printf("Configuration validation warning: %v", err) } +} +func runMCPServerManifestCache() (string, func()) { // Pre-cache lock-file manifests at startup, before any agent can modify the working tree. // The cache is serialised to a temp file so that each compile subprocess invocation // can reference it via --prior-manifest-file. - manifestCacheFile := "" manifestCache := CollectLockFileManifests("") - if len(manifestCache) > 0 { - cacheFile, err := WritePriorManifestFile(manifestCache) - if err != nil { - mcpLog.Printf("Failed to write manifest cache file: %v (safe update will fall back to git HEAD / filesystem)", err) - } else { - manifestCacheFile = cacheFile - mcpLog.Printf("Manifest cache written to %s (%d entries)", cacheFile, len(manifestCache)) - // Clean up the temp file when the server exits - defer func() { - if removeErr := os.Remove(cacheFile); removeErr != nil && !os.IsNotExist(removeErr) { - mcpLog.Printf("Failed to remove manifest cache file %s: %v", cacheFile, removeErr) - } - }() - } + if len(manifestCache) == 0 { + return "", nil } - // Create the server configuration - server := createMCPServer(cmdPath, actor, validateActor, manifestCacheFile, mcpServerEnv) - - if port > 0 { - // Run HTTP server with SSE transport - return runHTTPServer(server, port) + cacheFile, err := WritePriorManifestFile(manifestCache) + if err != nil { + mcpLog.Printf("Failed to write manifest cache file: %v (safe update will fall back to git HEAD / filesystem)", err) + return "", nil } - // Run stdio transport - mcpLog.Print("MCP server ready on stdio") - return server.Run(ctx, &mcp.StdioTransport{}) + mcpLog.Printf("Manifest cache written to %s (%d entries)", cacheFile, len(manifestCache)) + return cacheFile, func() { + if removeErr := os.Remove(cacheFile); removeErr != nil && !os.IsNotExist(removeErr) { + mcpLog.Printf("Failed to remove manifest cache file %s: %v", cacheFile, removeErr) + } + } } diff --git a/pkg/cli/mcp_tool_table.go b/pkg/cli/mcp_tool_table.go index b3f7a22187c..8b48ab853f8 100644 --- a/pkg/cli/mcp_tool_table.go +++ b/pkg/cli/mcp_tool_table.go @@ -35,49 +35,12 @@ func renderMCPToolTable(info *parser.MCPServerInfo, opts MCPToolTableOptions) st return "" } - // Create a map for quick lookup of allowed tools from workflow configuration - allowedMap := make(map[string]struct { - }) - - // Check for wildcard "*" which means all tools are allowed - hasWildcard := false - for _, allowed := range info.Config.Allowed { - if allowed == "*" { - hasWildcard = true - } - allowedMap[allowed] = struct { - }{} - } - + allowedMap, hasWildcard := renderMCPToolTableAllowedMap(info) mcpToolTableLog.Printf("Tool permissions: has_wildcard=%v, allowed_count=%d", hasWildcard, len(allowedMap)) // Build table headers and rows headers := []string{"Tool Name", "Allow", "Description"} - rows := make([][]string, 0, len(info.Tools)) - - for _, tool := range info.Tools { - description := tool.Description - - // Apply truncation if requested - if opts.TruncateLength > 0 && len(description) > opts.TruncateLength { - // Leave room for "..." - truncateAt := opts.TruncateLength - 3 - if truncateAt > 0 { - description = description[:truncateAt] + "..." - } - } - - // Determine status - status := "🚫" - if len(info.Config.Allowed) == 0 || hasWildcard { - // If no allowed list is specified or "*" wildcard is present, assume all tools are allowed - status = "✅" - } else if setutil.Contains(allowedMap, tool.Name) { - status = "✅" - } - - rows = append(rows, []string{tool.Name, status, description}) - } + rows := renderMCPToolTableRows(info, opts, allowedMap, hasWildcard) // Render the table table := console.RenderTable(console.TableConfig{ @@ -89,20 +52,7 @@ func renderMCPToolTable(info *parser.MCPServerInfo, opts MCPToolTableOptions) st // Add summary if requested if opts.ShowSummary { - allowedCount := 0 - for _, tool := range info.Tools { - if len(info.Config.Allowed) == 0 || hasWildcard || setutil.Contains(allowedMap, tool.Name) { - allowedCount++ - } - } - - summaryFormat := opts.SummaryFormat - if summaryFormat == "" { - summaryFormat = "\n📊 Summary: %d allowed, %d not allowed out of %d total tools\n" - } - - result += fmt.Sprintf(summaryFormat, - allowedCount, len(info.Tools)-allowedCount, len(info.Tools)) + result = renderMCPToolTableAppendSummary(result, info, opts, allowedMap, hasWildcard) } // Add verbose hint if requested @@ -112,3 +62,62 @@ func renderMCPToolTable(info *parser.MCPServerInfo, opts MCPToolTableOptions) st return result } + +func renderMCPToolTableAllowedMap(info *parser.MCPServerInfo) (map[string]struct{}, bool) { + // Create a map for quick lookup of allowed tools from workflow configuration. + allowedMap := make(map[string]struct{}) + hasWildcard := false + for _, allowed := range info.Config.Allowed { + if allowed == "*" { + hasWildcard = true + } + allowedMap[allowed] = struct{}{} + } + return allowedMap, hasWildcard +} + +func renderMCPToolTableRows(info *parser.MCPServerInfo, opts MCPToolTableOptions, allowedMap map[string]struct{}, hasWildcard bool) [][]string { + rows := make([][]string, 0, len(info.Tools)) + for _, tool := range info.Tools { + description := renderMCPToolTableDescription(tool.Description, opts.TruncateLength) + status := renderMCPToolTableStatus(info, allowedMap, hasWildcard, tool.Name) + rows = append(rows, []string{tool.Name, status, description}) + } + return rows +} + +func renderMCPToolTableDescription(description string, truncateLength int) string { + if truncateLength > 0 && len(description) > truncateLength { + // Leave room for "..." + truncateAt := truncateLength - 3 + if truncateAt > 0 { + return description[:truncateAt] + "..." + } + } + return description +} + +func renderMCPToolTableStatus(info *parser.MCPServerInfo, allowedMap map[string]struct{}, hasWildcard bool, toolName string) string { + if len(info.Config.Allowed) == 0 || hasWildcard || setutil.Contains(allowedMap, toolName) { + // If no allowed list is specified or "*" wildcard is present, assume all tools are allowed + return "✅" + } + return "🚫" +} + +func renderMCPToolTableAppendSummary(result string, info *parser.MCPServerInfo, opts MCPToolTableOptions, allowedMap map[string]struct{}, hasWildcard bool) string { + allowedCount := 0 + for _, tool := range info.Tools { + if len(info.Config.Allowed) == 0 || hasWildcard || setutil.Contains(allowedMap, tool.Name) { + allowedCount++ + } + } + + summaryFormat := opts.SummaryFormat + if summaryFormat == "" { + summaryFormat = "\n📊 Summary: %d allowed, %d not allowed out of %d total tools\n" + } + + return result + fmt.Sprintf(summaryFormat, + allowedCount, len(info.Tools)-allowedCount, len(info.Tools)) +} diff --git a/pkg/cli/mcp_tools_management.go b/pkg/cli/mcp_tools_management.go index 69c849ed751..cf44294e4e0 100644 --- a/pkg/cli/mcp_tools_management.go +++ b/pkg/cli/mcp_tools_management.go @@ -105,23 +105,8 @@ func registerUpdateTool(server *mcp.Server, execCmd execCmdFunc) { Annotations: &mcp.ToolAnnotations{ OpenWorldHint: new(true), }, - Description: `Update workflows from their source repositories and check for gh-aw updates. - -The command: -1. Checks if a newer version of gh-aw is available -2. Updates workflows using the 'source' field in the workflow frontmatter -3. Compiles each workflow immediately after update - -For workflow updates, it fetches the latest version based on the current ref: -- If the ref is a tag, it updates to the latest release (use major flag for major version updates) -- If the ref is a branch, it fetches the latest commit from that branch -- Otherwise, it fetches the latest commit from the default branch - -Returns formatted text output showing: -- Extension update status -- Updated workflows with their new versions -- Compilation status for each updated workflow`, - Icons: mcpToolIcons("🔄"), + Description: registerUpdateToolDescription(), + Icons: mcpToolIcons("🔄"), }, func(ctx context.Context, req *mcp.CallToolRequest, args updateArgs) (*mcp.CallToolResult, any, error) { // Check for cancellation before starting select { @@ -161,6 +146,25 @@ Returns formatted text output showing: }) } +func registerUpdateToolDescription() string { + return `Update workflows from their source repositories and check for gh-aw updates. + +The command: +1. Checks if a newer version of gh-aw is available +2. Updates workflows using the 'source' field in the workflow frontmatter +3. Compiles each workflow immediately after update + +For workflow updates, it fetches the latest version based on the current ref: +- If the ref is a tag, it updates to the latest release (use major flag for major version updates) +- If the ref is a branch, it fetches the latest commit from that branch +- Otherwise, it fetches the latest commit from the default branch + +Returns formatted text output showing: +- Extension update status +- Updated workflows with their new versions +- Compilation status for each updated workflow` +} + // fixArgs holds the input parameters for the fix tool. type fixArgs struct { Workflows []string `json:"workflows,omitempty" jsonschema:"Workflow IDs to fix (empty for all workflows)"` @@ -202,40 +206,42 @@ Returns formatted text output showing: - Summary of fixes applied`, Icons: mcpToolIcons("🔧"), }, func(ctx context.Context, req *mcp.CallToolRequest, args fixArgs) (*mcp.CallToolResult, any, error) { - // Check for cancellation before starting - select { - case <-ctx.Done(): - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) - default: - } - - mcpToolsManagementLog.Printf("fix tool invoked: workflows=%d, write=%v, list_codemods=%v", len(args.Workflows), args.Write, args.ListCodemods) - - // Build command arguments - cmdArgs := []string{"fix"} + return registerFixToolHandler(ctx, execCmd, args) + }) +} - // Add workflow IDs if specified - cmdArgs = append(cmdArgs, args.Workflows...) +func registerFixToolHandler(ctx context.Context, execCmd execCmdFunc, args fixArgs) (*mcp.CallToolResult, any, error) { + // Check for cancellation before starting + select { + case <-ctx.Done(): + return nil, nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) + default: + } - // Add optional flags - if args.Write { - cmdArgs = append(cmdArgs, "--write") - } - if args.ListCodemods { - cmdArgs = append(cmdArgs, "--list-codemods") - } + mcpToolsManagementLog.Printf("fix tool invoked: workflows=%d, write=%v, list_codemods=%v", len(args.Workflows), args.Write, args.ListCodemods) + cmdArgs := registerFixToolArgs(args) - // Execute the CLI command - output, err := runMCPExecCombinedOutput(ctx, execCmd, cmdArgs...) + // Execute the CLI command + output, err := runMCPExecCombinedOutput(ctx, execCmd, cmdArgs...) + if err != nil { + return nil, nil, newMCPError(jsonrpc.CodeInternalError, "failed to fix workflows", map[string]any{"error": err.Error(), "output": string(output)}) + } - if err != nil { - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "failed to fix workflows", map[string]any{"error": err.Error(), "output": string(output)}) - } + return &mcp.CallToolResult{ + Content: []mcp.Content{ + &mcp.TextContent{Text: string(output)}, + }, + }, nil, nil +} - return &mcp.CallToolResult{ - Content: []mcp.Content{ - &mcp.TextContent{Text: string(output)}, - }, - }, nil, nil - }) +func registerFixToolArgs(args fixArgs) []string { + cmdArgs := []string{"fix"} + cmdArgs = append(cmdArgs, args.Workflows...) + if args.Write { + cmdArgs = append(cmdArgs, "--write") + } + if args.ListCodemods { + cmdArgs = append(cmdArgs, "--list-codemods") + } + return cmdArgs } diff --git a/pkg/cli/mcp_tools_privileged.go b/pkg/cli/mcp_tools_privileged.go index 3ab9282b644..8d1e68a61ba 100644 --- a/pkg/cli/mcp_tools_privileged.go +++ b/pkg/cli/mcp_tools_privileged.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/github/gh-aw/pkg/constants" + "github.com/google/jsonschema-go/jsonschema" "github.com/modelcontextprotocol/go-sdk/jsonrpc" "github.com/modelcontextprotocol/go-sdk/mcp" ) @@ -78,18 +79,37 @@ func effectiveMCPLogsToolTimeoutMinutes(requestedTimeout, count int) int { // The logs tool requires write+ access and checks actor permissions. // Returns an error if schema generation fails. func registerLogsTool(server *mcp.Server, execCmd execCmdFunc, actor string, validateActor bool) error { - // Generate schema with elicitation defaults + logsSchema, err := registerLogsToolSchema() + if err != nil { + return err + } + + mcp.AddTool(server, &mcp.Tool{ + Name: "logs", + Annotations: &mcp.ToolAnnotations{ + ReadOnlyHint: true, + IdempotentHint: true, + OpenWorldHint: boolPtr(true), + }, + Description: registerLogsToolDescription(), + InputSchema: logsSchema, + Icons: mcpToolIcons("📝"), + }, func(ctx context.Context, req *mcp.CallToolRequest, args logsArgs) (*mcp.CallToolResult, any, error) { + return registerLogsToolHandler(ctx, req, execCmd, actor, validateActor, args) + }) + + return nil +} + +func registerLogsToolSchema() (*jsonschema.Schema, error) { logsSchema, err := GenerateSchema[logsArgs]() if err != nil { mcpLog.Printf("Failed to generate logs tool schema: %v", err) - return err + return nil, err } - // Add elicitation defaults for common parameters if err := AddSchemaDefault(logsSchema, "count", defaultMCPLogsToolCount); err != nil { mcpLog.Printf("Failed to add default for count: %v", err) } - // Schema default corresponds to defaultMCPLogsToolCount; runtime timeout - // scales with the effective count used for the request. if err := AddSchemaDefault(logsSchema, "timeout", defaultMCPLogsToolTimeoutMinutesForCount(defaultMCPLogsToolCount)); err != nil { mcpLog.Printf("Failed to add default for timeout: %v", err) } @@ -99,15 +119,11 @@ func registerLogsTool(server *mcp.Server, execCmd execCmdFunc, actor string, val if err := AddSchemaDefault(logsSchema, "artifacts", []string{"usage"}); err != nil { mcpLog.Printf("Failed to add default for artifacts: %v", err) } + return logsSchema, nil +} - mcp.AddTool(server, &mcp.Tool{ - Name: "logs", - Annotations: &mcp.ToolAnnotations{ - ReadOnlyHint: true, - IdempotentHint: true, - OpenWorldHint: boolPtr(true), - }, - Description: `Download and analyze workflow logs. +func registerLogsToolDescription() string { + return `Download and analyze workflow logs. In the normal case, returns a file path to a JSON file with workflow run data and metrics. The data is written to a file to avoid large inline payloads. Use the returned file_path @@ -119,166 +135,140 @@ in the JSON data with updated parameters to continue fetching more data. Check for the presence of the continuation field to determine if there are more logs available. The continuation field includes all necessary parameters (before_run_id, etc.) to resume fetching -from where the previous request stopped due to timeout.`, - InputSchema: logsSchema, - Icons: mcpToolIcons("📝"), - }, func(ctx context.Context, req *mcp.CallToolRequest, args logsArgs) (*mcp.CallToolResult, any, error) { - // Check actor permissions first - if err := checkActorPermission(ctx, actor, validateActor, "logs"); err != nil { - return nil, nil, err - } - - // Check for cancellation before starting - select { - case <-ctx.Done(): - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) - default: - } - - // Validate firewall parameters - if args.Firewall && args.NoFirewall { - return nil, nil, newMCPError(jsonrpc.CodeInvalidParams, "conflicting parameters: cannot specify both 'firewall' and 'no_firewall'", nil) - } - - // Validate workflow name before executing command - if err := validateMCPWorkflowName(args.WorkflowName); err != nil { - mcpLog.Printf("Workflow name validation failed, returning empty result: %v", err) - // Return an empty structured result instead of an MCP protocol error so - // callers can always expect consistent JSON from this tool. - // Use explicit empty slices so JSON marshaling produces "runs":[], etc., - // rather than null (nil slices), and set TotalDuration to match the normal - // zero-duration formatting. - emptyData := LogsData{ - Runs: []RunData{}, - Episodes: []EpisodeData{}, - Edges: []EpisodeEdge{}, - Message: err.Error(), - } - emptyData.Summary.TotalDuration = "0ns" - jsonBytes, jsonErr := json.Marshal(emptyData) - if jsonErr != nil { - return nil, nil, newMCPError(jsonrpc.CodeInvalidParams, err.Error(), nil) - } - return &mcp.CallToolResult{ - Content: []mcp.Content{&mcp.TextContent{Text: string(jsonBytes)}}, - }, nil, nil - } - - // Build command arguments - // Force output directory to /tmp/gh-aw/aw-mcp/logs for MCP server - cmdArgs := []string{"logs", "-o", constants.TmpAwMcpLogsDir} - if args.WorkflowName != "" { - cmdArgs = append(cmdArgs, args.WorkflowName) - } - effectiveCount := effectiveMCPLogsToolCount(args.Count) - cmdArgs = append(cmdArgs, "-c", strconv.Itoa(effectiveCount)) - if args.StartDate != "" { - cmdArgs = append(cmdArgs, "--start-date", args.StartDate) - } - if args.EndDate != "" { - cmdArgs = append(cmdArgs, "--end-date", args.EndDate) - } - if args.Engine != "" { - cmdArgs = append(cmdArgs, "--engine", args.Engine) - } - if args.Firewall { - cmdArgs = append(cmdArgs, "--firewall") - } - if args.NoFirewall { - cmdArgs = append(cmdArgs, "--no-firewall") - } - if args.FilteredIntegrity { - cmdArgs = append(cmdArgs, "--filtered-integrity") - } - if args.Branch != "" { - // The MCP parameter is named "branch" for backwards compatibility, - // but the logs CLI flag is --ref (which accepts branches and tags). - cmdArgs = append(cmdArgs, "--ref", args.Branch) - } - if args.AfterRunID > 0 { - cmdArgs = append(cmdArgs, "--after-run-id", strconv.FormatInt(args.AfterRunID, 10)) - } - if args.BeforeRunID > 0 { - cmdArgs = append(cmdArgs, "--before-run-id", strconv.FormatInt(args.BeforeRunID, 10)) - } - if len(args.Artifacts) > 0 { - cmdArgs = append(cmdArgs, "--artifacts", strings.Join(args.Artifacts, ",")) - } - - cmdArgs = appendRepoFlagFromEnv(cmdArgs) - - // Scale the implicit MCP timeout with the requested fetch window so - // larger fleet-wide requests do not hit the default per-tool timeout. - timeoutValue := effectiveMCPLogsToolTimeoutMinutes(args.Timeout, effectiveCount) - cmdArgs = append(cmdArgs, "--timeout", strconv.Itoa(timeoutValue)) - - // Always use --json mode in MCP server - cmdArgs = append(cmdArgs, "--json") - - // Log the command being executed for debugging - mcpLog.Printf("Executing logs tool: workflow=%s, requested_count=%d, effective_count=%d, firewall=%v, no_firewall=%v, filtered_integrity=%v, timeout=%d, command_args=%v", - args.WorkflowName, args.Count, effectiveCount, args.Firewall, args.NoFirewall, args.FilteredIntegrity, timeoutValue, cmdArgs) - - notifyProgress(ctx, req, 0, 100, "Downloading workflow logs...") +from where the previous request stopped due to timeout.` +} - // Execute the CLI command - // Use separate stdout/stderr capture instead of CombinedOutput because: - // - Stdout contains JSON output (--json flag) - // - Stderr contains console messages and error details - stdout, err := runMCPExecOutput(ctx, execCmd, cmdArgs...) +func registerLogsToolHandler(ctx context.Context, req *mcp.CallToolRequest, execCmd execCmdFunc, actor string, validateActor bool, args logsArgs) (*mcp.CallToolResult, any, error) { + if result, err := registerLogsToolPreflight(ctx, actor, validateActor, args); result != nil || err != nil { + return result, nil, err + } - // The logs command outputs JSON to stdout when --json flag is used. - // If the command fails, we need to provide detailed error information. - outputStr := string(stdout) + cmdArgs, effectiveCount, timeoutValue := registerLogsToolArgs(args) + mcpLog.Printf("Executing logs tool: workflow=%s, requested_count=%d, effective_count=%d, firewall=%v, no_firewall=%v, filtered_integrity=%v, timeout=%d, command_args=%v", + args.WorkflowName, args.Count, effectiveCount, args.Firewall, args.NoFirewall, args.FilteredIntegrity, timeoutValue, cmdArgs) - if err != nil { - // Try to get stderr and exit code for detailed error reporting - var stderr string - var exitCode int - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - stderr = string(exitErr.Stderr) - exitCode = exitErr.ExitCode() - } - - mcpLog.Printf("Logs command exited with error: %v (stdout length: %d, stderr length: %d, exit_code: %d)", - err, len(outputStr), len(stderr), exitCode) - - // Build detailed error data - errorData := map[string]any{ - "error": err.Error(), - "command": strings.Join(cmdArgs, " "), - "exit_code": exitCode, - "stdout": outputStr, - "stderr": stderr, - "timeout": timeoutValue, - "workflow": args.WorkflowName, - } + notifyProgress(ctx, req, 0, 100, "Downloading workflow logs...") + stdout, err := runMCPExecOutput(ctx, execCmd, cmdArgs...) + outputStr := string(stdout) + if err != nil { + return nil, nil, registerLogsToolError(err, outputStr, cmdArgs, timeoutValue, args.WorkflowName) + } - // Extract the user-facing message from stderr, filtering out debug log lines - // (e.g. "workflow:script_registry Creating new script registry +151ns") - // to avoid leaking internal diagnostic output in the MCP error response. - mainMsg := extractLastConsoleMessage(stderr) - if mainMsg == "" { - mainMsg = err.Error() - } + finalOutput := buildLogsFileResponse(outputStr) + notifyProgress(ctx, req, 100, 100, "Workflow logs downloaded") + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: finalOutput}}, + }, nil, nil +} - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "failed to download workflow logs: "+mainMsg, errorData) - } +func registerLogsToolPreflight(ctx context.Context, actor string, validateActor bool, args logsArgs) (*mcp.CallToolResult, error) { + if err := checkActorPermission(ctx, actor, validateActor, "logs"); err != nil { + return nil, err + } + select { + case <-ctx.Done(): + return nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) + default: + } + if args.Firewall && args.NoFirewall { + return nil, newMCPError(jsonrpc.CodeInvalidParams, "conflicting parameters: cannot specify both 'firewall' and 'no_firewall'", nil) + } + if err := validateMCPWorkflowName(args.WorkflowName); err != nil { + return registerLogsToolEmptyResult(err) + } + return nil, nil +} - // Always write output to a file and return schema + file path - finalOutput := buildLogsFileResponse(outputStr) +func registerLogsToolEmptyResult(err error) (*mcp.CallToolResult, error) { + mcpLog.Printf("Workflow name validation failed, returning empty result: %v", err) + emptyData := LogsData{ + Runs: []RunData{}, + Episodes: []EpisodeData{}, + Edges: []EpisodeEdge{}, + Message: err.Error(), + } + emptyData.Summary.TotalDuration = "0ns" + jsonBytes, jsonErr := json.Marshal(emptyData) + if jsonErr != nil { + return nil, newMCPError(jsonrpc.CodeInvalidParams, err.Error(), nil) + } + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: string(jsonBytes)}}, + }, nil +} - notifyProgress(ctx, req, 100, 100, "Workflow logs downloaded") +func registerLogsToolArgs(args logsArgs) ([]string, int, int) { + cmdArgs := []string{"logs", "-o", constants.TmpAwMcpLogsDir} + if args.WorkflowName != "" { + cmdArgs = append(cmdArgs, args.WorkflowName) + } + effectiveCount := effectiveMCPLogsToolCount(args.Count) + cmdArgs = append(cmdArgs, "-c", strconv.Itoa(effectiveCount)) + cmdArgs = registerLogsToolArgsFilters(cmdArgs, args) + cmdArgs = appendRepoFlagFromEnv(cmdArgs) + timeoutValue := effectiveMCPLogsToolTimeoutMinutes(args.Timeout, effectiveCount) + cmdArgs = append(cmdArgs, "--timeout", strconv.Itoa(timeoutValue), "--json") + return cmdArgs, effectiveCount, timeoutValue +} - return &mcp.CallToolResult{ - Content: []mcp.Content{ - &mcp.TextContent{Text: finalOutput}, - }, - }, nil, nil - }) +func registerLogsToolArgsFilters(cmdArgs []string, args logsArgs) []string { + if args.StartDate != "" { + cmdArgs = append(cmdArgs, "--start-date", args.StartDate) + } + if args.EndDate != "" { + cmdArgs = append(cmdArgs, "--end-date", args.EndDate) + } + if args.Engine != "" { + cmdArgs = append(cmdArgs, "--engine", args.Engine) + } + if args.Firewall { + cmdArgs = append(cmdArgs, "--firewall") + } + if args.NoFirewall { + cmdArgs = append(cmdArgs, "--no-firewall") + } + if args.FilteredIntegrity { + cmdArgs = append(cmdArgs, "--filtered-integrity") + } + if args.Branch != "" { + cmdArgs = append(cmdArgs, "--ref", args.Branch) + } + if args.AfterRunID > 0 { + cmdArgs = append(cmdArgs, "--after-run-id", strconv.FormatInt(args.AfterRunID, 10)) + } + if args.BeforeRunID > 0 { + cmdArgs = append(cmdArgs, "--before-run-id", strconv.FormatInt(args.BeforeRunID, 10)) + } + if len(args.Artifacts) > 0 { + cmdArgs = append(cmdArgs, "--artifacts", strings.Join(args.Artifacts, ",")) + } + return cmdArgs +} - return nil +func registerLogsToolError(err error, outputStr string, cmdArgs []string, timeoutValue int, workflowName string) error { + var stderr string + var exitCode int + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + stderr = string(exitErr.Stderr) + exitCode = exitErr.ExitCode() + } + mcpLog.Printf("Logs command exited with error: %v (stdout length: %d, stderr length: %d, exit_code: %d)", + err, len(outputStr), len(stderr), exitCode) + errorData := map[string]any{ + "error": err.Error(), + "command": strings.Join(cmdArgs, " "), + "exit_code": exitCode, + "stdout": outputStr, + "stderr": stderr, + "timeout": timeoutValue, + "workflow": workflowName, + } + mainMsg := extractLastConsoleMessage(stderr) + if mainMsg == "" { + mainMsg = err.Error() + } + return newMCPError(jsonrpc.CodeInternalError, "failed to download workflow logs: "+mainMsg, errorData) } // auditArgs holds the input parameters for the audit tool. @@ -320,10 +310,8 @@ func normalizeAuditRunInput(input any, fieldName string) (string, bool, error) { // The audit tool requires write+ access and checks actor permissions. // Returns an error if schema generation fails. func registerAuditTool(server *mcp.Server, execCmd execCmdFunc, actor string, validateActor bool) error { - // Generate schema for audit tool - auditSchema, err := GenerateSchema[auditArgs]() + auditSchema, err := registerAuditToolSchema() if err != nil { - mcpLog.Printf("Failed to generate audit tool schema: %v", err) return err } @@ -334,7 +322,27 @@ func registerAuditTool(server *mcp.Server, execCmd execCmdFunc, actor string, va IdempotentHint: true, OpenWorldHint: boolPtr(true), }, - Description: `Investigate one or more workflow runs and generate a concise report. + Description: registerAuditToolDescription(), + InputSchema: auditSchema, + Icons: mcpToolIcons("🔍"), + }, func(ctx context.Context, req *mcp.CallToolRequest, args auditArgs) (*mcp.CallToolResult, any, error) { + return registerAuditToolHandler(ctx, req, execCmd, actor, validateActor, args) + }) + + return nil +} + +func registerAuditToolSchema() (*jsonschema.Schema, error) { + auditSchema, err := GenerateSchema[auditArgs]() + if err != nil { + mcpLog.Printf("Failed to generate audit tool schema: %v", err) + return nil, err + } + return auditSchema, nil +} + +func registerAuditToolDescription() string { + return `Investigate one or more workflow runs and generate a concise report. When a single run is provided, generates a detailed audit report. When two or more runs are provided, the first is the base (reference) run and @@ -370,133 +378,115 @@ Single-run returns JSON with: - firewall_analysis: Network firewall analysis if available (total_requests, allowed_requests, blocked_requests, allowed_domains, blocked_domains) - experiments: A/B experiment assignments if present (assignments map, cumulative_counts map) -Multi-run diff returns JSON describing changes between the base and each comparison run.`, - InputSchema: auditSchema, - Icons: mcpToolIcons("🔍"), - }, func(ctx context.Context, req *mcp.CallToolRequest, args auditArgs) (*mcp.CallToolResult, any, error) { - // Check actor permissions first - if err := checkActorPermission(ctx, actor, validateActor, "audit"); err != nil { - return nil, nil, err - } - - // Check for cancellation before starting - select { - case <-ctx.Done(): - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) - default: - } - - // Resolve the list of run IDs/URLs to pass to the audit command. - // run_ids_or_urls takes precedence; fall back to run_id, then deprecated run_id_or_url. - runItems := args.RunIDsOrURLs - if len(runItems) == 0 { - runID, hasRunID, err := normalizeAuditRunInput(args.RunID, "run_id") - if err != nil { - return nil, nil, newMCPError(jsonrpc.CodeInvalidParams, err.Error(), nil) - } - if hasRunID { - runItems = []string{runID} - } - } - if len(runItems) == 0 { - runIDOrURL, hasRunIDOrURL, err := normalizeAuditRunInput(args.RunIDOrURL, "run_id_or_url") - if err != nil { - return nil, nil, newMCPError(jsonrpc.CodeInvalidParams, err.Error(), nil) - } - if hasRunIDOrURL { - runItems = []string{runIDOrURL} - } - } - if len(runItems) == 0 { - return nil, nil, newMCPError(jsonrpc.CodeInvalidParams, "at least one run ID or URL must be provided via run_ids_or_urls, run_id, or run_id_or_url", nil) - } - - // Build command arguments. - // Force output directory to /tmp/gh-aw/aw-mcp/logs for MCP server (same as logs). - // Use --json flag to output structured JSON for MCP consumption. - // Pass all run IDs/URLs directly - the audit command handles single vs. diff mode. - cmdArgs := []string{"audit"} - cmdArgs = append(cmdArgs, runItems...) - cmdArgs = append(cmdArgs, "-o", constants.TmpAwMcpLogsDir, "--json") - if len(args.Artifacts) > 0 { - cmdArgs = append(cmdArgs, "--artifacts", strings.Join(args.Artifacts, ",")) - } - if args.Experiment != "" { - cmdArgs = append(cmdArgs, "--experiment", args.Experiment) - } - if args.Variant != "" { - cmdArgs = append(cmdArgs, "--variant", args.Variant) - } +Multi-run diff returns JSON describing changes between the base and each comparison run.` +} - cmdArgs = appendRepoFlagFromEnv(cmdArgs) +func registerAuditToolHandler(ctx context.Context, req *mcp.CallToolRequest, execCmd execCmdFunc, actor string, validateActor bool, args auditArgs) (*mcp.CallToolResult, any, error) { + runItems, err := registerAuditToolRunItems(ctx, actor, validateActor, args) + if err != nil { + return nil, nil, err + } - notifyProgress(ctx, req, 0, 100, "Downloading audit artifacts...") + cmdArgs := registerAuditToolArgs(runItems, args) + notifyProgress(ctx, req, 0, 100, "Downloading audit artifacts...") + stdout, err := runMCPExecOutput(ctx, execCmd, cmdArgs...) + outputStr := string(stdout) + if err != nil { + return registerAuditToolError(err, outputStr, runItems) + } - // Execute the CLI command. - // Use separate stdout/stderr capture instead of CombinedOutput because: - // - Stdout contains JSON output (--json flag) - // - Stderr contains console messages and debug logs that shouldn't be mixed with JSON - stdout, err := runMCPExecOutput(ctx, execCmd, cmdArgs...) + notifyProgress(ctx, req, 100, 100, "Audit complete") + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: outputStr}}, + }, nil, nil +} - // The audit command outputs JSON to stdout when --json flag is used. - // If the command fails, we need to provide detailed error information. - outputStr := string(stdout) +func registerAuditToolRunItems(ctx context.Context, actor string, validateActor bool, args auditArgs) ([]string, error) { + if err := checkActorPermission(ctx, actor, validateActor, "audit"); err != nil { + return nil, err + } + select { + case <-ctx.Done(): + return nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) + default: + } + runItems, err := registerAuditToolResolveRunItems(args) + if err != nil { + return nil, newMCPError(jsonrpc.CodeInvalidParams, err.Error(), nil) + } + if len(runItems) == 0 { + return nil, newMCPError(jsonrpc.CodeInvalidParams, "at least one run ID or URL must be provided via run_ids_or_urls, run_id, or run_id_or_url", nil) + } + return runItems, nil +} +func registerAuditToolResolveRunItems(args auditArgs) ([]string, error) { + runItems := args.RunIDsOrURLs + if len(runItems) == 0 { + runID, hasRunID, err := normalizeAuditRunInput(args.RunID, "run_id") if err != nil { - // Try to get stderr for message extraction - var stderr string - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - stderr = string(exitErr.Stderr) - } - - mcpLog.Printf("Audit command exited with error: %v (stdout length: %d, stderr length: %d)", - err, len(outputStr), len(stderr)) - - // Extract the user-facing message from stderr, filtering out debug log lines - // (e.g. "workflow:script_registry Creating new script registry +151ns") - // to avoid leaking internal diagnostic output in the MCP error response. - mainMsg := extractLastConsoleMessage(stderr) - if mainMsg == "" { - mainMsg = err.Error() - } - - // Return a JSON error envelope instead of an MCP protocol error so - // callers always receive consistent JSON and the run IDs are always present. - // IsError must be false so that callers (e.g. mcp_cli_bridge) treat this as - // a graceful not-found / failure response rather than a fatal protocol error. - errorMsg := "failed to audit workflow run: " + mainMsg - if len(runItems) > 1 { - errorMsg = "failed to audit workflow runs: " + mainMsg - } - errorEnvelope := map[string]any{ - "error": errorMsg, - "run_ids_or_urls": runItems, - "suggestions": []string{ - "Verify the run ID is correct", - "Use the 'logs' tool to list recent run IDs", - }, - } - jsonBytes, jsonErr := json.Marshal(errorEnvelope) - if jsonErr != nil { - return nil, nil, newMCPError(jsonrpc.CodeInternalError, errorMsg, nil) - } - return &mcp.CallToolResult{ - IsError: false, - Content: []mcp.Content{&mcp.TextContent{Text: string(jsonBytes)}}, - }, nil, nil + return nil, err } + if hasRunID { + runItems = []string{runID} + } + } + if len(runItems) == 0 { + runIDOrURL, hasRunIDOrURL, err := normalizeAuditRunInput(args.RunIDOrURL, "run_id_or_url") + if err != nil { + return nil, err + } + if hasRunIDOrURL { + runItems = []string{runIDOrURL} + } + } + return runItems, nil +} - notifyProgress(ctx, req, 100, 100, "Audit complete") - - return &mcp.CallToolResult{ - Content: []mcp.Content{ - &mcp.TextContent{Text: outputStr}, - }, - }, nil, nil - }) +func registerAuditToolArgs(runItems []string, args auditArgs) []string { + cmdArgs := []string{"audit"} + cmdArgs = append(cmdArgs, runItems...) + cmdArgs = append(cmdArgs, "-o", constants.TmpAwMcpLogsDir, "--json") + if len(args.Artifacts) > 0 { + cmdArgs = append(cmdArgs, "--artifacts", strings.Join(args.Artifacts, ",")) + } + if args.Experiment != "" { + cmdArgs = append(cmdArgs, "--experiment", args.Experiment) + } + if args.Variant != "" { + cmdArgs = append(cmdArgs, "--variant", args.Variant) + } + return appendRepoFlagFromEnv(cmdArgs) +} - return nil +func registerAuditToolError(err error, outputStr string, runItems []string) (*mcp.CallToolResult, any, error) { + var stderr string + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + stderr = string(exitErr.Stderr) + } + mcpLog.Printf("Audit command exited with error: %v (stdout length: %d, stderr length: %d)", err, len(outputStr), len(stderr)) + mainMsg := extractLastConsoleMessage(stderr) + if mainMsg == "" { + mainMsg = err.Error() + } + errorMsg := "failed to audit workflow run: " + mainMsg + if len(runItems) > 1 { + errorMsg = "failed to audit workflow runs: " + mainMsg + } + errorEnvelope := map[string]any{ + "error": errorMsg, + "run_ids_or_urls": runItems, + "suggestions": []string{"Verify the run ID is correct", "Use the 'logs' tool to list recent run IDs"}, + } + jsonBytes, jsonErr := json.Marshal(errorEnvelope) + if jsonErr != nil { + return nil, nil, newMCPError(jsonrpc.CodeInternalError, errorMsg, nil) + } + return &mcp.CallToolResult{ + IsError: false, + Content: []mcp.Content{&mcp.TextContent{Text: string(jsonBytes)}}, + }, nil, nil } // auditDiffArgs holds the input parameters for the audit-diff tool. @@ -509,9 +499,8 @@ type auditDiffArgs struct { // registerAuditDiffTool registers the audit-diff tool with the MCP server. // It exposes the `gh aw audit diff` subcommand for comparing two workflow runs. func registerAuditDiffTool(server *mcp.Server, execCmd execCmdFunc, actor string, validateActor bool) error { - schema, err := GenerateSchema[auditDiffArgs]() + schema, err := registerAuditDiffToolSchema() if err != nil { - mcpLog.Printf("Failed to generate audit-diff tool schema: %v", err) return err } @@ -522,7 +511,27 @@ func registerAuditDiffTool(server *mcp.Server, execCmd execCmdFunc, actor string IdempotentHint: true, OpenWorldHint: boolPtr(true), }, - Description: `Compare behavior between a base workflow run and one or more comparison runs. + Description: registerAuditDiffToolDescription(), + InputSchema: schema, + Icons: mcpToolIcons("🔎"), + }, func(ctx context.Context, req *mcp.CallToolRequest, args auditDiffArgs) (*mcp.CallToolResult, any, error) { + return registerAuditDiffToolHandler(ctx, req, execCmd, actor, validateActor, args) + }) + + return nil +} + +func registerAuditDiffToolSchema() (*jsonschema.Schema, error) { + schema, err := GenerateSchema[auditDiffArgs]() + if err != nil { + mcpLog.Printf("Failed to generate audit-diff tool schema: %v", err) + return nil, err + } + return schema, nil +} + +func registerAuditDiffToolDescription() string { + return `Compare behavior between a base workflow run and one or more comparison runs. Downloads artifacts for all referenced runs (using locally cached data when available), then produces a diff showing: @@ -532,80 +541,81 @@ then produces a diff showing: - MCP tool invocation changes (new/removed tools, call/error count diffs) - Run metrics comparison (token usage, duration, turns) -Returns JSON describing the differences between the base run and each comparison run.`, - InputSchema: schema, - Icons: mcpToolIcons("🔎"), - }, func(ctx context.Context, req *mcp.CallToolRequest, args auditDiffArgs) (*mcp.CallToolResult, any, error) { - if err := checkActorPermission(ctx, actor, validateActor, "audit-diff"); err != nil { - return nil, nil, err - } - - select { - case <-ctx.Done(): - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) - default: - } - - if args.BaseRunID == "" { - return nil, nil, newMCPError(jsonrpc.CodeInvalidParams, "base_run_id is required", nil) - } - if len(args.CompareRunIDs) == 0 { - return nil, nil, newMCPError(jsonrpc.CodeInvalidParams, "compare_run_ids must contain at least one run ID", nil) - } - - // Build: gh aw audit diff -o ... --json [--artifacts ...] - cmdArgs := []string{"audit", "diff", args.BaseRunID} - cmdArgs = append(cmdArgs, args.CompareRunIDs...) - cmdArgs = append(cmdArgs, "-o", constants.TmpAwMcpLogsDir, "--json") - if len(args.Artifacts) > 0 { - cmdArgs = append(cmdArgs, "--artifacts", strings.Join(args.Artifacts, ",")) - } - - cmdArgs = appendRepoFlagFromEnv(cmdArgs) +Returns JSON describing the differences between the base run and each comparison run.` +} - notifyProgress(ctx, req, 0, 100, "Downloading artifacts for diff...") +func registerAuditDiffToolHandler(ctx context.Context, req *mcp.CallToolRequest, execCmd execCmdFunc, actor string, validateActor bool, args auditDiffArgs) (*mcp.CallToolResult, any, error) { + if err := registerAuditDiffToolValidate(ctx, actor, validateActor, args); err != nil { + return nil, nil, err + } - stdout, err := runMCPExecOutput(ctx, execCmd, cmdArgs...) - outputStr := string(stdout) + cmdArgs := registerAuditDiffToolArgs(args) + notifyProgress(ctx, req, 0, 100, "Downloading artifacts for diff...") + stdout, err := runMCPExecOutput(ctx, execCmd, cmdArgs...) + outputStr := string(stdout) + if err != nil { + return registerAuditDiffToolError(err, outputStr, args) + } - if err != nil { - var stderr string - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - stderr = string(exitErr.Stderr) - } - mcpLog.Printf("Audit-diff command failed: %v (stdout: %d bytes, stderr: %d bytes)", err, len(outputStr), len(stderr)) - mainMsg := extractLastConsoleMessage(stderr) - if mainMsg == "" { - mainMsg = err.Error() - } - errorEnvelope := map[string]any{ - "error": "failed to diff workflow runs: " + mainMsg, - "base_run_id": args.BaseRunID, - "compare_runs": args.CompareRunIDs, - "suggestions": []string{ - "Verify the run IDs are correct", - "Use the 'logs' tool to list recent run IDs", - }, - } - jsonBytes, jsonErr := json.Marshal(errorEnvelope) - if jsonErr != nil { - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "failed to diff workflow runs: "+mainMsg, nil) - } - return &mcp.CallToolResult{ - IsError: false, - Content: []mcp.Content{&mcp.TextContent{Text: string(jsonBytes)}}, - }, nil, nil - } + notifyProgress(ctx, req, 100, 100, "Diff complete") + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: outputStr}}, + }, nil, nil +} - notifyProgress(ctx, req, 100, 100, "Diff complete") +func registerAuditDiffToolValidate(ctx context.Context, actor string, validateActor bool, args auditDiffArgs) error { + if err := checkActorPermission(ctx, actor, validateActor, "audit-diff"); err != nil { + return err + } + select { + case <-ctx.Done(): + return newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) + default: + } + if args.BaseRunID == "" { + return newMCPError(jsonrpc.CodeInvalidParams, "base_run_id is required", nil) + } + if len(args.CompareRunIDs) == 0 { + return newMCPError(jsonrpc.CodeInvalidParams, "compare_run_ids must contain at least one run ID", nil) + } + return nil +} - return &mcp.CallToolResult{ - Content: []mcp.Content{&mcp.TextContent{Text: outputStr}}, - }, nil, nil - }) +func registerAuditDiffToolArgs(args auditDiffArgs) []string { + cmdArgs := []string{"audit", "diff", args.BaseRunID} + cmdArgs = append(cmdArgs, args.CompareRunIDs...) + cmdArgs = append(cmdArgs, "-o", constants.TmpAwMcpLogsDir, "--json") + if len(args.Artifacts) > 0 { + cmdArgs = append(cmdArgs, "--artifacts", strings.Join(args.Artifacts, ",")) + } + return appendRepoFlagFromEnv(cmdArgs) +} - return nil +func registerAuditDiffToolError(err error, outputStr string, args auditDiffArgs) (*mcp.CallToolResult, any, error) { + var stderr string + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + stderr = string(exitErr.Stderr) + } + mcpLog.Printf("Audit-diff command failed: %v (stdout: %d bytes, stderr: %d bytes)", err, len(outputStr), len(stderr)) + mainMsg := extractLastConsoleMessage(stderr) + if mainMsg == "" { + mainMsg = err.Error() + } + errorEnvelope := map[string]any{ + "error": "failed to diff workflow runs: " + mainMsg, + "base_run_id": args.BaseRunID, + "compare_runs": args.CompareRunIDs, + "suggestions": []string{"Verify the run IDs are correct", "Use the 'logs' tool to list recent run IDs"}, + } + jsonBytes, jsonErr := json.Marshal(errorEnvelope) + if jsonErr != nil { + return nil, nil, newMCPError(jsonrpc.CodeInternalError, "failed to diff workflow runs: "+mainMsg, nil) + } + return &mcp.CallToolResult{ + IsError: false, + Content: []mcp.Content{&mcp.TextContent{Text: string(jsonBytes)}}, + }, nil, nil } // notifyProgress sends a progress notification to the MCP client if the request diff --git a/pkg/cli/mcp_tools_readonly.go b/pkg/cli/mcp_tools_readonly.go index d30972c2187..76ecf155a6c 100644 --- a/pkg/cli/mcp_tools_readonly.go +++ b/pkg/cli/mcp_tools_readonly.go @@ -7,6 +7,7 @@ import ( "os/exec" "path/filepath" + "github.com/google/jsonschema-go/jsonschema" "github.com/modelcontextprotocol/go-sdk/jsonrpc" "github.com/modelcontextprotocol/go-sdk/mcp" ) @@ -86,16 +87,10 @@ type compileArgs struct { // enforcement. An empty string disables this feature. // Returns an error if schema generation fails, which causes the server to stop registering tools. func registerCompileTool(server *mcp.Server, execCmd execCmdFunc, manifestCacheFile string) error { - // Generate schema with elicitation defaults - compileSchema, err := GenerateSchema[compileArgs]() + compileSchema, err := registerCompileToolSchema() if err != nil { - mcpLog.Printf("Failed to generate compile tool schema: %v", err) return err } - // Add elicitation default: strict defaults to true (most common case) - if err := AddSchemaDefault(compileSchema, "strict", true); err != nil { - mcpLog.Printf("Failed to add default for strict: %v", err) - } mcp.AddTool(server, &mcp.Tool{ Name: "compile", @@ -104,7 +99,30 @@ func registerCompileTool(server *mcp.Server, execCmd execCmdFunc, manifestCacheF DestructiveHint: boolPtr(false), OpenWorldHint: boolPtr(false), }, - Description: `Compile Markdown workflows to GitHub Actions YAML with optional static analysis tools. + Description: registerCompileToolDescription(), + InputSchema: compileSchema, + Icons: mcpToolIcons("📋"), + }, func(ctx context.Context, req *mcp.CallToolRequest, args compileArgs) (*mcp.CallToolResult, any, error) { + return registerCompileToolHandler(ctx, execCmd, manifestCacheFile, args) + }) + + return nil +} + +func registerCompileToolSchema() (*jsonschema.Schema, error) { + compileSchema, err := GenerateSchema[compileArgs]() + if err != nil { + mcpLog.Printf("Failed to generate compile tool schema: %v", err) + return nil, err + } + if err := AddSchemaDefault(compileSchema, "strict", true); err != nil { + mcpLog.Printf("Failed to add default for strict: %v", err) + } + return compileSchema, nil +} + +func registerCompileToolDescription() string { + return `Compile Markdown workflows to GitHub Actions YAML with optional static analysis tools. ⚠️ IMPORTANT: Any change to .github/workflows/*.md files MUST be compiled using this tool. This tool generates .lock.yml files from .md workflow files. The .lock.yml files are what GitHub Actions @@ -119,144 +137,113 @@ Returns JSON array with validation results for each workflow: - valid: Boolean indicating if compilation was successful - errors: Array of error objects with type, message, and optional line number - warnings: Array of warning objects -- compiled_file: Path to the generated .lock.yml file`, - InputSchema: compileSchema, - Icons: mcpToolIcons("📋"), - }, func(ctx context.Context, req *mcp.CallToolRequest, args compileArgs) (*mcp.CallToolResult, any, error) { - // Check for cancellation before starting - select { - case <-ctx.Done(): - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) - default: - } - - // dockerUnavailableWarning is set when Docker is not accessible but the compile - // should still proceed without the static-analysis tools. After the compile - // attempt, the warning is appended to workflow results in the JSON output so - // the caller knows linting was skipped, while preserving each workflow's - // valid/invalid status. - var dockerUnavailableWarning string - - // Check if any static analysis tools are requested that require Docker images - if args.Zizmor || args.Poutine || args.Actionlint || args.RunnerGuard { - // Check if Docker images are available; if not, start downloading and return retry message - if err := CheckAndPrepareDockerImages(ctx, args.Zizmor, args.Poutine, args.Actionlint, args.RunnerGuard); err != nil { - var dockerUnavailableErr *DockerUnavailableError - if errors.As(err, &dockerUnavailableErr) { - // Docker daemon is not running. Instead of failing every workflow, - // compile without the Docker-based tools and surface a warning so - // the caller knows static analysis was skipped. - dockerUnavailableWarning = err.Error() - args.Zizmor = false - args.Poutine = false - args.Actionlint = false - args.RunnerGuard = false - } else { - // Images are still downloading — ask the caller to retry. - // Build per-workflow validation errors instead of throwing an MCP protocol error, - // so callers always receive consistent JSON regardless of the failure mode. - results := buildDockerErrorResults(args.Workflows, err.Error()) - jsonBytes, jsonErr := json.Marshal(results) - if jsonErr != nil { - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "failed to marshal docker error results", jsonErr.Error()) - } - return &mcp.CallToolResult{ - Content: []mcp.Content{&mcp.TextContent{Text: string(jsonBytes)}}, - }, nil, nil - } - } - - // Check for cancellation after Docker image preparation - select { - case <-ctx.Done(): - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) - default: - } - } +- compiled_file: Path to the generated .lock.yml file` +} - // Build command arguments - // Always validate workflows during compilation and use JSON output for MCP - cmdArgs := []string{"compile", "--validate", "--json"} +func registerCompileToolHandler(ctx context.Context, execCmd execCmdFunc, manifestCacheFile string, args compileArgs) (*mcp.CallToolResult, any, error) { + if err := registerCompileToolCheckCancelled(ctx); err != nil { + return nil, nil, err + } - // Add fix flag if requested - if args.Fix { - cmdArgs = append(cmdArgs, "--fix") - } + dockerUnavailableWarning, dockerResult, err := registerCompileToolPrepareDocker(ctx, &args) + if dockerResult != nil || err != nil { + return dockerResult, nil, err + } - // Add strict flag if requested - if args.Strict { - cmdArgs = append(cmdArgs, "--strict") - } + cmdArgs := registerCompileToolArgs(args, manifestCacheFile) + mcpLog.Printf("Executing compile tool: workflows=%v, strict=%v, fix=%v, zizmor=%v, poutine=%v, actionlint=%v, runner-guard=%v", + args.Workflows, args.Strict, args.Fix, args.Zizmor, args.Poutine, args.Actionlint, args.RunnerGuard) - // Add static analysis flags if requested - if args.Zizmor { - cmdArgs = append(cmdArgs, "--zizmor") - } - if args.Poutine { - cmdArgs = append(cmdArgs, "--poutine") - } - if args.Actionlint { - cmdArgs = append(cmdArgs, "--actionlint") - } - if args.RunnerGuard { - cmdArgs = append(cmdArgs, "--runner-guard") - } + stdout, err := runMCPExecOutput(ctx, execCmd, cmdArgs...) + outputStr, mcpErr := registerCompileToolOutput(stdout, err) + if mcpErr != nil { + return nil, nil, mcpErr + } + if dockerUnavailableWarning != "" { + outputStr = injectDockerUnavailableWarning(outputStr, dockerUnavailableWarning) + } + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: outputStr}}, + }, nil, nil +} - cmdArgs = append(cmdArgs, args.Workflows...) +func registerCompileToolCheckCancelled(ctx context.Context) error { + select { + case <-ctx.Done(): + return newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) + default: + return nil + } +} - // Pass the pre-cached manifest file when available so the compiler uses - // the tamper-proof manifest baseline captured at server startup. - if manifestCacheFile != "" { - cmdArgs = append(cmdArgs, "--prior-manifest-file", manifestCacheFile) +func registerCompileToolPrepareDocker(ctx context.Context, args *compileArgs) (string, *mcp.CallToolResult, error) { + if !(args.Zizmor || args.Poutine || args.Actionlint || args.RunnerGuard) { + return "", nil, nil + } + if err := CheckAndPrepareDockerImages(ctx, args.Zizmor, args.Poutine, args.Actionlint, args.RunnerGuard); err != nil { + var dockerUnavailableErr *DockerUnavailableError + if errors.As(err, &dockerUnavailableErr) { + args.Zizmor, args.Poutine, args.Actionlint, args.RunnerGuard = false, false, false, false + return err.Error(), nil, nil } + dockerResult, dockerErr := registerCompileToolDockerError(args.Workflows, err) + return "", dockerResult, dockerErr + } + return "", nil, registerCompileToolCheckCancelled(ctx) +} - mcpLog.Printf("Executing compile tool: workflows=%v, strict=%v, fix=%v, zizmor=%v, poutine=%v, actionlint=%v, runner-guard=%v", - args.Workflows, args.Strict, args.Fix, args.Zizmor, args.Poutine, args.Actionlint, args.RunnerGuard) - - // Execute the CLI command - // Use separate stdout/stderr capture instead of CombinedOutput because: - // - Stdout contains JSON output (--json flag) - // - Stderr contains console messages that shouldn't be mixed with JSON - stdout, err := runMCPExecOutput(ctx, execCmd, cmdArgs...) +func registerCompileToolDockerError(workflows []string, err error) (*mcp.CallToolResult, error) { + results := buildDockerErrorResults(workflows, err.Error()) + jsonBytes, jsonErr := json.Marshal(results) + if jsonErr != nil { + return nil, newMCPError(jsonrpc.CodeInternalError, "failed to marshal docker error results", jsonErr.Error()) + } + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: string(jsonBytes)}}, + }, nil +} - // The compile command always outputs JSON to stdout when --json flag is used, even on error. - // We should return the JSON output to the LLM so it can see validation errors. - // Only return an MCP error if we cannot get any output at all. - outputStr := string(stdout) +func registerCompileToolArgs(args compileArgs, manifestCacheFile string) []string { + cmdArgs := []string{"compile", "--validate", "--json"} + if args.Fix { + cmdArgs = append(cmdArgs, "--fix") + } + if args.Strict { + cmdArgs = append(cmdArgs, "--strict") + } + if args.Zizmor { + cmdArgs = append(cmdArgs, "--zizmor") + } + if args.Poutine { + cmdArgs = append(cmdArgs, "--poutine") + } + if args.Actionlint { + cmdArgs = append(cmdArgs, "--actionlint") + } + if args.RunnerGuard { + cmdArgs = append(cmdArgs, "--runner-guard") + } + cmdArgs = append(cmdArgs, args.Workflows...) + if manifestCacheFile != "" { + cmdArgs = append(cmdArgs, "--prior-manifest-file", manifestCacheFile) + } + return cmdArgs +} - // If the command failed but we have output, it's likely compilation errors - // which are included in the JSON output. Return the output, not an MCP error. +func registerCompileToolOutput(stdout []byte, err error) (string, error) { + outputStr := string(stdout) + if err == nil || outputStr != "" { if err != nil { mcpLog.Printf("Compile command exited with error: %v (output length: %d)", err, len(outputStr)) - // If we have no output, this is a real execution failure - if outputStr == "" { - // Try to get stderr for error details - var stderr string - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - stderr = string(exitErr.Stderr) - } - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "failed to compile workflows", map[string]any{"error": err.Error(), "stderr": stderr}) - } - // Otherwise, we have output (likely validation errors in JSON), so continue - // and return it to the LLM - } - - // When Docker was unavailable, inject a warning into every workflow result so the - // caller knows that static analysis was skipped — but does NOT mark valid - // workflows as invalid. - if dockerUnavailableWarning != "" { - outputStr = injectDockerUnavailableWarning(outputStr, dockerUnavailableWarning) } - - return &mcp.CallToolResult{ - Content: []mcp.Content{ - &mcp.TextContent{Text: outputStr}, - }, - }, nil, nil - }) - - return nil + return outputStr, nil + } + var stderr string + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + stderr = string(exitErr.Stderr) + } + return "", newMCPError(jsonrpc.CodeInternalError, "failed to compile workflows", map[string]any{"error": err.Error(), "stderr": stderr}) } // mcpInspectArgs holds the input parameters for the mcp-inspect tool. @@ -275,7 +262,15 @@ func registerMCPInspectTool(server *mcp.Server, execCmd execCmdFunc) { IdempotentHint: true, OpenWorldHint: boolPtr(true), }, - Description: `Inspect MCP servers used by a workflow and list available tools, resources, and roots. + Description: registerMCPInspectToolDescription(), + Icons: mcpToolIcons("🔬"), + }, func(ctx context.Context, req *mcp.CallToolRequest, args mcpInspectArgs) (*mcp.CallToolResult, any, error) { + return registerMCPInspectToolHandler(ctx, execCmd, args) + }) +} + +func registerMCPInspectToolDescription() string { + return `Inspect MCP servers used by a workflow and list available tools, resources, and roots. This tool starts each MCP server configured in the workflow, queries its capabilities, and displays the results. It supports stdio, Docker, and HTTP MCP servers. @@ -293,47 +288,38 @@ Returns formatted text output showing: - Available MCP servers in the workflow - Tools, resources, and roots exposed by each server - Secret availability status (if GitHub token is available) -- Detailed tool information when tool parameter is specified`, - Icons: mcpToolIcons("🔬"), - }, func(ctx context.Context, req *mcp.CallToolRequest, args mcpInspectArgs) (*mcp.CallToolResult, any, error) { - // Check for cancellation before starting - select { - case <-ctx.Done(): - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "request cancelled", ctx.Err().Error()) - default: - } - - // Build command arguments - cmdArgs := []string{"mcp", "inspect"} - - if args.WorkflowFile != "" { - cmdArgs = append(cmdArgs, args.WorkflowFile) - } - - if args.Server != "" { - cmdArgs = append(cmdArgs, "--server", args.Server) - } - - if args.Tool != "" { - cmdArgs = append(cmdArgs, "--tool", args.Tool) - } +- Detailed tool information when tool parameter is specified` +} - // Always enable secret checking (will be silently ignored if GitHub token is not available) - cmdArgs = append(cmdArgs, "--check-secrets") +func registerMCPInspectToolHandler(ctx context.Context, execCmd execCmdFunc, args mcpInspectArgs) (*mcp.CallToolResult, any, error) { + if err := registerCompileToolCheckCancelled(ctx); err != nil { + return nil, nil, err + } - // Execute the CLI command - output, err := runMCPExecCombinedOutput(ctx, execCmd, cmdArgs...) + cmdArgs := registerMCPInspectToolArgs(args) + output, err := runMCPExecCombinedOutput(ctx, execCmd, cmdArgs...) + if err != nil { + return nil, nil, newMCPError(jsonrpc.CodeInternalError, "failed to inspect MCP servers", map[string]any{"error": err.Error(), "output": string(output)}) + } - if err != nil { - return nil, nil, newMCPError(jsonrpc.CodeInternalError, "failed to inspect MCP servers", map[string]any{"error": err.Error(), "output": string(output)}) - } + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: string(output)}}, + }, nil, nil +} - return &mcp.CallToolResult{ - Content: []mcp.Content{ - &mcp.TextContent{Text: string(output)}, - }, - }, nil, nil - }) +func registerMCPInspectToolArgs(args mcpInspectArgs) []string { + cmdArgs := []string{"mcp", "inspect"} + if args.WorkflowFile != "" { + cmdArgs = append(cmdArgs, args.WorkflowFile) + } + if args.Server != "" { + cmdArgs = append(cmdArgs, "--server", args.Server) + } + if args.Tool != "" { + cmdArgs = append(cmdArgs, "--tool", args.Tool) + } + // Always enable secret checking (will be silently ignored if GitHub token is not available) + return append(cmdArgs, "--check-secrets") } // checksArgs holds the input parameters for the checks tool. diff --git a/pkg/cli/mcp_validation.go b/pkg/cli/mcp_validation.go index 05c46c8ec91..1d96d4ff55d 100644 --- a/pkg/cli/mcp_validation.go +++ b/pkg/cli/mcp_validation.go @@ -30,108 +30,113 @@ func validateServerSecrets(config parser.RegistryMCPServerConfig, verbose bool, requiredSecrets := extractSecretsFromConfig(config) // Special case: Check for GH_AW_GITHUB_TOKEN when GitHub tool is in remote mode - if config.Name == "github" && config.Type == "http" { - mcpValidationLog.Print("GitHub remote mode detected, checking for GH_AW_GITHUB_TOKEN") - // GitHub remote mode requires GH_AW_GITHUB_TOKEN secret - // Check if a custom token is already specified in the env - hasCustomToken := false - for _, value := range config.Env { - if strings.Contains(value, "secrets.") && !strings.Contains(value, "GH_AW_GITHUB_TOKEN") { - // Custom token specified, no need to check GH_AW_GITHUB_TOKEN - hasCustomToken = true - break - } + requiredSecrets = validateServerSecretsGitHubRemote(config, requiredSecrets) + + if len(requiredSecrets) == 0 { + mcpValidationLog.Printf("No required secrets found, validating %d environment variables", len(config.Env)) + return validateServerSecretsEnv(config) + } + + // Check availability of required secrets + mcpValidationLog.Printf("Checking availability of %d required secrets", len(requiredSecrets)) + secretsStatus := checkSecretsAvailability(requiredSecrets, useActionsSecrets) + + // Separate secrets by availability + availableSecrets, missingSecrets := validateServerSecretsSplit(secretsStatus) + + // Display information about secrets + validateServerSecretsDisplayAvailable(availableSecrets, verbose) + + // Warn about missing secrets + validateServerSecretsDisplayMissing(missingSecrets) + + mcpValidationLog.Printf("Secret validation completed: available=%d, missing=%d", len(availableSecrets), len(missingSecrets)) + return nil +} + +func validateServerSecretsGitHubRemote(config parser.RegistryMCPServerConfig, requiredSecrets []SecretInfo) []SecretInfo { + if config.Name != "github" || config.Type != "http" { + return requiredSecrets + } + + mcpValidationLog.Print("GitHub remote mode detected, checking for GH_AW_GITHUB_TOKEN") + hasCustomToken := false + for _, value := range config.Env { + if strings.Contains(value, "secrets.") && !strings.Contains(value, "GH_AW_GITHUB_TOKEN") { + hasCustomToken = true + break } + } + if hasCustomToken || slices.ContainsFunc(requiredSecrets, func(secret SecretInfo) bool { + return secret.Name == "GH_AW_GITHUB_TOKEN" + }) { + return requiredSecrets + } + return append(requiredSecrets, SecretInfo{Name: "GH_AW_GITHUB_TOKEN", EnvKey: "GITHUB_TOKEN"}) +} - if !hasCustomToken { - // Add GH_AW_GITHUB_TOKEN to required secrets if not already present - alreadyPresent := false - for _, secret := range requiredSecrets { - if secret.Name == "GH_AW_GITHUB_TOKEN" { - alreadyPresent = true - break - } - } - if !alreadyPresent { - requiredSecrets = append(requiredSecrets, SecretInfo{ - Name: "GH_AW_GITHUB_TOKEN", - EnvKey: "GITHUB_TOKEN", - }) +func validateServerSecretsEnv(config parser.RegistryMCPServerConfig) error { + // No secrets required, proceed with normal env var validation + for key, value := range config.Env { + if strings.Contains(value, "${") { + if err := validateServerSecretsEnvReference(config, key, value); err != nil { + return err } + } else if err := validateServerSecretsEnvDirect(config, key, value); err != nil { + return err } } + return nil +} - if len(requiredSecrets) == 0 { - mcpValidationLog.Printf("No required secrets found, validating %d environment variables", len(config.Env)) - // No secrets required, proceed with normal env var validation - for key, value := range config.Env { - // Check if value contains variable references - if strings.Contains(value, "${") { - // Extract variable name (simplified parsing) - if strings.Contains(value, "secrets.") { - // This should have been caught by extractSecretsFromConfig - continue - } - if strings.Contains(value, "GH_TOKEN") || strings.Contains(value, "GITHUB_TOKEN") || strings.Contains(value, "GITHUB_PERSONAL_ACCESS_TOKEN") { - if token, err := parser.GetGitHubToken(); err != nil { - return errors.New("GitHub token not found in environment (set GH_TOKEN or GITHUB_TOKEN)") - } else { - config.Env[key] = token - } - } - // Handle our placeholder for GitHub token requirement - if strings.Contains(value, "GITHUB_TOKEN_REQUIRED") { - if token, err := parser.GetGitHubToken(); err != nil { - return fmt.Errorf("GitHub token required but not available: %w", err) - } else { - config.Env[key] = token - } - } +func validateServerSecretsEnvReference(config parser.RegistryMCPServerConfig, key string, value string) error { + if strings.Contains(value, "secrets.") { + return nil + } + if strings.Contains(value, "GH_TOKEN") || strings.Contains(value, "GITHUB_TOKEN") || strings.Contains(value, "GITHUB_PERSONAL_ACCESS_TOKEN") { + if token, err := parser.GetGitHubToken(); err != nil { + return errors.New("GitHub token not found in environment (set GH_TOKEN or GITHUB_TOKEN)") + } else { + config.Env[key] = token + } + } + if strings.Contains(value, "GITHUB_TOKEN_REQUIRED") { + if token, err := parser.GetGitHubToken(); err != nil { + return fmt.Errorf("GitHub token required but not available: %w", err) + } else { + config.Env[key] = token + } + } + return nil +} + +func validateServerSecretsEnvDirect(config parser.RegistryMCPServerConfig, key string, value string) error { + if value == "" { + return fmt.Errorf("environment variable '%s' has empty value", key) + } + if strings.Contains(value, "GITHUB_TOKEN_REQUIRED") { + if token, err := parser.GetGitHubToken(); err != nil { + return fmt.Errorf("GitHub token required but not available: %w", err) + } else { + config.Env[key] = token + } + } else if key == "GITHUB_PERSONAL_ACCESS_TOKEN" || key == "GITHUB_TOKEN" || key == "GH_TOKEN" { + if actualValue := os.Getenv(key); actualValue == "" { //nolint:osgetenvlibrary + if token, err := parser.GetGitHubToken(); err == nil { + config.Env[key] = token } else { - // For direct environment variable values (not containing ${}), - // check if they represent actual token values - if value == "" { - return fmt.Errorf("environment variable '%s' has empty value", key) - } - // If value contains "GITHUB_TOKEN_REQUIRED", treat it as needing validation - if strings.Contains(value, "GITHUB_TOKEN_REQUIRED") { - if token, err := parser.GetGitHubToken(); err != nil { - return fmt.Errorf("GitHub token required but not available: %w", err) - } else { - config.Env[key] = token - } - } else { - // Automatically try to get GitHub token for GitHub-related environment variables - if key == "GITHUB_PERSONAL_ACCESS_TOKEN" || key == "GITHUB_TOKEN" || key == "GH_TOKEN" { - if actualValue := os.Getenv(key); actualValue == "" { //nolint:osgetenvlibrary - // Try to automatically get the GitHub token - if token, err := parser.GetGitHubToken(); err == nil { - config.Env[key] = token - } else { - return fmt.Errorf("GitHub token required for '%s' but not available: %w", key, err) - } - } - } else { - // For backward compatibility: check if environment variable with this name exists - // This preserves the original behavior for existing tests - if actualValue := os.Getenv(key); actualValue == "" { //nolint:osgetenvlibrary - return fmt.Errorf("environment variable '%s' not set", key) - } - } - } + return fmt.Errorf("GitHub token required for '%s' but not available: %w", key, err) } } - return nil + } else if actualValue := os.Getenv(key); actualValue == "" { //nolint:osgetenvlibrary + return fmt.Errorf("environment variable '%s' not set", key) } + return nil +} - // Check availability of required secrets - mcpValidationLog.Printf("Checking availability of %d required secrets", len(requiredSecrets)) - secretsStatus := checkSecretsAvailability(requiredSecrets, useActionsSecrets) - - // Separate secrets by availability +func validateServerSecretsSplit(secretsStatus []SecretInfo) ([]SecretInfo, []SecretInfo) { var availableSecrets []SecretInfo var missingSecrets []SecretInfo - for _, secret := range secretsStatus { if secret.Available { availableSecrets = append(availableSecrets, secret) @@ -139,32 +144,32 @@ func validateServerSecrets(config parser.RegistryMCPServerConfig, verbose bool, missingSecrets = append(missingSecrets, secret) } } + return availableSecrets, missingSecrets +} - // Display information about secrets - if verbose { - if len(availableSecrets) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d available secret(s):", len(availableSecrets)))) - for _, secret := range availableSecrets { - source := "environment" - if secret.Source == "actions" { - source = "GitHub Actions" - } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf(" ✓ %s (from %s)", secret.Name, source))) - } - } +func validateServerSecretsDisplayAvailable(availableSecrets []SecretInfo, verbose bool) { + if !verbose || len(availableSecrets) == 0 { + return } - - // Warn about missing secrets - if len(missingSecrets) > 0 { - mcpValidationLog.Printf("Found %d missing secrets", len(missingSecrets)) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("⚠️ %d required secret(s) not found:", len(missingSecrets)))) - for _, secret := range missingSecrets { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(" ✗ "+secret.Name)) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d available secret(s):", len(availableSecrets)))) + for _, secret := range availableSecrets { + source := "environment" + if secret.Source == "actions" { + source = "GitHub Actions" } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf(" ✓ %s (from %s)", secret.Name, source))) } +} - mcpValidationLog.Printf("Secret validation completed: available=%d, missing=%d", len(availableSecrets), len(missingSecrets)) - return nil +func validateServerSecretsDisplayMissing(missingSecrets []SecretInfo) { + if len(missingSecrets) == 0 { + return + } + mcpValidationLog.Printf("Found %d missing secrets", len(missingSecrets)) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("⚠️ %d required secret(s) not found:", len(missingSecrets)))) + for _, secret := range missingSecrets { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(" ✗ "+secret.Name)) + } } // validateMCPServerConfiguration validates that the CLI is properly configured diff --git a/pkg/cli/model_costs.go b/pkg/cli/model_costs.go index 545c56eabfb..03fdd2b56ed 100644 --- a/pkg/cli/model_costs.go +++ b/pkg/cli/model_costs.go @@ -94,13 +94,24 @@ func findModelPricing(provider, model string) (map[string]float64, bool) { } comparableFullID := modelsdev.NormalizeComparableModelID(fullID) + if pricing, ok := findModelPricingExact(provider, model, fullID, comparableFullID); ok { + return pricing, true + } + + return findModelPricingBest(provider, model, normalizedProvider, normalizedModel, comparableModel) +} + +func findModelPricingExact(provider string, model string, fullID string, comparableFullID string) (map[string]float64, bool) { for _, record := range modelPriceRecords { if (fullID != "" && record.id == fullID) || (comparableFullID != "" && modelsdev.NormalizeComparableModelID(record.id) == comparableFullID) { modelCostsLog.Printf("Exact pricing match: provider=%s, model=%s -> %s", provider, model, record.id) return record.pricing, true } } + return nil, false +} +func findModelPricingBest(provider string, model string, normalizedProvider string, normalizedModel string, comparableModel string) (map[string]float64, bool) { var bestProviderScoped map[string]float64 bestProviderScopedLen := -1 var bestGeneric map[string]float64 diff --git a/pkg/cli/observability_insights.go b/pkg/cli/observability_insights.go index 11b175275dd..7ab765dbc48 100644 --- a/pkg/cli/observability_insights.go +++ b/pkg/cli/observability_insights.go @@ -42,9 +42,20 @@ func buildAuditObservabilityInsights(processedRun ProcessedRun, metrics MetricsD insights := make([]ObservabilityInsight, 0, 5) toolTypes := len(toolUsage) + insights = buildAuditObservabilityInsightsExecution(insights, metrics, toolTypes) + insights = buildAuditObservabilityInsightsActuation(insights, processedRun, createdItems) + insights = buildAuditObservabilityInsightsFriction(insights, processedRun) + insights = buildAuditObservabilityInsightsNetwork(insights, processedRun) + insights = buildAuditObservabilityInsightsPrivacy(insights, processedRun) + + observabilityInsightsLog.Printf("Audit observability insights built: count=%d", len(insights)) + return insights +} + +func buildAuditObservabilityInsightsExecution(insights []ObservabilityInsight, metrics MetricsData, toolTypes int) []ObservabilityInsight { switch { case metrics.Turns >= 12 || toolTypes >= 6: - insights = append(insights, ObservabilityInsight{ + return append(insights, ObservabilityInsight{ Category: "execution", Severity: "medium", Title: "Exploratory execution path", @@ -52,7 +63,7 @@ func buildAuditObservabilityInsights(processedRun ProcessedRun, metrics MetricsD Evidence: fmt.Sprintf("turns=%d tool_types=%d", metrics.Turns, toolTypes), }) case metrics.Turns >= 6 || toolTypes >= 4: - insights = append(insights, ObservabilityInsight{ + return append(insights, ObservabilityInsight{ Category: "execution", Severity: "info", Title: "Adaptive execution path", @@ -60,7 +71,7 @@ func buildAuditObservabilityInsights(processedRun ProcessedRun, metrics MetricsD Evidence: fmt.Sprintf("turns=%d tool_types=%d", metrics.Turns, toolTypes), }) default: - insights = append(insights, ObservabilityInsight{ + return append(insights, ObservabilityInsight{ Category: "execution", Severity: "info", Title: "Directed execution path", @@ -68,27 +79,30 @@ func buildAuditObservabilityInsights(processedRun ProcessedRun, metrics MetricsD Evidence: fmt.Sprintf("turns=%d tool_types=%d", metrics.Turns, toolTypes), }) } +} +func buildAuditObservabilityInsightsActuation(insights []ObservabilityInsight, processedRun ProcessedRun, createdItems []CreatedItemReport) []ObservabilityInsight { createdCount := len(createdItems) safeItemsCount := processedRun.Run.SafeItemsCount if createdCount > 0 || safeItemsCount > 0 { - insights = append(insights, ObservabilityInsight{ + return append(insights, ObservabilityInsight{ Category: "actuation", Severity: "info", Title: "Write path executed", Summary: fmt.Sprintf("The workflow crossed from analysis into action, producing %d created item(s) and %d safe output action(s).", createdCount, safeItemsCount), Evidence: fmt.Sprintf("created_items=%d safe_items=%d", createdCount, safeItemsCount), }) - } else { - insights = append(insights, ObservabilityInsight{ - Category: "actuation", - Severity: "info", - Title: "Read-only posture observed", - Summary: "The workflow stayed in an analysis posture and did not emit any GitHub write actions.", - Evidence: "created_items=0 safe_items=0", - }) } + return append(insights, ObservabilityInsight{ + Category: "actuation", + Severity: "info", + Title: "Read-only posture observed", + Summary: "The workflow stayed in an analysis posture and did not emit any GitHub write actions.", + Evidence: "created_items=0 safe_items=0", + }) +} +func buildAuditObservabilityInsightsFriction(insights []ObservabilityInsight, processedRun ProcessedRun) []ObservabilityInsight { frictionEvents := len(processedRun.MissingTools) + len(processedRun.MCPFailures) + len(processedRun.MissingData) if frictionEvents > 0 { severity := "medium" @@ -103,7 +117,10 @@ func buildAuditObservabilityInsights(processedRun ProcessedRun, metrics MetricsD Evidence: fmt.Sprintf("missing_tools=%d mcp_failures=%d missing_data=%d", len(processedRun.MissingTools), len(processedRun.MCPFailures), len(processedRun.MissingData)), }) } + return insights +} +func buildAuditObservabilityInsightsNetwork(insights []ObservabilityInsight, processedRun ProcessedRun) []ObservabilityInsight { if processedRun.FirewallAnalysis != nil && processedRun.FirewallAnalysis.TotalRequests > 0 { blockedRate := float64(processedRun.FirewallAnalysis.BlockedRequests) / float64(processedRun.FirewallAnalysis.TotalRequests) severity := "info" @@ -125,7 +142,10 @@ func buildAuditObservabilityInsights(processedRun ProcessedRun, metrics MetricsD Evidence: fmt.Sprintf("blocked=%d total=%d", processedRun.FirewallAnalysis.BlockedRequests, processedRun.FirewallAnalysis.TotalRequests), }) } + return insights +} +func buildAuditObservabilityInsightsPrivacy(insights []ObservabilityInsight, processedRun ProcessedRun) []ObservabilityInsight { if processedRun.RedactedDomainsAnalysis != nil && processedRun.RedactedDomainsAnalysis.TotalDomains > 0 { insights = append(insights, ObservabilityInsight{ Category: "privacy", @@ -135,8 +155,6 @@ func buildAuditObservabilityInsights(processedRun ProcessedRun, metrics MetricsD Evidence: fmt.Sprintf("redacted_domains=%d", processedRun.RedactedDomainsAnalysis.TotalDomains), }) } - - observabilityInsightsLog.Printf("Audit observability insights built: count=%d", len(insights)) return insights } @@ -147,40 +165,25 @@ func buildLogsObservabilityInsights(processedRuns []ProcessedRun, toolUsage []To observabilityInsightsLog.Printf("Building logs observability insights: processed_runs=%d tool_usage_entries=%d", len(processedRuns), len(toolUsage)) insights := make([]ObservabilityInsight, 0, 6) + workflowStats, writeRuns, readOnlyRuns := buildLogsObservabilityInsightsStats(processedRuns) + + insights = buildLogsObservabilityInsightsFailure(insights, workflowStats) + insights = buildLogsObservabilityInsightsDrift(insights, workflowStats) + insights = buildLogsObservabilityInsightsTooling(insights, workflowStats) + insights = buildLogsObservabilityInsightsNetwork(insights, workflowStats) + insights = buildLogsObservabilityInsightsActuation(insights, processedRuns, writeRuns, readOnlyRuns) + insights = buildLogsObservabilityInsightsToolConcentration(insights, toolUsage) + + observabilityInsightsLog.Printf("Logs observability insights built: count=%d write_runs=%d read_only_runs=%d", len(insights), writeRuns, readOnlyRuns) + return insights +} + +func buildLogsObservabilityInsightsStats(processedRuns []ProcessedRun) (map[string]*workflowObservabilityStats, int, int) { workflowStats := make(map[string]*workflowObservabilityStats) writeRuns := 0 readOnlyRuns := 0 - for _, pr := range processedRuns { - wfID := workflowIDFromRun(pr.Run.WorkflowPath, pr.Run.WorkflowName) - stats, exists := workflowStats[wfID] - if !exists { - stats = &workflowObservabilityStats{ - workflowName: wfID, - minTurns: pr.Run.Turns, - maxTurns: pr.Run.Turns, - } - workflowStats[wfID] = stats - } - - stats.runs++ - stats.totalTurns += pr.Run.Turns - if stats.runs == 1 || pr.Run.Turns < stats.minTurns { - stats.minTurns = pr.Run.Turns - } - if pr.Run.Turns > stats.maxTurns { - stats.maxTurns = pr.Run.Turns - } - if pr.Run.Conclusion == "failure" { - stats.failures++ - } - if pr.Run.Conclusion == "timed_out" { - stats.timedOuts++ - } - stats.missingTools += len(pr.MissingTools) - stats.mcpFailures += len(pr.MCPFailures) - stats.missingData += len(pr.MissingData) - stats.safeItems += pr.Run.SafeItemsCount + stats := buildLogsObservabilityInsightsStatsForRun(workflowStats, pr) if pr.Run.SafeItemsCount > 0 { writeRuns++ } else { @@ -194,16 +197,39 @@ func buildLogsObservabilityInsights(processedRuns []ProcessedRun, toolUsage []To } } } + return workflowStats, writeRuns, readOnlyRuns +} - var failureHotspot *workflowObservabilityStats - for _, stats := range workflowStats { - if stats.failures == 0 { - continue - } - if failureHotspot == nil || stats.failures > failureHotspot.failures || (stats.failures == failureHotspot.failures && stats.workflowName < failureHotspot.workflowName) { - failureHotspot = stats - } +func buildLogsObservabilityInsightsStatsForRun(workflowStats map[string]*workflowObservabilityStats, pr ProcessedRun) *workflowObservabilityStats { + wfID := workflowIDFromRun(pr.Run.WorkflowPath, pr.Run.WorkflowName) + stats, exists := workflowStats[wfID] + if !exists { + stats = &workflowObservabilityStats{workflowName: wfID, minTurns: pr.Run.Turns, maxTurns: pr.Run.Turns} + workflowStats[wfID] = stats + } + stats.runs++ + stats.totalTurns += pr.Run.Turns + if stats.runs == 1 || pr.Run.Turns < stats.minTurns { + stats.minTurns = pr.Run.Turns + } + if pr.Run.Turns > stats.maxTurns { + stats.maxTurns = pr.Run.Turns } + if pr.Run.Conclusion == "failure" { + stats.failures++ + } + if pr.Run.Conclusion == "timed_out" { + stats.timedOuts++ + } + stats.missingTools += len(pr.MissingTools) + stats.mcpFailures += len(pr.MCPFailures) + stats.missingData += len(pr.MissingData) + stats.safeItems += pr.Run.SafeItemsCount + return stats +} + +func buildLogsObservabilityInsightsFailure(insights []ObservabilityInsight, workflowStats map[string]*workflowObservabilityStats) []ObservabilityInsight { + failureHotspot := buildLogsObservabilityInsightsFailureHotspot(workflowStats) if failureHotspot != nil { failureRate := float64(failureHotspot.failures) / float64(failureHotspot.runs) severity := "medium" @@ -219,19 +245,24 @@ func buildLogsObservabilityInsights(processedRuns []ProcessedRun, toolUsage []To Evidence: fmt.Sprintf("workflow=%s failures=%d runs=%d", failureHotspot.workflowName, failureHotspot.failures, failureHotspot.runs), }) } + return insights +} - var driftHotspot *workflowObservabilityStats +func buildLogsObservabilityInsightsFailureHotspot(workflowStats map[string]*workflowObservabilityStats) *workflowObservabilityStats { + var failureHotspot *workflowObservabilityStats for _, stats := range workflowStats { - if stats.runs < 2 { - continue - } - if stats.maxTurns-stats.minTurns < 4 { + if stats.failures == 0 { continue } - if driftHotspot == nil || (stats.maxTurns-stats.minTurns) > (driftHotspot.maxTurns-driftHotspot.minTurns) { - driftHotspot = stats + if failureHotspot == nil || stats.failures > failureHotspot.failures || (stats.failures == failureHotspot.failures && stats.workflowName < failureHotspot.workflowName) { + failureHotspot = stats } } + return failureHotspot +} + +func buildLogsObservabilityInsightsDrift(insights []ObservabilityInsight, workflowStats map[string]*workflowObservabilityStats) []ObservabilityInsight { + driftHotspot := buildLogsObservabilityInsightsDriftHotspot(workflowStats) if driftHotspot != nil { avgTurns := float64(driftHotspot.totalTurns) / float64(driftHotspot.runs) observabilityInsightsLog.Printf("Execution drift detected: workflow=%s min_turns=%d max_turns=%d avg_turns=%.1f", driftHotspot.workflowName, driftHotspot.minTurns, driftHotspot.maxTurns, avgTurns) @@ -243,17 +274,23 @@ func buildLogsObservabilityInsights(processedRuns []ProcessedRun, toolUsage []To Evidence: fmt.Sprintf("workflow=%s min_turns=%d max_turns=%d", driftHotspot.workflowName, driftHotspot.minTurns, driftHotspot.maxTurns), }) } + return insights +} - var toolingHotspot *workflowObservabilityStats +func buildLogsObservabilityInsightsDriftHotspot(workflowStats map[string]*workflowObservabilityStats) *workflowObservabilityStats { + var driftHotspot *workflowObservabilityStats for _, stats := range workflowStats { - friction := stats.missingTools + stats.mcpFailures + stats.missingData - if friction == 0 { - continue - } - if toolingHotspot == nil || friction > (toolingHotspot.missingTools+toolingHotspot.mcpFailures+toolingHotspot.missingData) { - toolingHotspot = stats + if stats.runs >= 2 && stats.maxTurns-stats.minTurns >= 4 { + if driftHotspot == nil || (stats.maxTurns-stats.minTurns) > (driftHotspot.maxTurns-driftHotspot.minTurns) { + driftHotspot = stats + } } } + return driftHotspot +} + +func buildLogsObservabilityInsightsTooling(insights []ObservabilityInsight, workflowStats map[string]*workflowObservabilityStats) []ObservabilityInsight { + toolingHotspot := buildLogsObservabilityInsightsToolingHotspot(workflowStats) if toolingHotspot != nil { friction := toolingHotspot.missingTools + toolingHotspot.mcpFailures + toolingHotspot.missingData severity := "medium" @@ -268,19 +305,25 @@ func buildLogsObservabilityInsights(processedRuns []ProcessedRun, toolUsage []To Evidence: fmt.Sprintf("workflow=%s missing_tools=%d mcp_failures=%d missing_data=%d", toolingHotspot.workflowName, toolingHotspot.missingTools, toolingHotspot.mcpFailures, toolingHotspot.missingData), }) } + return insights +} - var networkHotspot *workflowObservabilityStats - var networkRate float64 +func buildLogsObservabilityInsightsToolingHotspot(workflowStats map[string]*workflowObservabilityStats) *workflowObservabilityStats { + var toolingHotspot *workflowObservabilityStats for _, stats := range workflowStats { - if stats.totalNet == 0 || stats.blocked == 0 { + friction := stats.missingTools + stats.mcpFailures + stats.missingData + if friction == 0 { continue } - rate := float64(stats.blocked) / float64(stats.totalNet) - if networkHotspot == nil || rate > networkRate { - networkHotspot = stats - networkRate = rate + if toolingHotspot == nil || friction > (toolingHotspot.missingTools+toolingHotspot.mcpFailures+toolingHotspot.missingData) { + toolingHotspot = stats } } + return toolingHotspot +} + +func buildLogsObservabilityInsightsNetwork(insights []ObservabilityInsight, workflowStats map[string]*workflowObservabilityStats) []ObservabilityInsight { + networkHotspot, networkRate := buildLogsObservabilityInsightsNetworkHotspot(workflowStats) if networkHotspot != nil { severity := "medium" if networkRate >= 0.5 || (networkHotspot.blocked >= 10 && !networkHotspot.blockedAtCap) { @@ -294,7 +337,26 @@ func buildLogsObservabilityInsights(processedRuns []ProcessedRun, toolUsage []To Evidence: fmt.Sprintf("workflow=%s blocked=%d total=%d", networkHotspot.workflowName, networkHotspot.blocked, networkHotspot.totalNet), }) } + return insights +} +func buildLogsObservabilityInsightsNetworkHotspot(workflowStats map[string]*workflowObservabilityStats) (*workflowObservabilityStats, float64) { + var networkHotspot *workflowObservabilityStats + var networkRate float64 + for _, stats := range workflowStats { + if stats.totalNet == 0 || stats.blocked == 0 { + continue + } + rate := float64(stats.blocked) / float64(stats.totalNet) + if networkHotspot == nil || rate > networkRate { + networkHotspot = stats + networkRate = rate + } + } + return networkHotspot, networkRate +} + +func buildLogsObservabilityInsightsActuation(insights []ObservabilityInsight, processedRuns []ProcessedRun, writeRuns int, readOnlyRuns int) []ObservabilityInsight { if writeRuns > 0 || readOnlyRuns > 0 { insights = append(insights, ObservabilityInsight{ Category: "actuation", @@ -304,7 +366,10 @@ func buildLogsObservabilityInsights(processedRuns []ProcessedRun, toolUsage []To Evidence: fmt.Sprintf("write_runs=%d read_only_runs=%d", writeRuns, readOnlyRuns), }) } + return insights +} +func buildLogsObservabilityInsightsToolConcentration(insights []ObservabilityInsight, toolUsage []ToolUsageSummary) []ObservabilityInsight { totalToolCalls := 0 for _, tool := range toolUsage { totalToolCalls += tool.TotalCalls @@ -326,8 +391,6 @@ func buildLogsObservabilityInsights(processedRuns []ProcessedRun, toolUsage []To }) } } - - observabilityInsightsLog.Printf("Logs observability insights built: count=%d write_runs=%d read_only_runs=%d", len(insights), writeRuns, readOnlyRuns) return insights } diff --git a/pkg/cli/org_runner.go b/pkg/cli/org_runner.go index dd4e4eb51bc..0709edd263b 100644 --- a/pkg/cli/org_runner.go +++ b/pkg/cli/org_runner.go @@ -136,6 +136,44 @@ func confirmOrgAction(preview orgRepoPreview, action string, autoYes bool) (bool // 7. When createPR or createIssue is set, iterates through results calling // cbs.ApplyFn or cbs.IssueFn, continuing past per-repo errors. func runCommandForOrg(ctx context.Context, org string, repoGlobs []string, cbs orgRunCallbacks, createPR bool, createIssue bool, verbose bool) error { + if err := runCommandForOrgValidate(org, repoGlobs, cbs, createPR, createIssue); err != nil { + return err + } + + // Handle Ctrl-C / SIGTERM so an interrupted run still renders the report + // gathered so far instead of exiting abruptly. + ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + defer stop() + + repos, err := runCommandForOrgDiscover(ctx, org, repoGlobs, cbs, verbose) + if err != nil { + return err + } + if len(repos) == 0 { + return nil + } + + results, ok := runCommandForOrgScan(ctx, repos, cbs, verbose) + if !ok { + return nil + } + + runCommandForOrgSort(results) + + // Always render the report before applying anything: it is cheap and lets + // the user see results even if the run is stopped early. + cbs.ReportFn(results, createPR || createIssue) + + if !createPR && !createIssue { + return nil + } + if createIssue { + return runCommandForOrgCreateIssues(ctx, results, cbs, verbose) + } + return runCommandForOrgCreatePRs(ctx, results, cbs, verbose) +} + +func runCommandForOrgValidate(org string, repoGlobs []string, cbs orgRunCallbacks, createPR bool, createIssue bool) error { if strings.TrimSpace(org) == "" { return errors.New("--org cannot be empty") } @@ -160,12 +198,10 @@ func runCommandForOrg(ctx context.Context, org string, repoGlobs []string, cbs o if (createPR || createIssue) && !cbs.AutoYes && isRunningInCIFn() { return errors.New("confirmation is required for --org create operations in CI; re-run with --yes to auto-accept") } + return nil +} - // Handle Ctrl-C / SIGTERM so an interrupted run still renders the report - // gathered so far instead of exiting abruptly. - ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) - defer stop() - +func runCommandForOrgDiscover(ctx context.Context, org string, repoGlobs []string, cbs orgRunCallbacks, verbose bool) ([]string, error) { discMsg := cbs.DiscoveringMsg if discMsg == "" { discMsg = "Discovering repositories in " + org + "..." @@ -174,7 +210,7 @@ func runCommandForOrg(ctx context.Context, org string, repoGlobs []string, cbs o repoPaths, err := cbs.SearchFn(ctx, org, verbose) if err != nil { - return err + return nil, err } if len(repoPaths) == 0 { noReposMsg := cbs.NoReposMsg @@ -182,87 +218,114 @@ func runCommandForOrg(ctx context.Context, org string, repoGlobs []string, cbs o noReposMsg = "No repositories found" } fmt.Fprintln(os.Stderr, console.FormatInfoMessage(noReposMsg)) - return nil + return nil, nil } repos := filterOrgRepos(repoPaths, repoGlobs) if len(repos) == 0 { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No repositories matched the requested --repos filters")) - return nil } + return repos, nil +} - // Build the result set. - var results []orgRepoPreview - +func runCommandForOrgScan(ctx context.Context, repos []string, cbs orgRunCallbacks, verbose bool) ([]orgRepoPreview, bool) { if cbs.ScanFn == nil { - // No per-repo scanning: include every discovered repo directly. - results = make([]orgRepoPreview, 0, len(repos)) + results := make([]orgRepoPreview, 0, len(repos)) for _, repo := range repos { results = append(results, orgRepoPreview{Repo: repo}) } - } else { - total := len(repos) - scanLabel := cbs.ScanLabel - if scanLabel == "" { - scanLabel = "Inspecting" + return results, true + } + + results, stopped := runCommandForOrgScanRepos(ctx, repos, cbs, verbose) + if len(results) > 0 { + return results, true + } + if stopped { + msg := cbs.NoResultsStopMsg + if msg == "" { + msg = "No results found before processing stopped" } - results = make([]orgRepoPreview, 0, len(repos)) - stopped := false - - for i, repo := range repos { - // Honor a cancellation signal between repos so we can still show - // the report for the work completed so far. - if ctx.Err() != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Cancellation requested; stopping after %d/%d repositories", i, total))) - orgRunnerLog.Printf("Context canceled during scan at repo %d/%d: %v", i, total, ctx.Err()) - stopped = true - break - } - - fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("[%d/%d] %s %s", i+1, total, scanLabel, repo))) - - if err := waitForOrgRateLimitFn(ctx, "core", verbose); err != nil { - if errors.Is(err, errOrgRateLimitCritical) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("GitHub API budget critical; stopping after %d/%d repositories and reporting what was found", i, total))) - orgRunnerLog.Printf("Rate limit critical during scan at repo %d/%d", i, total) - stopped = true - break - } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Continuing after rate limit check failure for %s: %v", repo, err))) - } - } - - preview, include, scanErr := cbs.ScanFn(ctx, repo, verbose) - if scanErr != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: %v", repo, scanErr))) - orgRunnerLog.Printf("Failed to scan %s: %v", repo, scanErr) - continue - } - if !include { - continue - } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(msg)) + return nil, false + } + msg := cbs.NoResultsMsg + if msg == "" { + msg = "All matching repositories are already up to date" + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(msg)) + return nil, false +} + +func runCommandForOrgScanRepos(ctx context.Context, repos []string, cbs orgRunCallbacks, verbose bool) ([]orgRepoPreview, bool) { + total := len(repos) + scanLabel := cbs.ScanLabel + if scanLabel == "" { + scanLabel = "Inspecting" + } + results := make([]orgRepoPreview, 0, len(repos)) + stopped := false + + for i, repo := range repos { + if runCommandForOrgScanStopped(ctx, i, total) { + stopped = true + break + } + fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("[%d/%d] %s %s", i+1, total, scanLabel, repo))) + if runCommandForOrgRateLimited(ctx, "scan", i, total, repo, 0, verbose) { + stopped = true + break + } + preview, include, scanErr := cbs.ScanFn(ctx, repo, verbose) + if scanErr != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: %v", repo, scanErr))) + orgRunnerLog.Printf("Failed to scan %s: %v", repo, scanErr) + continue + } + if include { results = append(results, preview) } + } + return results, stopped +} - if len(results) == 0 { - if stopped { - msg := cbs.NoResultsStopMsg - if msg == "" { - msg = "No results found before processing stopped" - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(msg)) - return nil - } - msg := cbs.NoResultsMsg - if msg == "" { - msg = "All matching repositories are already up to date" - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(msg)) - return nil +func runCommandForOrgScanStopped(ctx context.Context, index int, total int) bool { + if ctx.Err() == nil { + return false + } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Cancellation requested; stopping after %d/%d repositories", index, total))) + orgRunnerLog.Printf("Context canceled during scan at repo %d/%d: %v", index, total, ctx.Err()) + return true +} + +func runCommandForOrgRateLimited(ctx context.Context, phase string, index int, total int, repo string, processed int, verbose bool) bool { + if err := waitForOrgRateLimitFn(ctx, "core", verbose); err != nil { + if errors.Is(err, errOrgRateLimitCritical) { + runCommandForOrgRateLimitCritical(phase, index, total, processed) + return true } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Continuing after rate limit check failure for %s: %v", repo, err))) + } + } + return false +} + +func runCommandForOrgRateLimitCritical(phase string, index int, total int, processed int) { + switch phase { + case "scan": + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("GitHub API budget critical; stopping after %d/%d repositories and reporting what was found", index, total))) + orgRunnerLog.Printf("Rate limit critical during scan at repo %d/%d", index, total) + case "issue": + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("GitHub API budget critical; created issues for %d/%d repositories", processed, total))) + orgRunnerLog.Printf("Rate limit critical during issue creation at %d/%d", index, total) + default: + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("GitHub API budget critical; processed %d/%d repositories", processed, total))) + orgRunnerLog.Printf("Rate limit critical during apply at %d/%d", index, total) } +} +func runCommandForOrgSort(results []orgRepoPreview) { // Sort by oldest-edit time (oldest first); ties broken alphabetically. slices.SortStableFunc(results, func(a, b orgRepoPreview) int { if a.OldestEdit.IsZero() && b.OldestEdit.IsZero() { @@ -282,120 +345,108 @@ func runCommandForOrg(ctx context.Context, org string, repoGlobs []string, cbs o } return 1 }) +} - // Always render the report before applying anything: it is cheap and lets - // the user see results even if the run is stopped early. - cbs.ReportFn(results, createPR || createIssue) - - if !createPR && !createIssue { - return nil +func runCommandForOrgCreateIssues(ctx context.Context, results []orgRepoPreview, cbs orgRunCallbacks, verbose bool) error { + issueLabel := cbs.IssueLabel + if issueLabel == "" { + issueLabel = "Creating issue in" } - - if createIssue { - issueLabel := cbs.IssueLabel - if issueLabel == "" { - issueLabel = "Creating issue in" - } - processed := 0 - attempted := 0 - for i, result := range results { - if ctx.Err() != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Cancellation requested; created issues for %d/%d repositories", processed, len(results)))) - orgRunnerLog.Printf("Context canceled during issue creation at %d/%d: %v", i, len(results), ctx.Err()) - return nil - } - if err := waitForOrgRateLimitFn(ctx, "core", verbose); err != nil { - if errors.Is(err, errOrgRateLimitCritical) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("GitHub API budget critical; created issues for %d/%d repositories", processed, len(results)))) - orgRunnerLog.Printf("Rate limit critical during issue creation at %d/%d", i, len(results)) - return nil - } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Continuing after rate limit check failure for %s: %v", result.Repo, err))) - } - } - fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("[%d/%d] %s %s%s", i+1, len(results), issueLabel, result.Repo, orgWorkflowCountSuffix(result)))) - confirmed, err := confirmOrgAction(result, "create an issue", cbs.AutoYes) - if err != nil { - return err - } - if !confirmed { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipped "+result.Repo)) - continue - } - attempted++ - if err := cbs.IssueFn(ctx, result, verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: %v", result.Repo, err))) - orgRunnerLog.Printf("Failed to create issue in %s: %v", result.Repo, err) - continue - } - processed++ - } - if attempted == 0 { + processed, attempted, stopped, err := runCommandForOrgApplyLoop(ctx, results, cbs, verbose, issueLabel, "create an issue", "issue") + if err != nil || stopped || attempted == 0 { + if attempted == 0 && !stopped && err == nil { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No repositories were accepted for issue creation")) - return nil } - if processed == 0 { - msg := cbs.AllFailIssueMsg - if msg == "" { - msg = "failed to create issues in any repository" - } - return errors.New(msg) + return err + } + if processed == 0 { + msg := cbs.AllFailIssueMsg + if msg == "" { + msg = "failed to create issues in any repository" } - return nil + return errors.New(msg) } + return nil +} - // createPR +func runCommandForOrgCreatePRs(ctx context.Context, results []orgRepoPreview, cbs orgRunCallbacks, verbose bool) error { applyLabel := cbs.ApplyLabel if applyLabel == "" { applyLabel = "Processing" } + processed, attempted, stopped, err := runCommandForOrgApplyLoop(ctx, results, cbs, verbose, applyLabel, "create a pull request", "apply") + if err != nil || stopped || attempted == 0 { + if attempted == 0 && !stopped && err == nil { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No repositories were accepted for pull request creation")) + } + return err + } + if processed == 0 { + msg := cbs.AllFailApplyMsg + if msg == "" { + msg = "failed to process any repository" + } + return errors.New(msg) + } + return nil +} + +func runCommandForOrgApplyLoop(ctx context.Context, results []orgRepoPreview, cbs orgRunCallbacks, verbose bool, label string, action string, phase string) (int, int, bool, error) { processed := 0 attempted := 0 for i, result := range results { - if ctx.Err() != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Cancellation requested; processed %d/%d repositories", processed, len(results)))) - orgRunnerLog.Printf("Context canceled during apply at %d/%d: %v", i, len(results), ctx.Err()) - return nil + if runCommandForOrgApplyStopped(ctx, phase, i, len(results), processed) { + return processed, attempted, true, nil } - if err := waitForOrgRateLimitFn(ctx, "core", verbose); err != nil { - if errors.Is(err, errOrgRateLimitCritical) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("GitHub API budget critical; processed %d/%d repositories", processed, len(results)))) - orgRunnerLog.Printf("Rate limit critical during apply at %d/%d", i, len(results)) - return nil - } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Continuing after rate limit check failure for %s: %v", result.Repo, err))) - } + if runCommandForOrgRateLimited(ctx, phase, i, len(results), result.Repo, processed, verbose) { + return processed, attempted, true, nil } - fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("[%d/%d] %s %s%s", i+1, len(results), applyLabel, result.Repo, orgWorkflowCountSuffix(result)))) - confirmed, err := confirmOrgAction(result, "create a pull request", cbs.AutoYes) + fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("[%d/%d] %s %s%s", i+1, len(results), label, result.Repo, orgWorkflowCountSuffix(result)))) + confirmed, err := confirmOrgAction(result, action, cbs.AutoYes) if err != nil { - return err + return processed, attempted, false, err } if !confirmed { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipped "+result.Repo)) continue } attempted++ - if err := cbs.ApplyFn(ctx, result, verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: %v", result.Repo, err))) - orgRunnerLog.Printf("Failed to apply to %s: %v", result.Repo, err) - continue + if runCommandForOrgApplyOne(ctx, result, cbs, verbose, phase) { + processed++ } - processed++ } - if attempted == 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No repositories were accepted for pull request creation")) - return nil + return processed, attempted, false, nil +} + +func runCommandForOrgApplyStopped(ctx context.Context, phase string, index int, total int, processed int) bool { + if ctx.Err() == nil { + return false } - if processed == 0 { - msg := cbs.AllFailApplyMsg - if msg == "" { - msg = "failed to process any repository" - } - return errors.New(msg) + if phase == "issue" { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Cancellation requested; created issues for %d/%d repositories", processed, total))) + orgRunnerLog.Printf("Context canceled during issue creation at %d/%d: %v", index, total, ctx.Err()) + } else { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Cancellation requested; processed %d/%d repositories", processed, total))) + orgRunnerLog.Printf("Context canceled during apply at %d/%d: %v", index, total, ctx.Err()) } + return true +} - return nil +func runCommandForOrgApplyOne(ctx context.Context, result orgRepoPreview, cbs orgRunCallbacks, verbose bool, phase string) bool { + var err error + if phase == "issue" { + err = cbs.IssueFn(ctx, result, verbose) + } else { + err = cbs.ApplyFn(ctx, result, verbose) + } + if err == nil { + return true + } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: %v", result.Repo, err))) + if phase == "issue" { + orgRunnerLog.Printf("Failed to create issue in %s: %v", result.Repo, err) + } else { + orgRunnerLog.Printf("Failed to apply to %s: %v", result.Repo, err) + } + return false } diff --git a/pkg/cli/outcome_domain_breakdown.go b/pkg/cli/outcome_domain_breakdown.go index a0677045d99..fd135fdb94c 100644 --- a/pkg/cli/outcome_domain_breakdown.go +++ b/pkg/cli/outcome_domain_breakdown.go @@ -30,9 +30,16 @@ func ComputeDomainBreakdowns(reports []OutcomeReport) []DomainBreakdown { return []DomainBreakdown{} } - // Map domain label → metrics - domains := make(map[string]*DomainBreakdown) + domains := computeDomainBreakdownsAggregate(reports) + result := computeDomainBreakdownsFinalize(domains) + computeDomainBreakdownsSort(result) + + domainBreakdownLog.Printf("Computed domain breakdowns: domains=%d, total_attempted=%d", len(result), countTotalAttempted(result)) + return result +} +func computeDomainBreakdownsAggregate(reports []OutcomeReport) map[string]*DomainBreakdown { + domains := make(map[string]*DomainBreakdown) for _, report := range reports { // If outcome has objective labels, aggregate by each label for _, label := range report.ObjectiveLabels { @@ -78,7 +85,10 @@ func ComputeDomainBreakdowns(reports []OutcomeReport) []DomainBreakdown { } } } + return domains +} +func computeDomainBreakdownsFinalize(domains map[string]*DomainBreakdown) []DomainBreakdown { // Compute efficiency metrics for each domain result := make([]DomainBreakdown, 0, len(domains)) for _, domain := range domains { @@ -90,7 +100,10 @@ func ComputeDomainBreakdowns(reports []OutcomeReport) []DomainBreakdown { } result = append(result, *domain) } + return result +} +func computeDomainBreakdownsSort(result []DomainBreakdown) { // Sort by total_objective_value descending slices.SortFunc(result, func(a, b DomainBreakdown) int { if a.TotalObjectiveValue != b.TotalObjectiveValue { @@ -98,9 +111,6 @@ func ComputeDomainBreakdowns(reports []OutcomeReport) []DomainBreakdown { } return strings.Compare(a.Label, b.Label) }) - - domainBreakdownLog.Printf("Computed domain breakdowns: domains=%d, total_attempted=%d", len(result), countTotalAttempted(result)) - return result } func countTotalAttempted(breakdowns []DomainBreakdown) int { diff --git a/pkg/cli/outcome_eval.go b/pkg/cli/outcome_eval.go index 70fbc981966..ad8c4834af8 100644 --- a/pkg/cli/outcome_eval.go +++ b/pkg/cli/outcome_eval.go @@ -158,50 +158,72 @@ func EvaluateOutcomes(items []CreatedItemReport, repoOverride string, mapping *g // ComputeOutcomeSummary aggregates outcome reports into a summary. // The mapping parameter is required and defines how labels map to objective values. func ComputeOutcomeSummary(reports []OutcomeReport, mapping *github.ObjectiveMapping) OutcomeSummary { + s, times := computeOutcomeSummaryAccumulate(reports) + computeOutcomeSummaryRates(&s, times) + + // Compute domain breakdowns + s.DomainBreakdowns = ComputeDomainBreakdowns(reports) + + return s +} + +func computeOutcomeSummaryAccumulate(reports []OutcomeReport) (OutcomeSummary, []float64) { s := OutcomeSummary{Total: len(reports)} var times []float64 for _, r := range reports { - eval := normalizeOutcomeEvaluation(r) - switch eval.OutcomeStatus { - case OutcomeStatusAccepted: - s.Accepted++ - switch eval.EvidenceStrength { - case EvidenceStrong: - s.AcceptedStrong++ - case EvidenceMedium: - s.AcceptedMedium++ - case EvidenceWeak: - s.AcceptedWeak++ - } - if r.ZeroTouch { - s.ZeroTouch++ - } - case OutcomeStatusRejected: - s.Rejected++ - case OutcomeStatusIgnored: - s.Ignored++ - case OutcomeStatusPending: - s.Pending++ - } - if eval.Signal == "target_exists_only" { - s.FallbackExistsOnlyCount++ - } - switch r.Result { - case OutcomeLifecycle, OutcomeLifecycleClose: - s.Lifecycle++ - case OutcomeError: - s.Errors++ - } - if r.TimeToOutcomeHours > 0 { - times = append(times, r.TimeToOutcomeHours) - } + computeOutcomeSummaryApplyReport(&s, r, ×) + } + return s, times +} + +func computeOutcomeSummaryApplyReport(s *OutcomeSummary, r OutcomeReport, times *[]float64) { + eval := normalizeOutcomeEvaluation(r) + computeOutcomeSummaryApplyStatus(s, eval, r) + if eval.Signal == "target_exists_only" { + s.FallbackExistsOnlyCount++ + } + switch r.Result { + case OutcomeLifecycle, OutcomeLifecycleClose: + s.Lifecycle++ + case OutcomeError: + s.Errors++ + } + if r.TimeToOutcomeHours > 0 { + *times = append(*times, r.TimeToOutcomeHours) + } + + // Aggregate objective values + s.TotalObjectiveValue += r.ObjectiveValue + if eval.OutcomeStatus == OutcomeStatusAccepted { + s.AcceptedObjectiveValue += r.ObjectiveValue + } +} - // Aggregate objective values - s.TotalObjectiveValue += r.ObjectiveValue - if eval.OutcomeStatus == OutcomeStatusAccepted { - s.AcceptedObjectiveValue += r.ObjectiveValue +func computeOutcomeSummaryApplyStatus(s *OutcomeSummary, eval OutcomeEvaluation, r OutcomeReport) { + switch eval.OutcomeStatus { + case OutcomeStatusAccepted: + s.Accepted++ + switch eval.EvidenceStrength { + case EvidenceStrong: + s.AcceptedStrong++ + case EvidenceMedium: + s.AcceptedMedium++ + case EvidenceWeak: + s.AcceptedWeak++ } + if r.ZeroTouch { + s.ZeroTouch++ + } + case OutcomeStatusRejected: + s.Rejected++ + case OutcomeStatusIgnored: + s.Ignored++ + case OutcomeStatusPending: + s.Pending++ } +} + +func computeOutcomeSummaryRates(s *OutcomeSummary, times []float64) { resolved := s.Accepted + s.Rejected if resolved > 0 { s.AcceptanceRate = float64(s.Accepted) / float64(resolved) @@ -220,11 +242,6 @@ func ComputeOutcomeSummary(reports []OutcomeReport, mapping *github.ObjectiveMap if s.TotalObjectiveValue > 0 { s.ObjectiveEfficiency = float64(s.AcceptedObjectiveValue) / float64(s.TotalObjectiveValue) } - - // Compute domain breakdowns - s.DomainBreakdowns = ComputeDomainBreakdowns(reports) - - return s } // escapeOwnerRepo URL-path-encodes each component of an "owner/repo" string to @@ -501,7 +518,28 @@ func loadPullRequestIntentData(report OutcomeReport, repo string) (intent.PullRe return intent.PullRequestData{}, fmt.Errorf("invalid repo for root tracing: %s", repo) } - query := fmt.Sprintf(`query { + query := loadPullRequestIntentDataQuery(owner, name, prNumber) + result, err := objectiveMappingGHAPIGraphQL(query, repo) + if err != nil { + return intent.PullRequestData{}, err + } + + pullRequest := loadPullRequestIntentDataPullRequest(result) + prData := intent.PullRequestData{URL: report.ObjectURL} + if nodeID, ok := pullRequest["id"].(string); ok { + prData.NodeID = nodeID + } + + nodes := loadPullRequestIntentDataClosingIssueNodes(pullRequest) + if len(nodes) == 0 { + return loadPullRequestIntentDataWithFallbackLabels(report, repo, prData) + } + prData.ClosingIssues = loadPullRequestIntentDataClosingIssues(nodes) + return prData, nil +} + +func loadPullRequestIntentDataQuery(owner string, name string, prNumber int) string { + return fmt.Sprintf(`query { repository(owner: "%s", name: "%s") { pullRequest(number: %d) { id @@ -522,30 +560,32 @@ func loadPullRequestIntentData(report OutcomeReport, repo string) (intent.PullRe escapeGraphQLString(name), prNumber, ) +} - result, err := objectiveMappingGHAPIGraphQL(query, repo) - if err != nil { - return intent.PullRequestData{}, err - } +func loadPullRequestIntentDataPullRequest(result map[string]any) map[string]any { data, _ := result["data"].(map[string]any) repository, _ := data["repository"].(map[string]any) pullRequest, _ := repository["pullRequest"].(map[string]any) - prData := intent.PullRequestData{URL: report.ObjectURL} - if nodeID, ok := pullRequest["id"].(string); ok { - prData.NodeID = nodeID - } + return pullRequest +} + +func loadPullRequestIntentDataClosingIssueNodes(pullRequest map[string]any) []any { closingRefs, _ := pullRequest["closingIssuesReferences"].(map[string]any) nodes, _ := closingRefs["nodes"].([]any) - if len(nodes) == 0 { - labels, labelErr := objectiveMappingGHAPIGetArray(fmt.Sprintf("issues/%d/labels", report.ObjectNumber), repo) - if labelErr != nil { - return intent.PullRequestData{}, labelErr - } - prData.Labels = labelsToStringsFromMaps(labels) - return prData, nil + return nodes +} + +func loadPullRequestIntentDataWithFallbackLabels(report OutcomeReport, repo string, prData intent.PullRequestData) (intent.PullRequestData, error) { + labels, labelErr := objectiveMappingGHAPIGetArray(fmt.Sprintf("issues/%d/labels", report.ObjectNumber), repo) + if labelErr != nil { + return intent.PullRequestData{}, labelErr } + prData.Labels = labelsToStringsFromMaps(labels) + return prData, nil +} - prData.ClosingIssues = make([]intent.RootReference, 0, len(nodes)) +func loadPullRequestIntentDataClosingIssues(nodes []any) []intent.RootReference { + closingIssues := make([]intent.RootReference, 0, len(nodes)) for _, node := range nodes { rootNode, _ := node.(map[string]any) root := intent.RootReference{Type: "issue"} @@ -560,10 +600,9 @@ func loadPullRequestIntentData(report OutcomeReport, repo string) (intent.PullRe root.Labels = labelsToStringsFromNodes(labelNodes) } } - prData.ClosingIssues = append(prData.ClosingIssues, root) + closingIssues = append(closingIssues, root) } - - return prData, nil + return closingIssues } func labelsToStringsFromNodes(nodes []any) []string { diff --git a/pkg/cli/outcome_eval_agent.go b/pkg/cli/outcome_eval_agent.go index 041f6ac91f6..7f466c5e48a 100644 --- a/pkg/cli/outcome_eval_agent.go +++ b/pkg/cli/outcome_eval_agent.go @@ -40,75 +40,88 @@ func evalAssignToAgent(item CreatedItemReport, repoOverride string) OutcomeRepor // Search for linked PRs from copilot-swe-agent via timeline events events, err := ghAPIGetArray(fmt.Sprintf("issues/%d/timeline", num), repo) - var agentPR map[string]any if err == nil { - for _, event := range events { - eventType, _ := event["event"].(string) - if eventType != "cross-referenced" { - continue - } - source, _ := event["source"].(map[string]any) - if source == nil { - continue - } - issue, _ := source["issue"].(map[string]any) - if issue == nil { - continue - } - // Check if it's a PR (has pull_request field) - if _, hasPR := issue["pull_request"]; !hasPR { - continue - } - user, _ := issue["user"].(map[string]any) - login, _ := user["login"].(string) - if strings.Contains(login, "copilot") || strings.Contains(login, "github-actions") { - agentPR = issue - break - } + agentPR := evalAssignToAgentFindLinkedPR(events) + if prReport, ok := evalAssignToAgentEvaluateLinkedPR(agentPR, repo, item, report); ok { + return prReport } } - if agentPR != nil { - prNumber := 0 - if n, ok := agentPR["number"].(float64); ok { - prNumber = int(n) + return evalAssignToAgentFallbackIssue(issueData, item, report, state, stateReason) +} + +func evalAssignToAgentFindLinkedPR(events []map[string]any) map[string]any { + for _, event := range events { + eventType, _ := event["event"].(string) + if eventType != "cross-referenced" { + continue + } + source, _ := event["source"].(map[string]any) + if source == nil { + continue + } + issue, _ := source["issue"].(map[string]any) + if issue == nil { + continue + } + // Check if it's a PR (has pull_request field) + if _, hasPR := issue["pull_request"]; !hasPR { + continue + } + user, _ := issue["user"].(map[string]any) + login, _ := user["login"].(string) + if strings.Contains(login, "copilot") || strings.Contains(login, "github-actions") { + return issue } - outcomeEvalAgentLog.Printf("Found agent-linked PR for issue #%d: pr=%d", num, prNumber) + } + return nil +} - // Fetch the actual PR to check merge status - if prNumber > 0 { - prData, perr := ghAPIGet(fmt.Sprintf("pulls/%d", prNumber), repo) - if perr == nil { - merged, _ := prData["merged"].(bool) - prState, _ := prData["state"].(string) - mergedAt, _ := prData["merged_at"].(string) - closedAt, _ := prData["closed_at"].(string) +func evalAssignToAgentEvaluateLinkedPR(agentPR map[string]any, repo string, item CreatedItemReport, report OutcomeReport) (OutcomeReport, bool) { + if agentPR == nil { + return report, false + } + prNumber := 0 + if n, ok := agentPR["number"].(float64); ok { + prNumber = int(n) + } + outcomeEvalAgentLog.Printf("Found agent-linked PR for issue #%d: pr=%d", report.ObjectNumber, prNumber) + if prNumber == 0 { + return report, false + } + prData, perr := ghAPIGet(fmt.Sprintf("pulls/%d", prNumber), repo) + if perr != nil { + return report, false + } + return evalAssignToAgentReportForPR(prData, prNumber, item, report), true +} - switch { - case merged: - report.Result = OutcomeAccepted - report.Detail = fmt.Sprintf("agent PR #%d merged", prNumber) - if mergedAt != "" && item.Timestamp != "" { - report.TimeToOutcomeHours = timeBetween(item.Timestamp, mergedAt) - } - return report - case prState == "closed": - report.Result = OutcomeRejected - report.Detail = fmt.Sprintf("agent PR #%d closed without merge", prNumber) - if closedAt != "" && item.Timestamp != "" { - report.TimeToOutcomeHours = timeBetween(item.Timestamp, closedAt) - } - return report - default: - report.Result = OutcomePending - report.Detail = fmt.Sprintf("agent PR #%d open", prNumber) - return report - } - } +func evalAssignToAgentReportForPR(prData map[string]any, prNumber int, item CreatedItemReport, report OutcomeReport) OutcomeReport { + merged, _ := prData["merged"].(bool) + prState, _ := prData["state"].(string) + mergedAt, _ := prData["merged_at"].(string) + closedAt, _ := prData["closed_at"].(string) + switch { + case merged: + report.Result = OutcomeAccepted + report.Detail = fmt.Sprintf("agent PR #%d merged", prNumber) + if mergedAt != "" && item.Timestamp != "" { + report.TimeToOutcomeHours = timeBetween(item.Timestamp, mergedAt) } + case prState == "closed": + report.Result = OutcomeRejected + report.Detail = fmt.Sprintf("agent PR #%d closed without merge", prNumber) + if closedAt != "" && item.Timestamp != "" { + report.TimeToOutcomeHours = timeBetween(item.Timestamp, closedAt) + } + default: + report.Result = OutcomePending + report.Detail = fmt.Sprintf("agent PR #%d open", prNumber) } + return report +} - // No agent PR found — check if issue was resolved by other means +func evalAssignToAgentFallbackIssue(issueData map[string]any, item CreatedItemReport, report OutcomeReport, state string, stateReason string) OutcomeReport { switch { case state == "closed" && stateReason == "completed": report.Result = OutcomeAccepted diff --git a/pkg/cli/outcome_eval_comment.go b/pkg/cli/outcome_eval_comment.go index 9c2997f970d..0553be964de 100644 --- a/pkg/cli/outcome_eval_comment.go +++ b/pkg/cli/outcome_eval_comment.go @@ -44,13 +44,7 @@ func evalAddComment(item CreatedItemReport, repoOverride string) OutcomeReport { } // Check reactions - reactions, _ := data["reactions"].(map[string]any) - totalReactions := 0 - if reactions != nil { - if tc, ok := reactions["total_count"].(float64); ok { - totalReactions = int(tc) - } - } + totalReactions := evalAddCommentTotalReactions(data) // Check if the comment is minimized (hidden) // The REST API field is "performed_via_github_app" but minimized state @@ -58,27 +52,47 @@ func evalAddComment(item CreatedItemReport, repoOverride string) OutcomeReport { // or the node_id can be checked via GraphQL. For now, use reactions+replies. // To check replies, we need the issue number and look for comments posted after this one - issueNumber := parseNumberFromURL(item.URL) + replyCount := evalAddCommentReplyCount(item.URL, repo, data) + report.HumanComments = replyCount + return evalAddCommentClassify(report, totalReactions, replyCount) +} + +func evalAddCommentTotalReactions(data map[string]any) int { + reactions, _ := data["reactions"].(map[string]any) + if reactions == nil { + return 0 + } + if tc, ok := reactions["total_count"].(float64); ok { + return int(tc) + } + return 0 +} + +func evalAddCommentReplyCount(itemURL string, repo string, data map[string]any) int { + issueNumber := parseNumberFromURL(itemURL) + if issueNumber == 0 { + return 0 + } + commentList, cerr := ghAPIGetArray(fmt.Sprintf("issues/%d/comments", issueNumber), repo) + if cerr != nil { + return 0 + } + createdAt, _ := data["created_at"].(string) replyCount := 0 - if issueNumber > 0 { - commentList, cerr := ghAPIGetArray(fmt.Sprintf("issues/%d/comments", issueNumber), repo) - if cerr == nil { - createdAt, _ := data["created_at"].(string) - for _, c := range commentList { - cCreatedAt, _ := c["created_at"].(string) - if cCreatedAt > createdAt { - user, _ := c["user"].(map[string]any) - login, _ := user["login"].(string) - if !isBotUser(login) { - replyCount++ - } - } + for _, c := range commentList { + cCreatedAt, _ := c["created_at"].(string) + if cCreatedAt > createdAt { + user, _ := c["user"].(map[string]any) + login, _ := user["login"].(string) + if !isBotUser(login) { + replyCount++ } } } + return replyCount +} - report.HumanComments = replyCount - +func evalAddCommentClassify(report OutcomeReport, totalReactions int, replyCount int) OutcomeReport { switch { case totalReactions > 0 || replyCount > 0: report.Result = OutcomeAccepted diff --git a/pkg/cli/outcome_eval_enrich.go b/pkg/cli/outcome_eval_enrich.go index fa744e6c1c8..b4db1d7f08c 100644 --- a/pkg/cli/outcome_eval_enrich.go +++ b/pkg/cli/outcome_eval_enrich.go @@ -12,6 +12,15 @@ import ( var outcomeEvalEnrichLog = logger.New("cli:outcome_eval_enrich") +type enrichItemsFromAgentOutputRawEntry struct { + Type string `json:"type"` + IssueNumber int `json:"issue_number"` + ItemNumber int `json:"item_number"` + PullNumber int `json:"pull_number"` + Branch string `json:"branch"` + Title string `json:"title"` +} + // enrichItemsFromAgentOutput reads the raw agent output (safeoutputs.jsonl) and fills in // missing fields like issue_number and item_number that the executed manifest may omit. func enrichItemsFromAgentOutput(items []CreatedItemReport, runDir string, defaultRepo string) []CreatedItemReport { @@ -25,30 +34,29 @@ func enrichItemsFromAgentOutput(items []CreatedItemReport, runDir string, defaul defer f.Close() // Parse raw entries to extract issue/item numbers by type and order - type rawEntry struct { - Type string `json:"type"` - IssueNumber int `json:"issue_number"` - ItemNumber int `json:"item_number"` - PullNumber int `json:"pull_number"` - Branch string `json:"branch"` - Title string `json:"title"` - } + rawByType := enrichItemsFromAgentOutputReadRaw(f) + outcomeEvalEnrichLog.Printf("Parsed safeoutputs.jsonl: %d distinct types", len(rawByType)) + return enrichItemsFromAgentOutputApply(items, rawByType, defaultRepo) +} - var rawByType = make(map[string][]rawEntry) +func enrichItemsFromAgentOutputReadRaw(f *os.File) map[string][]enrichItemsFromAgentOutputRawEntry { + var rawByType = make(map[string][]enrichItemsFromAgentOutputRawEntry) scanner := bufio.NewScanner(f) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" { continue } - var entry rawEntry + var entry enrichItemsFromAgentOutputRawEntry if err := json.Unmarshal([]byte(line), &entry); err != nil { continue } rawByType[entry.Type] = append(rawByType[entry.Type], entry) } - outcomeEvalEnrichLog.Printf("Parsed safeoutputs.jsonl: %d distinct types", len(rawByType)) + return rawByType +} +func enrichItemsFromAgentOutputApply(items []CreatedItemReport, rawByType map[string][]enrichItemsFromAgentOutputRawEntry, defaultRepo string) []CreatedItemReport { // Match items to raw entries by type and order to fill in missing numbers typeCounters := make(map[string]int) for i := range items { diff --git a/pkg/cli/outcome_eval_issue.go b/pkg/cli/outcome_eval_issue.go index f96f20c6467..ff27ac31c1c 100644 --- a/pkg/cli/outcome_eval_issue.go +++ b/pkg/cli/outcome_eval_issue.go @@ -41,17 +41,27 @@ func evalCreateIssue(item CreatedItemReport, repoOverride string) OutcomeReport // Count human comments comments, _ := data["comments"].(float64) + report.HumanComments = evalCreateIssueHumanComments(num, repo) + return evalCreateIssueClassify(report, item, repo, num, state, stateReason, closedAt, comments) +} + +func evalCreateIssueHumanComments(num int, repo string) int { commentList, cerr := ghAPIGetArray(fmt.Sprintf("issues/%d/comments", num), repo) - if cerr == nil { - for _, c := range commentList { - user, _ := c["user"].(map[string]any) - login, _ := user["login"].(string) - if !isBotUser(login) { - report.HumanComments++ - } + if cerr != nil { + return 0 + } + count := 0 + for _, c := range commentList { + user, _ := c["user"].(map[string]any) + login, _ := user["login"].(string) + if !isBotUser(login) { + count++ } } + return count +} +func evalCreateIssueClassify(report OutcomeReport, item CreatedItemReport, repo string, num int, state string, stateReason string, closedAt string, comments float64) OutcomeReport { switch { case state == "closed" && stateReason == "completed": report.Result = OutcomeAccepted diff --git a/pkg/cli/outcome_eval_label.go b/pkg/cli/outcome_eval_label.go index 0dd4fe65348..b00fa21ed1a 100644 --- a/pkg/cli/outcome_eval_label.go +++ b/pkg/cli/outcome_eval_label.go @@ -24,14 +24,7 @@ func evalReplaceLabel(item CreatedItemReport, repoOverride string) OutcomeReport Repo: repo, } if num == 0 || repo == "" || item.BeforeState == nil || item.AfterState == nil { - report.Result = OutcomeUnknown - report.Detail = "missing execution state" - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusUnknown, - EvidenceStrength: EvidenceNone, - Signal: "missing_execution_state", - } - return report + return evalReplaceLabelUnknown(report, "missing execution state", "missing_execution_state") } beforeLabels := mutableStringSlice(item.BeforeState["labels"]) @@ -42,14 +35,7 @@ func evalReplaceLabel(item CreatedItemReport, repoOverride string) OutcomeReport removed := labelSetDiff(beforeLabels, afterLabels) if len(added) == 0 && len(removed) == 0 { - report.Result = OutcomeUnknown - report.Detail = "no label delta" - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusUnknown, - EvidenceStrength: EvidenceNone, - Signal: "no_state_delta", - } - return report + return evalReplaceLabelUnknown(report, "no label delta", "no_state_delta") } currentState, _, err := extractCurrentIssueUpdateState(repo, num) @@ -64,7 +50,21 @@ func evalReplaceLabel(item CreatedItemReport, repoOverride string) OutcomeReport // all removed labels are still absent, regardless of any other label changes. addedRetained := labelSetContainsAll(currentLabels, added) removedStillAbsent := !labelSetContainsAny(currentLabels, removed) + return evalReplaceLabelClassify(report, currentLabels, added, removed, addedRetained, removedStillAbsent) +} + +func evalReplaceLabelUnknown(report OutcomeReport, detail string, signal string) OutcomeReport { + report.Result = OutcomeUnknown + report.Detail = detail + report.OutcomeEvaluation = OutcomeEvaluation{ + OutcomeStatus: OutcomeStatusUnknown, + EvidenceStrength: EvidenceNone, + Signal: signal, + } + return report +} +func evalReplaceLabelClassify(report OutcomeReport, currentLabels []string, added []string, removed []string, addedRetained bool, removedStillAbsent bool) OutcomeReport { if addedRetained && removedStillAbsent { report.Result = OutcomeAccepted report.Detail = "label replacement retained" diff --git a/pkg/cli/outcome_eval_pr.go b/pkg/cli/outcome_eval_pr.go index 63102c7bcc9..219d7ebd14e 100644 --- a/pkg/cli/outcome_eval_pr.go +++ b/pkg/cli/outcome_eval_pr.go @@ -58,14 +58,7 @@ func evalCreatePullRequest(item CreatedItemReport, repoOverride string) OutcomeR } // If no PR number, try to find the PR by searching recent PRs from github-actions - if num == 0 && repo != "" { - found := findPRByTimestamp(repo, item.Timestamp) - if found > 0 { - outcomeEvalPRLog.Printf("Resolved missing PR number via timestamp search: num=%d", found) - num = found - report.ObjectNumber = num - } - } + num = evalCreatePullRequestResolveNumber(num, repo, item.Timestamp, &report) if num == 0 || repo == "" { report.Result = OutcomeError @@ -85,6 +78,38 @@ func evalCreatePullRequest(item CreatedItemReport, repoOverride string) OutcomeR mergedAt, _ := data["merged_at"].(string) closedAt, _ := data["closed_at"].(string) + evalCreatePullRequestClassify(&report, item, merged, state, mergedAt, closedAt) + + // Count human comments (non-bot) + report.HumanComments = evalCreatePullRequestHumanComments(num, repo) + + // Count reviews (used for ZeroTouch, stored separately from edits to avoid conflation) + reviews, err := ghAPIGetArray(fmt.Sprintf("pulls/%d/reviews", num), repo) + if err == nil { + report.HumanReviews = len(reviews) + } + + if report.Result == OutcomeAccepted { + report.ZeroTouch = report.HumanComments == 0 && report.HumanReviews == 0 + } + + return report +} + +func evalCreatePullRequestResolveNumber(num int, repo string, timestamp string, report *OutcomeReport) int { + if num != 0 || repo == "" { + return num + } + found := findPRByTimestamp(repo, timestamp) + if found > 0 { + outcomeEvalPRLog.Printf("Resolved missing PR number via timestamp search: num=%d", found) + report.ObjectNumber = found + return found + } + return num +} + +func evalCreatePullRequestClassify(report *OutcomeReport, item CreatedItemReport, merged bool, state string, mergedAt string, closedAt string) { switch { case merged: report.Result = OutcomeAccepted @@ -102,28 +127,20 @@ func evalCreatePullRequest(item CreatedItemReport, repoOverride string) OutcomeR report.Result = OutcomePending report.Detail = "open" } +} - // Count human comments (non-bot) +func evalCreatePullRequestHumanComments(num int, repo string) int { comments, err := ghAPIGetArray(fmt.Sprintf("issues/%d/comments", num), repo) - if err == nil { - for _, c := range comments { - user, _ := c["user"].(map[string]any) - login, _ := user["login"].(string) - if !isBotUser(login) { - report.HumanComments++ - } - } - } - - // Count reviews (used for ZeroTouch, stored separately from edits to avoid conflation) - reviews, err := ghAPIGetArray(fmt.Sprintf("pulls/%d/reviews", num), repo) - if err == nil { - report.HumanReviews = len(reviews) + if err != nil { + return 0 } - - if report.Result == OutcomeAccepted { - report.ZeroTouch = report.HumanComments == 0 && report.HumanReviews == 0 + count := 0 + for _, c := range comments { + user, _ := c["user"].(map[string]any) + login, _ := user["login"].(string) + if !isBotUser(login) { + count++ + } } - - return report + return count } diff --git a/pkg/cli/outcome_eval_review.go b/pkg/cli/outcome_eval_review.go index 05330d2bf5b..4e3b748f42d 100644 --- a/pkg/cli/outcome_eval_review.go +++ b/pkg/cli/outcome_eval_review.go @@ -48,6 +48,28 @@ func evalAddReviewer(item CreatedItemReport, repoOverride string) OutcomeReport } outcomeReviewLog.Printf("Fetched %d reviews for PR #%d (requested reviewers=%d, teams=%d)", len(reviews), num, len(requestedReviewers), len(requestedTeams)) + latestByReviewer := evalAddReviewerLatestByReviewer(requestedReviewers, reviews, item.Timestamp) + if reviewerReport, ok := evalAddReviewerFromRequestedReviews(report, latestByReviewer); ok { + return reviewerReport + } + + // We cannot cheaply verify team membership for each reviewer from this endpoint, + // so any submitted post-request review counts as medium-evidence team activity. + if len(requestedTeams) > 0 && hasReviewAfterTimestamp(reviews, item.Timestamp) { + return evalAddReviewerTeamReviewSubmitted(report) + } + + currentUsers := extractLogins(requested["users"]) + currentTeams := extractTeamSlugs(requested["teams"]) + stillPending := intersectsFold(requestedReviewers, currentUsers) || intersectsFold(requestedTeams, currentTeams) + if stillPending { + return evalAddReviewerStillPending(report) + } + + return evalAddReviewerNoSubmittedReview(report, requestedReviewers, requestedTeams) +} + +func evalAddReviewerLatestByReviewer(requestedReviewers []string, reviews []map[string]any, timestamp string) map[string]map[string]any { requestedReviewerSet := make(map[string]struct{}, len(requestedReviewers)) for _, reviewer := range requestedReviewers { requestedReviewerSet[strings.ToLower(reviewer)] = struct{}{} @@ -64,7 +86,7 @@ func evalAddReviewer(item CreatedItemReport, repoOverride string) OutcomeReport if state == "" || state == "PENDING" || submittedAt == "" { //nolint:tolowerequalfold continue } - if !timestampOnOrAfter(submittedAt, item.Timestamp) { + if !timestampOnOrAfter(submittedAt, timestamp) { continue } prev, ok := latestByReviewer[login] @@ -72,7 +94,10 @@ func evalAddReviewer(item CreatedItemReport, repoOverride string) OutcomeReport latestByReviewer[login] = review } } + return latestByReviewer +} +func evalAddReviewerFromRequestedReviews(report OutcomeReport, latestByReviewer map[string]map[string]any) (OutcomeReport, bool) { var approvedReviewer string var submittedReviewer string for login, review := range latestByReviewer { @@ -84,7 +109,6 @@ func evalAddReviewer(item CreatedItemReport, repoOverride string) OutcomeReport submittedReviewer = login } } - switch { case approvedReviewer != "": report.Result = OutcomeAccepted @@ -94,7 +118,7 @@ func evalAddReviewer(item CreatedItemReport, repoOverride string) OutcomeReport EvidenceStrength: EvidenceStrong, Signal: "review_approved", } - return report + return report, true case submittedReviewer != "": report.Result = OutcomeAccepted report.Detail = fmt.Sprintf("requested reviewer %s submitted a review", submittedReviewer) @@ -103,36 +127,34 @@ func evalAddReviewer(item CreatedItemReport, repoOverride string) OutcomeReport EvidenceStrength: EvidenceMedium, Signal: "review_submitted", } - return report + return report, true } + return report, false +} - // We cannot cheaply verify team membership for each reviewer from this endpoint, - // so any submitted post-request review counts as medium-evidence team activity. - if len(requestedTeams) > 0 && hasReviewAfterTimestamp(reviews, item.Timestamp) { - report.Result = OutcomeAccepted - report.Detail = "team review request received a review" - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusAccepted, - EvidenceStrength: EvidenceMedium, - Signal: "review_submitted", - } - return report +func evalAddReviewerTeamReviewSubmitted(report OutcomeReport) OutcomeReport { + report.Result = OutcomeAccepted + report.Detail = "team review request received a review" + report.OutcomeEvaluation = OutcomeEvaluation{ + OutcomeStatus: OutcomeStatusAccepted, + EvidenceStrength: EvidenceMedium, + Signal: "review_submitted", } + return report +} - currentUsers := extractLogins(requested["users"]) - currentTeams := extractTeamSlugs(requested["teams"]) - stillPending := intersectsFold(requestedReviewers, currentUsers) || intersectsFold(requestedTeams, currentTeams) - if stillPending { - report.Result = OutcomePending - report.Detail = "review request still pending" - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusPending, - EvidenceStrength: EvidenceMedium, - Signal: "awaiting_review", - } - return report +func evalAddReviewerStillPending(report OutcomeReport) OutcomeReport { + report.Result = OutcomePending + report.Detail = "review request still pending" + report.OutcomeEvaluation = OutcomeEvaluation{ + OutcomeStatus: OutcomeStatusPending, + EvidenceStrength: EvidenceMedium, + Signal: "awaiting_review", } + return report +} +func evalAddReviewerNoSubmittedReview(report OutcomeReport, requestedReviewers []string, requestedTeams []string) OutcomeReport { if len(requestedReviewers) > 0 || len(requestedTeams) > 0 { report.Result = OutcomeRejected report.Detail = "review request removed without submitted review" @@ -204,68 +226,90 @@ func evalSubmitPullRequestReview(item CreatedItemReport, repoOverride string) Ou reviewSubmittedAt := outcomeString(review["submitted_at"]) prMerged, _ := pr["merged"].(bool) prState, _ := pr["state"].(string) + return evalSubmitPullRequestReviewClassify(evalSubmitPullRequestReviewClassifyParams{Report: report, Item: item, Repo: repo, Num: num, PR: pr, Reviews: reviews, Review: review, ReviewState: reviewState, ReviewSubmittedAt: reviewSubmittedAt, PRMerged: prMerged, PRState: prState}) +} +type evalSubmitPullRequestReviewClassifyParams struct { + Report OutcomeReport + Item CreatedItemReport + Repo string + Num int + PR map[string]any + Reviews []map[string]any + Review map[string]any + ReviewState string + ReviewSubmittedAt string + PRMerged bool + PRState string +} + +func evalSubmitPullRequestReviewClassify(p evalSubmitPullRequestReviewClassifyParams) OutcomeReport { switch { - case reviewState == "DISMISSED": //nolint:tolowerequalfold - report.Result = OutcomeRejected - report.Detail = "review dismissed by repo admin" - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusRejected, - EvidenceStrength: EvidenceStrong, - Signal: "review_dismissed", - } - return report - case prMerged && reviewState == "APPROVED": //nolint:tolowerequalfold - report.Result = OutcomeAccepted - report.Detail = "approved review followed by merge" - report.TimeToOutcomeHours = timeBetween(item.Timestamp, outcomeString(pr["merged_at"])) - report.OutcomeEvaluation = OutcomeEvaluation{ + case p.ReviewState == "DISMISSED": //nolint:tolowerequalfold + return evalSubmitPullRequestReviewRejected(p.Report, "review dismissed by repo admin", "review_dismissed", EvidenceStrong) + case p.PRMerged && p.ReviewState == "APPROVED": //nolint:tolowerequalfold + p.Report.Result = OutcomeAccepted + p.Report.Detail = "approved review followed by merge" + p.Report.TimeToOutcomeHours = timeBetween(p.Item.Timestamp, outcomeString(p.PR["merged_at"])) + p.Report.OutcomeEvaluation = OutcomeEvaluation{ OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceStrong, Signal: "review_approved", } - return report - case prMerged && reviewState == "CHANGES_REQUESTED": //nolint:tolowerequalfold - commits, err := outcomeReviewGHAPIGetArray(fmt.Sprintf("pulls/%d/commits", num), repo) - if err == nil && hasCommitAfterTimestamp(commits, reviewSubmittedAt) { - report.Result = OutcomeAccepted - report.Detail = "changes requested, updated, and merged" - report.TimeToOutcomeHours = timeBetween(item.Timestamp, outcomeString(pr["merged_at"])) - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusAccepted, - EvidenceStrength: EvidenceMedium, - Signal: "changes_requested_addressed", - } - return report + return p.Report + case p.PRMerged && p.ReviewState == "CHANGES_REQUESTED": //nolint:tolowerequalfold + if updatedReport, ok := evalSubmitPullRequestReviewChangesAddressed(p.Report, p.Item, p.Repo, p.Num, p.PR, p.ReviewSubmittedAt); ok { + return updatedReport } - case prState == "closed" && !prMerged: - report.Result = OutcomeRejected - report.Detail = "PR closed without merge after review submission" - report.TimeToOutcomeHours = timeBetween(item.Timestamp, outcomeString(pr["closed_at"])) - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusRejected, - EvidenceStrength: EvidenceMedium, - Signal: "closed_without_merge_after_review", - } - return report - case prState == "open" && isLatestReview(reviews, review): - report.Result = OutcomePending - report.Detail = "review is latest review on open PR" - report.OutcomeEvaluation = OutcomeEvaluation{ + case p.PRState == "closed" && !p.PRMerged: + p.Report = evalSubmitPullRequestReviewRejected(p.Report, "PR closed without merge after review submission", "closed_without_merge_after_review", EvidenceMedium) + p.Report.TimeToOutcomeHours = timeBetween(p.Item.Timestamp, outcomeString(p.PR["closed_at"])) + return p.Report + case p.PRState == "open" && isLatestReview(p.Reviews, p.Review): + p.Report.Result = OutcomePending + p.Report.Detail = "review is latest review on open PR" + p.Report.OutcomeEvaluation = OutcomeEvaluation{ OutcomeStatus: OutcomeStatusPending, EvidenceStrength: EvidenceMedium, Signal: "latest_review_pending", } - return report + return p.Report } - report.Result = OutcomeUnknown - report.Detail = "review outcome could not be determined" - report.OutcomeEvaluation = OutcomeEvaluation{ + p.Report.Result = OutcomeUnknown + p.Report.Detail = "review outcome could not be determined" + p.Report.OutcomeEvaluation = OutcomeEvaluation{ OutcomeStatus: OutcomeStatusUnknown, EvidenceStrength: EvidenceWeak, Signal: "unknown", } + return p.Report +} + +func evalSubmitPullRequestReviewChangesAddressed(report OutcomeReport, item CreatedItemReport, repo string, num int, pr map[string]any, reviewSubmittedAt string) (OutcomeReport, bool) { + commits, err := outcomeReviewGHAPIGetArray(fmt.Sprintf("pulls/%d/commits", num), repo) + if err == nil && hasCommitAfterTimestamp(commits, reviewSubmittedAt) { + report.Result = OutcomeAccepted + report.Detail = "changes requested, updated, and merged" + report.TimeToOutcomeHours = timeBetween(item.Timestamp, outcomeString(pr["merged_at"])) + report.OutcomeEvaluation = OutcomeEvaluation{ + OutcomeStatus: OutcomeStatusAccepted, + EvidenceStrength: EvidenceMedium, + Signal: "changes_requested_addressed", + } + return report, true + } + return report, false +} + +func evalSubmitPullRequestReviewRejected(report OutcomeReport, detail string, signal string, strength EvidenceStrength) OutcomeReport { + report.Result = OutcomeRejected + report.Detail = detail + report.OutcomeEvaluation = OutcomeEvaluation{ + OutcomeStatus: OutcomeStatusRejected, + EvidenceStrength: strength, + Signal: signal, + } return report } diff --git a/pkg/cli/outcome_eval_update.go b/pkg/cli/outcome_eval_update.go index 4a888e06d7b..5ee609fbef1 100644 --- a/pkg/cli/outcome_eval_update.go +++ b/pkg/cli/outcome_eval_update.go @@ -36,24 +36,10 @@ func evalRetainedUpdate(item CreatedItemReport, repoOverride string, objectKind } if num == 0 || repo == "" { outcomeEvalUpdateLog.Printf("Missing execution state: num=%d, repo=%s", num, repo) - report.Result = OutcomeUnknown - report.Detail = "missing execution state" - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusUnknown, - EvidenceStrength: EvidenceNone, - Signal: "missing_execution_state", - } - return report + return evalRetainedUpdateUnknown(report, "missing execution state", "missing_execution_state") } if item.BeforeState == nil || item.AfterState == nil { - report.Result = OutcomeUnknown - report.Detail = "missing execution state" - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusUnknown, - EvidenceStrength: EvidenceNone, - Signal: "missing_execution_state", - } - return report + return evalRetainedUpdateUnknown(report, "missing execution state", "missing_execution_state") } currentState, merged, err := load(repo, num) @@ -67,16 +53,23 @@ func evalRetainedUpdate(item CreatedItemReport, repoOverride string, objectKind outcomeEvalUpdateLog.Printf("State comparison for %s #%d: changed=%d, retained=%d, reverted=%d, replaced=%d, merged=%v", objectKind, num, len(comparison.Changed), len(comparison.Retained), len(comparison.Reverted), len(comparison.Replaced), merged) if len(comparison.Changed) == 0 { - report.Result = OutcomeUnknown - report.Detail = "no persisted state delta" - report.OutcomeEvaluation = OutcomeEvaluation{ - OutcomeStatus: OutcomeStatusUnknown, - EvidenceStrength: EvidenceNone, - Signal: "no_state_delta", - } - return report + return evalRetainedUpdateUnknown(report, "no persisted state delta", "no_state_delta") + } + return evalRetainedUpdateClassify(report, comparison, objectKind, strongOnMerge, merged) +} + +func evalRetainedUpdateUnknown(report OutcomeReport, detail string, signal string) OutcomeReport { + report.Result = OutcomeUnknown + report.Detail = detail + report.OutcomeEvaluation = OutcomeEvaluation{ + OutcomeStatus: OutcomeStatusUnknown, + EvidenceStrength: EvidenceNone, + Signal: signal, } + return report +} +func evalRetainedUpdateClassify(report OutcomeReport, comparison retainedStateComparison, objectKind string, strongOnMerge bool, merged bool) OutcomeReport { switch { case len(comparison.Retained) == len(comparison.Changed): report.Result = OutcomeAccepted diff --git a/pkg/cli/outcome_eval_workflow.go b/pkg/cli/outcome_eval_workflow.go index 447f3db26e9..48f5de9d9d9 100644 --- a/pkg/cli/outcome_eval_workflow.go +++ b/pkg/cli/outcome_eval_workflow.go @@ -31,24 +31,7 @@ func evalDispatchWorkflow(item CreatedItemReport, repoOverride string) OutcomeRe // Extract run_id from metadata if available. // JSON numbers unmarshal as float64; convert carefully to int64 to avoid // precision loss for large GitHub run IDs (which can exceed 2^32). - var runID int64 - if item.Metadata != nil { - if v, ok := item.Metadata["run_id"]; ok { - switch id := v.(type) { - case float64: - // Guard against float64 values that cannot round-trip to int64. - // math.MaxInt64 is not representable exactly as float64; use 2^53 - // (maxSafeFloat64Int) as the safe upper bound for lossless conversion. - if id > 0 && id <= maxSafeFloat64Int && id == math.Trunc(id) { - runID = int64(id) - } - case int: - runID = int64(id) - case int64: - runID = id - } - } - } + runID := evalDispatchWorkflowRunID(item.Metadata) if runID <= 0 { // No run ID available — workflow may not have been dispatched or ID not captured @@ -67,7 +50,33 @@ func evalDispatchWorkflow(item CreatedItemReport, repoOverride string) OutcomeRe status, _ := data["status"].(string) conclusion, _ := data["conclusion"].(string) outcomeEvalWorkflowLog.Printf("dispatch_workflow run %d: status=%s, conclusion=%s", runID, status, conclusion) + return evalDispatchWorkflowClassify(report, status, conclusion) +} + +func evalDispatchWorkflowRunID(metadata map[string]any) int64 { + var runID int64 + if metadata == nil { + return runID + } + if v, ok := metadata["run_id"]; ok { + switch id := v.(type) { + case float64: + // Guard against float64 values that cannot round-trip to int64. + // math.MaxInt64 is not representable exactly as float64; use 2^53 + // (maxSafeFloat64Int) as the safe upper bound for lossless conversion. + if id > 0 && id <= maxSafeFloat64Int && id == math.Trunc(id) { + runID = int64(id) + } + case int: + runID = int64(id) + case int64: + runID = id + } + } + return runID +} +func evalDispatchWorkflowClassify(report OutcomeReport, status string, conclusion string) OutcomeReport { switch { case status == "completed" && conclusion == "success": report.Result = OutcomeAccepted diff --git a/pkg/cli/outcome_evaluation.go b/pkg/cli/outcome_evaluation.go index 4d64de5e61b..2936abaceca 100644 --- a/pkg/cli/outcome_evaluation.go +++ b/pkg/cli/outcome_evaluation.go @@ -55,48 +55,57 @@ func normalizeOutcomeEvaluation(report OutcomeReport) OutcomeEvaluation { } detail := strings.ToLower(strings.TrimSpace(report.Detail)) + if eval, ok := normalizeOutcomeEvaluationFromDetail(report, detail); ok { + return eval + } + return normalizeOutcomeEvaluationFromResult(report.Result) +} +func normalizeOutcomeEvaluationFromDetail(report OutcomeReport, detail string) (OutcomeEvaluation, bool) { switch { case strings.Contains(detail, "object still exists"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusUnknown, EvidenceStrength: EvidenceWeak, Signal: "target_exists_only"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusUnknown, EvidenceStrength: EvidenceWeak, Signal: "target_exists_only"}, true case strings.Contains(detail, "closed without merge"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "closed_without_merge"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "closed_without_merge"}, true case strings.Contains(detail, "closed as not planned"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "closed_not_planned"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "closed_not_planned"}, true case strings.Contains(detail, "closed by bot") && strings.Contains(detail, "lifecycle_close"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusLifecycleClose, EvidenceStrength: EvidenceMedium, Signal: "lifecycle_close"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusLifecycleClose, EvidenceStrength: EvidenceMedium, Signal: "lifecycle_close"}, true case strings.Contains(detail, "closed by bot"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusLifecycle, EvidenceStrength: EvidenceMedium, Signal: "lifecycle"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusLifecycle, EvidenceStrength: EvidenceMedium, Signal: "lifecycle"}, true case strings.Contains(detail, "merged"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceStrong, Signal: "merged"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceStrong, Signal: "merged"}, true case strings.Contains(detail, "reopened"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "reopened"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "reopened"}, true case strings.Contains(detail, "deleted"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "deleted"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "deleted"}, true case strings.Contains(detail, "completed"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceStrong, Signal: "completed"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceStrong, Signal: "completed"}, true case strings.Contains(detail, "milestone still assigned"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceMedium, Signal: "milestone_assigned"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceMedium, Signal: "milestone_assigned"}, true case strings.Contains(detail, "milestone removed"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceMedium, Signal: "milestone_removed"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceMedium, Signal: "milestone_removed"}, true case strings.Contains(detail, "reviews submitted"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceMedium, Signal: "reviewed"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceMedium, Signal: "reviewed"}, true case strings.Contains(detail, "awaiting review"), strings.Contains(detail, "no reviews yet"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusPending, EvidenceStrength: EvidenceMedium, Signal: "awaiting_review"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusPending, EvidenceStrength: EvidenceMedium, Signal: "awaiting_review"}, true case strings.Contains(detail, "no engagement"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusIgnored, EvidenceStrength: EvidenceMedium, Signal: "no_engagement"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusIgnored, EvidenceStrength: EvidenceMedium, Signal: "no_engagement"}, true case strings.Contains(detail, "human comments"), strings.Contains(detail, "with comments"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusPending, EvidenceStrength: EvidenceMedium, Signal: "acted_on"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusPending, EvidenceStrength: EvidenceMedium, Signal: "acted_on"}, true case strings.Contains(detail, "open"): - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusPending, EvidenceStrength: EvidenceMedium, Signal: "open"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusPending, EvidenceStrength: EvidenceMedium, Signal: "open"}, true case strings.Contains(detail, "closed"): if report.Result == OutcomeRejected { - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "closed"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusRejected, EvidenceStrength: EvidenceStrong, Signal: "closed"}, true } - return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceStrong, Signal: "closed"} + return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceStrong, Signal: "closed"}, true } + return OutcomeEvaluation{}, false +} - switch report.Result { +func normalizeOutcomeEvaluationFromResult(result OutcomeResult) OutcomeEvaluation { + switch result { case OutcomeAccepted: return OutcomeEvaluation{OutcomeStatus: OutcomeStatusAccepted, EvidenceStrength: EvidenceMedium, Signal: "acted_on"} case OutcomeRejected: diff --git a/pkg/cli/outcomes_command.go b/pkg/cli/outcomes_command.go index 68ae7a3fbc1..1232ab751b9 100644 --- a/pkg/cli/outcomes_command.go +++ b/pkg/cli/outcomes_command.go @@ -89,75 +89,20 @@ type OutcomesData struct { func RunOutcomes(config OutcomesConfig) error { outcomesLog.Printf("Evaluating outcomes for run %d", config.RunID) - // Resolve repo - repo := config.RepoOverride - if repo == "" { - slug, err := GetCurrentRepoSlug() - if err != nil { - return fmt.Errorf("could not determine repository: %w", err) - } - repo = slug - } - - // Parse owner/repo for artifact download - var owner, repoName, hostname string - parts := strings.SplitN(repo, "/", 2) - if len(parts) == 2 { - owner = parts[0] - repoName = parts[1] - } - - // Determine output directory for this run - outputDir := config.OutputDir - if outputDir == "" { - outputDir = defaultLogsOutputDir + repo, err := runOutcomesResolveRepo(config.RepoOverride) + if err != nil { + return err } - runDir := filepath.Join(outputDir, fmt.Sprintf("run-%d", config.RunID)) - - // Try to load from cache first + owner, repoName, hostname := runOutcomesRepoParts(repo) + runDir := runOutcomesRunDir(config) summary, cached := loadRunSummary(runDir, config.Verbose) - var items []CreatedItemReport - - if cached && summary != nil { - items = extractCreatedItemsFromManifest(runDir) - if len(items) == 0 { - items = extractCreatedItemsFromManifest(filepath.Join(runDir, "safe-outputs-items")) - } - // Enrich with data from raw agent output (has issue_number etc.) - items = enrichItemsFromAgentOutput(items, runDir, repo) - if config.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Loaded %d safe output items from cache", len(items)))) - } + items, err := runOutcomesLoadItems(config, runDir, repo, owner, repoName, hostname, cached, summary) + if err != nil { + return err } if len(items) == 0 { - if config.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Downloading artifacts for run %d...", config.RunID))) - } - ctx := context.Background() - err := downloadRunArtifacts(ctx, downloadArtifactsOptions{runID: config.RunID, outputDir: runDir, verbose: config.Verbose, owner: owner, repo: repoName, hostname: hostname}) - if err != nil { - return fmt.Errorf("failed to download artifacts for run %d: %w", config.RunID, err) - } - items = extractCreatedItemsFromManifest(runDir) - if len(items) == 0 { - items = extractCreatedItemsFromManifest(filepath.Join(runDir, "safe-outputs-items")) - } - items = enrichItemsFromAgentOutput(items, runDir, repo) - } - - if len(items) == 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No safe output items found for this run")) - if config.JSONOutput { - data := OutcomesData{ - RunID: config.RunID, - Items: []OutcomeReport{}, - Summary: OutcomeSummary{}, - } - out, _ := json.MarshalIndent(data, "", " ") - fmt.Fprintln(os.Stdout, string(out)) - } - return nil + return runOutcomesNoItems(config) } if config.Verbose { @@ -179,85 +124,155 @@ func RunOutcomes(config OutcomesConfig) error { writeOutcomeJSONL(outcomesDir, config.RunID, reports) } - // Get workflow name from cache if available - workflowName := "" + workflowName := runOutcomesWorkflowName(cached, summary) + + if config.JSONOutput { + return runOutcomesJSON(config, reports, outcomeSummary, workflowName) + } + runOutcomesConsole(config, reports, outcomeSummary, workflowName) + return nil +} + +func runOutcomesResolveRepo(repoOverride string) (string, error) { + if repoOverride != "" { + return repoOverride, nil + } + slug, err := GetCurrentRepoSlug() + if err != nil { + return "", fmt.Errorf("could not determine repository: %w", err) + } + return slug, nil +} + +func runOutcomesRepoParts(repo string) (string, string, string) { + var owner, repoName, hostname string + parts := strings.SplitN(repo, "/", 2) + if len(parts) == 2 { + owner = parts[0] + repoName = parts[1] + } + return owner, repoName, hostname +} + +func runOutcomesRunDir(config OutcomesConfig) string { + outputDir := config.OutputDir + if outputDir == "" { + outputDir = defaultLogsOutputDir + } + return filepath.Join(outputDir, fmt.Sprintf("run-%d", config.RunID)) +} + +func runOutcomesLoadItems(config OutcomesConfig, runDir string, repo string, owner string, repoName string, hostname string, cached bool, summary *RunSummary) ([]CreatedItemReport, error) { + var items []CreatedItemReport if cached && summary != nil { - workflowName = summary.Run.WorkflowName + items = runOutcomesExtractItems(runDir, repo) + if config.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Loaded %d safe output items from cache", len(items)))) + } + } + if len(items) > 0 { + return items, nil } + if config.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Downloading artifacts for run %d...", config.RunID))) + } + ctx := context.Background() + err := downloadRunArtifacts(ctx, downloadArtifactsOptions{runID: config.RunID, outputDir: runDir, verbose: config.Verbose, owner: owner, repo: repoName, hostname: hostname}) + if err != nil { + return nil, fmt.Errorf("failed to download artifacts for run %d: %w", config.RunID, err) + } + return runOutcomesExtractItems(runDir, repo), nil +} + +func runOutcomesExtractItems(runDir string, repo string) []CreatedItemReport { + items := extractCreatedItemsFromManifest(runDir) + if len(items) == 0 { + items = extractCreatedItemsFromManifest(filepath.Join(runDir, "safe-outputs-items")) + } + return enrichItemsFromAgentOutput(items, runDir, repo) +} +func runOutcomesNoItems(config OutcomesConfig) error { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No safe output items found for this run")) if config.JSONOutput { - data := OutcomesData{ - RunID: config.RunID, - Workflow: workflowName, - Items: reports, - Summary: outcomeSummary, - } - out, err := json.MarshalIndent(data, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal JSON: %w", err) - } + data := OutcomesData{RunID: config.RunID, Items: []OutcomeReport{}, Summary: OutcomeSummary{}} + out, _ := json.MarshalIndent(data, "", " ") fmt.Fprintln(os.Stdout, string(out)) - return nil } + return nil +} - // Console output +func runOutcomesWorkflowName(cached bool, summary *RunSummary) string { + if cached && summary != nil { + return summary.Run.WorkflowName + } + return "" +} + +func runOutcomesJSON(config OutcomesConfig, reports []OutcomeReport, outcomeSummary OutcomeSummary, workflowName string) error { + data := OutcomesData{RunID: config.RunID, Workflow: workflowName, Items: reports, Summary: outcomeSummary} + out, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON: %w", err) + } + fmt.Fprintln(os.Stdout, string(out)) + return nil +} + +func runOutcomesConsole(config OutcomesConfig, reports []OutcomeReport, outcomeSummary OutcomeSummary, workflowName string) { if workflowName != "" { fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage(fmt.Sprintf("Outcomes for %s (run %d)", workflowName, config.RunID))) } else { fmt.Fprintf(os.Stderr, "\n%s\n", console.FormatInfoMessage(fmt.Sprintf("Outcomes for run %d", config.RunID))) } + runOutcomesConsoleItems(reports) + runOutcomesConsoleSummary(outcomeSummary) +} - // Render the items +func runOutcomesConsoleItems(reports []OutcomeReport) { fmt.Fprintln(os.Stderr) for _, r := range reports { resultStr := string(r.Result) - detail := r.Detail - if detail != "" { - resultStr += " (" + detail + ")" + if r.Detail != "" { + resultStr += " (" + r.Detail + ")" } numStr := "" if r.ObjectNumber > 0 { numStr = fmt.Sprintf("#%d", r.ObjectNumber) } - timeStr := "" - if r.TimeToOutcomeHours > 0 { - if r.TimeToOutcomeHours < 1 { - timeStr = fmt.Sprintf("%.0fm", r.TimeToOutcomeHours*60) - } else { - timeStr = fmt.Sprintf("%.1fh", r.TimeToOutcomeHours) - } - } - fmt.Fprintf(os.Stderr, " %-28s %-12s %-40s %s\n", r.Type, numStr, resultStr, timeStr) + fmt.Fprintf(os.Stderr, " %-28s %-12s %-40s %s\n", r.Type, numStr, resultStr, runOutcomesTimeString(r.TimeToOutcomeHours)) } fmt.Fprintln(os.Stderr) +} - // Render summary +func runOutcomesTimeString(hours float64) string { + if hours <= 0 { + return "" + } + if hours < 1 { + return fmt.Sprintf("%.0fm", hours*60) + } + return fmt.Sprintf("%.1fh", hours) +} + +func runOutcomesConsoleSummary(outcomeSummary OutcomeSummary) { resolved := outcomeSummary.Accepted + outcomeSummary.Rejected fmt.Fprintf(os.Stderr, " Acceptance: %d/%d", outcomeSummary.Accepted, resolved) if resolved > 0 { fmt.Fprintf(os.Stderr, " (%.0f%%)", outcomeSummary.AcceptanceRate*100) } fmt.Fprintln(os.Stderr) - if outcomeSummary.Accepted > 0 { - fmt.Fprintf(os.Stderr, " Zero-touch: %d/%d (%.0f%%)\n", - outcomeSummary.ZeroTouch, outcomeSummary.Accepted, outcomeSummary.ZeroTouchRate*100) + fmt.Fprintf(os.Stderr, " Zero-touch: %d/%d (%.0f%%)\n", outcomeSummary.ZeroTouch, outcomeSummary.Accepted, outcomeSummary.ZeroTouchRate*100) } - if outcomeSummary.Rejected > 0 { - fmt.Fprintf(os.Stderr, " Waste: %d/%d (%.0f%%)\n", - outcomeSummary.Rejected, outcomeSummary.Total, outcomeSummary.WasteRate*100) + fmt.Fprintf(os.Stderr, " Waste: %d/%d (%.0f%%)\n", outcomeSummary.Rejected, outcomeSummary.Total, outcomeSummary.WasteRate*100) } - if outcomeSummary.Pending > 0 { fmt.Fprintf(os.Stderr, " Pending: %d\n", outcomeSummary.Pending) } - if outcomeSummary.MedianTimeToOutcome > 0 { fmt.Fprintf(os.Stderr, " Median time to outcome: %.1fh\n", outcomeSummary.MedianTimeToOutcome) } - fmt.Fprintln(os.Stderr) - - return nil } diff --git a/pkg/cli/outcomes_history.go b/pkg/cli/outcomes_history.go index bb664603d6d..cf89720d10d 100644 --- a/pkg/cli/outcomes_history.go +++ b/pkg/cli/outcomes_history.go @@ -115,26 +115,16 @@ has been closing or merging under the current objective mapping.`, } func RunOutcomesHistory(config OutcomesHistoryConfig) error { - repo := config.RepoOverride - if repo == "" { - slug, err := GetCurrentRepoSlug() - if err != nil { - return fmt.Errorf("could not determine repository: %w", err) - } - repo = slug - } - - if config.Limit <= 0 { - config.Limit = 200 - } - - source := strings.ToLower(strings.TrimSpace(config.Source)) - if source == "" { - source = historySourceAll + repo, err := runOutcomesHistoryRepo(config.RepoOverride) + if err != nil { + return err } - if source != historySourceAll && source != historySourceIssues && source != historySourcePRs { - return fmt.Errorf("invalid --source %q: expected issues, prs, or all", config.Source) + limit := runOutcomesHistoryLimit(config.Limit) + source, err := runOutcomesHistorySource(config.Source) + if err != nil { + return err } + config.Limit = limit outcomesHistoryLog.Printf("Running outcomes history: repo=%s, source=%s, limit=%d, json=%v", repo, source, config.Limit, config.JSONOutput) @@ -179,6 +169,35 @@ func RunOutcomesHistory(config OutcomesHistoryConfig) error { return nil } +func runOutcomesHistoryRepo(repoOverride string) (string, error) { + if repoOverride != "" { + return repoOverride, nil + } + slug, err := GetCurrentRepoSlug() + if err != nil { + return "", fmt.Errorf("could not determine repository: %w", err) + } + return slug, nil +} + +func runOutcomesHistoryLimit(limit int) int { + if limit <= 0 { + return 200 + } + return limit +} + +func runOutcomesHistorySource(configSource string) (string, error) { + source := strings.ToLower(strings.TrimSpace(configSource)) + if source == "" { + source = historySourceAll + } + if source != historySourceAll && source != historySourceIssues && source != historySourcePRs { + return "", fmt.Errorf("invalid --source %q: expected issues, prs, or all", configSource) + } + return source, nil +} + func fetchHistoricalGitHubItems(repo string, limit int, source string) ([]historicalGitHubItem, error) { args := []string{"--repo", repo, "--limit", strconv.Itoa(limit), "--json", "number,title,labels,url"} spinner := "Listing closed issues..." @@ -212,46 +231,61 @@ func buildHistoricalObjectiveReport(source string, items []historicalGitHubItem, scoredItems := 0 for _, item := range items { - labels := make([]string, 0, len(item.Labels)) - for _, label := range item.Labels { - labels = append(labels, label.Name) - } - - objectiveLabels := mapping.GetObjectiveLabels(labels) - objectiveValue := mapping.ComputeObjectiveValue(labels) + row, objectiveLabels, objectiveValue := buildHistoricalObjectiveReportRow(source, item, mapping) if objectiveValue > 0 { scoredItems++ } totalObjectiveValue += objectiveValue + buildHistoricalObjectiveReportCountBuckets(bucketCounts, objectiveLabels) + rows = append(rows, row) + } - for _, label := range objectiveLabels { - normalized := strings.ToLower(strings.TrimSpace(label)) - bucketCounts[normalized]++ - } + buckets := buildHistoricalObjectiveReportBuckets(bucketCounts, mapping) + buildHistoricalObjectiveReportSortRows(rows) + representative := buildHistoricalObjectiveReportRepresentative(rows) + outcomesHistoryLog.Printf("Built %s report: scored=%d/%d, totalValue=%d, buckets=%d", source, scoredItems, len(items), totalObjectiveValue, len(buckets)) + return historicalObjectiveReport{ + Source: source, + SampleSize: len(items), + ScoredItems: scoredItems, + TotalObjectiveValue: totalObjectiveValue, + ObjectiveBuckets: buckets, + RepresentativeItems: representative, + } +} + +func buildHistoricalObjectiveReportRow(source string, item historicalGitHubItem, mapping *ghmapping.ObjectiveMapping) (historicalObjectiveItem, []string, int) { + labels := make([]string, 0, len(item.Labels)) + for _, label := range item.Labels { + labels = append(labels, label.Name) + } + objectiveLabels := mapping.GetObjectiveLabels(labels) + objectiveValue := mapping.ComputeObjectiveValue(labels) + return historicalObjectiveItem{ + Kind: source, + Number: item.Number, + Title: item.Title, + URL: item.URL, + ClosedAt: item.ClosedAt, + MergedAt: item.MergedAt, + ObjectiveLabels: objectiveLabels, + ObjectiveValue: objectiveValue, + }, objectiveLabels, objectiveValue +} - rows = append(rows, historicalObjectiveItem{ - Kind: source, - Number: item.Number, - Title: item.Title, - URL: item.URL, - ClosedAt: item.ClosedAt, - MergedAt: item.MergedAt, - ObjectiveLabels: objectiveLabels, - ObjectiveValue: objectiveValue, - }) +func buildHistoricalObjectiveReportCountBuckets(bucketCounts map[string]int, objectiveLabels []string) { + for _, label := range objectiveLabels { + normalized := strings.ToLower(strings.TrimSpace(label)) + bucketCounts[normalized]++ } +} +func buildHistoricalObjectiveReportBuckets(bucketCounts map[string]int, mapping *ghmapping.ObjectiveMapping) []historicalObjectiveBucket { buckets := make([]historicalObjectiveBucket, 0, len(bucketCounts)) for label, count := range bucketCounts { mappedValue := mapping.LabelToValue[label] - buckets = append(buckets, historicalObjectiveBucket{ - Label: label, - Count: count, - MappedValue: mappedValue, - ContributedValue: mappedValue * count, - }) + buckets = append(buckets, historicalObjectiveBucket{Label: label, Count: count, MappedValue: mappedValue, ContributedValue: mappedValue * count}) } - slices.SortFunc(buckets, func(a, b historicalObjectiveBucket) int { if a.ContributedValue != b.ContributedValue { return cmp.Compare(b.ContributedValue, a.ContributedValue) @@ -264,7 +298,10 @@ func buildHistoricalObjectiveReport(source string, items []historicalGitHubItem, } return strings.Compare(a.Label, b.Label) }) + return buckets +} +func buildHistoricalObjectiveReportSortRows(rows []historicalObjectiveItem) { slices.SortFunc(rows, func(a, b historicalObjectiveItem) int { if a.ObjectiveValue != b.ObjectiveValue { return cmp.Compare(b.ObjectiveValue, a.ObjectiveValue) @@ -279,7 +316,9 @@ func buildHistoricalObjectiveReport(source string, items []historicalGitHubItem, } return strings.Compare(leftTime, rightTime) }) +} +func buildHistoricalObjectiveReportRepresentative(rows []historicalObjectiveItem) []historicalObjectiveItem { representative := make([]historicalObjectiveItem, 0, min(len(rows), 15)) for _, row := range rows { if row.ObjectiveValue <= 0 { @@ -290,16 +329,7 @@ func buildHistoricalObjectiveReport(source string, items []historicalGitHubItem, break } } - - outcomesHistoryLog.Printf("Built %s report: scored=%d/%d, totalValue=%d, buckets=%d", source, scoredItems, len(items), totalObjectiveValue, len(buckets)) - return historicalObjectiveReport{ - Source: source, - SampleSize: len(items), - ScoredItems: scoredItems, - TotalObjectiveValue: totalObjectiveValue, - ObjectiveBuckets: buckets, - RepresentativeItems: representative, - } + return representative } func renderHistoricalObjectiveReport(report historicalObjectiveReport) { diff --git a/pkg/cli/packages.go b/pkg/cli/packages.go index 33a41063090..b54d18b9eb2 100644 --- a/pkg/cli/packages.go +++ b/pkg/cli/packages.go @@ -28,8 +28,7 @@ var ( func collectLocalIncludeDependencies(content, packagePath string, verbose bool) ([]IncludeDependency, error) { packagesLog.Printf("Collecting include dependencies: packagePath=%s, content_size=%d", packagePath, len(content)) var dependencies []IncludeDependency - seen := make(map[string]struct { - }) + seen := make(map[string]struct{}) if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Collecting package dependencies from: "+packagePath)) @@ -41,147 +40,143 @@ func collectLocalIncludeDependencies(content, packagePath string, verbose bool) } // collectLocalIncludeDependenciesRecursive recursively processes @include directives in package content -func collectLocalIncludeDependenciesRecursive(content, baseDir string, dependencies *[]IncludeDependency, seen map[string]struct { -}, verbose bool) error { +func collectLocalIncludeDependenciesRecursive(content, baseDir string, dependencies *[]IncludeDependency, seen map[string]struct{}, verbose bool) error { scanner := bufio.NewScanner(strings.NewReader(content)) for scanner.Scan() { - line := scanner.Text() - if matches := includePattern.FindStringSubmatch(line); matches != nil { - isOptional := matches[1] == "?" - includePath := strings.TrimSpace(matches[2]) - - // Handle section references (file.md#Section) - var filePath string - if strings.Contains(includePath, "#") { - parts := strings.SplitN(includePath, "#", 2) - filePath = parts[0] - } else { - filePath = includePath - } - - // Resolve the full source path relative to base directory - fullSourcePath := filepath.Join(baseDir, filePath) - - // Skip if we've already processed this file - if setutil.Contains(seen, fullSourcePath) { - continue - } - seen[fullSourcePath] = struct { - }{} - - // Add dependency - dep := IncludeDependency{ - SourcePath: fullSourcePath, - TargetPath: filePath, // Keep relative path for target - IsOptional: isOptional, - } - *dependencies = append(*dependencies, dep) + collectLocalIncludeDependenciesRecursiveLine(scanner.Text(), baseDir, dependencies, seen, verbose) + } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found include dependency: %s -> %s", fullSourcePath, filePath))) - } + return scanner.Err() +} - // Read the included file and process its includes recursively - includedContent, err := os.ReadFile(fullSourcePath) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not read include file %s: %v", fullSourcePath, err))) - } - continue - } +func collectLocalIncludeDependenciesRecursiveLine(line string, baseDir string, dependencies *[]IncludeDependency, seen map[string]struct{}, verbose bool) { + matches := includePattern.FindStringSubmatch(line) + if matches == nil { + return + } + isOptional := matches[1] == "?" + includePath := strings.TrimSpace(matches[2]) + filePath := collectLocalIncludeDependenciesRecursiveFilePath(includePath) + fullSourcePath := filepath.Join(baseDir, filePath) + if setutil.Contains(seen, fullSourcePath) { + return + } + seen[fullSourcePath] = struct{}{} + *dependencies = append(*dependencies, IncludeDependency{SourcePath: fullSourcePath, TargetPath: filePath, IsOptional: isOptional}) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found include dependency: %s -> %s", fullSourcePath, filePath))) + } + collectLocalIncludeDependenciesRecursiveDescend(fullSourcePath, dependencies, seen, verbose) +} - // Extract markdown content from the included file - markdownContent, err := parser.ExtractMarkdownContent(string(includedContent)) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not extract markdown from %s: %v", fullSourcePath, err))) - } - continue - } +func collectLocalIncludeDependenciesRecursiveFilePath(includePath string) string { + if strings.Contains(includePath, "#") { + parts := strings.SplitN(includePath, "#", 2) + return parts[0] + } + return includePath +} - // Recursively process includes in the included file - includedDir := filepath.Dir(fullSourcePath) - if err := collectLocalIncludeDependenciesRecursive(markdownContent, includedDir, dependencies, seen, verbose); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Error processing includes in %s: %v", fullSourcePath, err))) - } - } +func collectLocalIncludeDependenciesRecursiveDescend(fullSourcePath string, dependencies *[]IncludeDependency, seen map[string]struct{}, verbose bool) { + includedContent, err := os.ReadFile(fullSourcePath) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not read include file %s: %v", fullSourcePath, err))) } + return + } + markdownContent, err := parser.ExtractMarkdownContent(string(includedContent)) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not extract markdown from %s: %v", fullSourcePath, err))) + } + return + } + includedDir := filepath.Dir(fullSourcePath) + if err := collectLocalIncludeDependenciesRecursive(markdownContent, includedDir, dependencies, seen, verbose); err != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Error processing includes in %s: %v", fullSourcePath, err))) } - - return scanner.Err() } // copyIncludeDependenciesFromPackageWithForce copies include dependencies from package filesystem with force option func copyIncludeDependenciesFromPackageWithForce(dependencies []IncludeDependency, githubWorkflowsDir string, verbose bool, force bool, tracker *FileTracker) error { packagesLog.Printf("Copying %d include dependencies to %s (force=%t)", len(dependencies), githubWorkflowsDir, force) for _, dep := range dependencies { - // Create the target path in .github/workflows - targetPath := filepath.Join(githubWorkflowsDir, dep.TargetPath) - - // Create target directory if it doesn't exist - targetDir := filepath.Dir(targetPath) - if err := os.MkdirAll(targetDir, constants.DirPermPublic); err != nil { - return fmt.Errorf("failed to create directory %s: %w", targetDir, err) - } - - // Read source content from package - sourceContent, err := os.ReadFile(dep.SourcePath) - if err != nil { - if dep.IsOptional { - // For optional includes, just show an informational message and skip - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Optional include file not found: %s (you can create this file to configure the workflow)", dep.TargetPath))) - } - continue - } - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read include file %s: %v", dep.SourcePath, err))) - continue + if err := copyIncludeDependenciesFromPackageWithForceOne(dep, githubWorkflowsDir, verbose, force, tracker); err != nil { + return err } + } - // Check if target file already exists - fileExists := false - if existingContent, err := os.ReadFile(targetPath); err == nil { - fileExists = true - // File exists, compare contents - if bytes.Equal(existingContent, sourceContent) { - // Contents are the same, skip - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Include file %s already exists with same content, skipping", dep.TargetPath))) - } - continue - } - - // Contents are different - if !force { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Include file %s already exists with different content, skipping (use --force to overwrite)", dep.TargetPath))) - continue - } - - // Force is enabled, overwrite - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Overwriting existing include file: "+dep.TargetPath)) - } + return nil +} - // Track the file based on whether it existed before (if tracker is available) - if tracker != nil { - if fileExists { - tracker.TrackModified(targetPath) - } else { - tracker.TrackCreated(targetPath) - } - } +func copyIncludeDependenciesFromPackageWithForceOne(dep IncludeDependency, githubWorkflowsDir string, verbose bool, force bool, tracker *FileTracker) error { + targetPath := filepath.Join(githubWorkflowsDir, dep.TargetPath) + targetDir := filepath.Dir(targetPath) + if err := os.MkdirAll(targetDir, constants.DirPermPublic); err != nil { + return fmt.Errorf("failed to create directory %s: %w", targetDir, err) + } + sourceContent, ok, err := copyIncludeDependenciesFromPackageWithForceReadSource(dep, verbose) + if err != nil || !ok { + return err + } + fileExists, shouldWrite := copyIncludeDependenciesFromPackageWithForceShouldWrite(dep, targetPath, sourceContent, verbose, force) + if !shouldWrite { + return nil + } + copyIncludeDependenciesFromPackageWithForceTrack(tracker, targetPath, fileExists) + if err := os.WriteFile(targetPath, sourceContent, constants.FilePermPublic); err != nil { + return fmt.Errorf("failed to write include file %s: %w", targetPath, err) + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Copied include file: %s -> %s", dep.SourcePath, targetPath))) + } + return nil +} - // Write to target - if err := os.WriteFile(targetPath, sourceContent, constants.FilePermPublic); err != nil { - return fmt.Errorf("failed to write include file %s: %w", targetPath, err) +func copyIncludeDependenciesFromPackageWithForceReadSource(dep IncludeDependency, verbose bool) ([]byte, bool, error) { + sourceContent, err := os.ReadFile(dep.SourcePath) + if err == nil { + return sourceContent, true, nil + } + if dep.IsOptional { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Optional include file not found: %s (you can create this file to configure the workflow)", dep.TargetPath))) } + return nil, false, nil + } + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read include file %s: %v", dep.SourcePath, err))) + return nil, false, nil +} +func copyIncludeDependenciesFromPackageWithForceShouldWrite(dep IncludeDependency, targetPath string, sourceContent []byte, verbose bool, force bool) (bool, bool) { + existingContent, err := os.ReadFile(targetPath) + if err != nil { + return false, true + } + if bytes.Equal(existingContent, sourceContent) { if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Copied include file: %s -> %s", dep.SourcePath, targetPath))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Include file %s already exists with same content, skipping", dep.TargetPath))) } + return true, false + } + if !force { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Include file %s already exists with different content, skipping (use --force to overwrite)", dep.TargetPath))) + return true, false } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Overwriting existing include file: "+dep.TargetPath)) + return true, true +} - return nil +func copyIncludeDependenciesFromPackageWithForceTrack(tracker *FileTracker, targetPath string, fileExists bool) { + if tracker == nil { + return + } + if fileExists { + tracker.TrackModified(targetPath) + } else { + tracker.TrackCreated(targetPath) + } } // IncludeDependency represents a file dependency from @include directives diff --git a/pkg/cli/poutine.go b/pkg/cli/poutine.go index 5226b52c80a..31a1534e609 100644 --- a/pkg/cli/poutine.go +++ b/pkg/cli/poutine.go @@ -77,48 +77,16 @@ rulesConfig: func runPoutineOnDirectory(workflowDir string, verbose bool, strict bool) error { poutineLog.Printf("Running poutine security scanner on directory: %s", workflowDir) - // Find git root to get the absolute path for Docker volume mount - gitRoot, err := gitutil.FindGitRoot() + gitRoot, cmd, err := runPoutineOnDirectoryCommand() if err != nil { - return fmt.Errorf("failed to find git root: %w", err) - } - - // Validate gitRoot is an absolute path (security: ensure trusted path from git) - if !filepath.IsAbs(gitRoot) { - return fmt.Errorf("git root is not an absolute path: %s", gitRoot) - } - - // Ensure poutine config exists with custom runner configuration - if err := ensurePoutineConfig(gitRoot); err != nil { - return fmt.Errorf("failed to ensure poutine config: %w", err) + return err } - // Build the Docker command with JSON output for easier parsing - // docker run --rm -v "$(pwd)":/workdir -w /workdir ghcr.io/boostsecurityio/poutine:latest analyze_local . --format json - // #nosec G204 -- gitRoot comes from git rev-parse (trusted source) and is validated as absolute path - // exec.Command with separate args (not shell execution) prevents command injection - cmd := exec.Command( - "docker", - "run", - "--rm", - "-v", gitRoot+":/workdir", - "-w", "/workdir", - "ghcr.io/boostsecurityio/poutine:latest", - "analyze_local", - ".", - "--format", "json", - "--quiet", // Disable progress output - ) - // Always show that poutine is running (regular verbosity) fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Running poutine security scanner")) // In verbose mode, also show the command that users can run directly - if verbose { - dockerCmd := fmt.Sprintf("docker run --rm -v \"%s:/workdir\" -w /workdir ghcr.io/boostsecurityio/poutine:latest analyze_local . --format json --quiet", - gitRoot) - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Run poutine directly: "+dockerCmd)) - } + runPoutineOnDirectoryVerboseCommand(verbose, gitRoot) // Capture output var stdout, stderr bytes.Buffer @@ -142,84 +110,78 @@ func runPoutineOnDirectory(workflowDir string, verbose bool, strict bool) error } // Check if the error is due to findings or actual failure - if err != nil { - // poutine exits with non-zero code when findings are present - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - exitCode := exitErr.ExitCode() - poutineLog.Printf("Poutine exited with code %d (warnings=%d)", exitCode, totalWarnings) - // Exit code 1 typically indicates findings in the repository - if exitCode == 1 { - // In strict mode, any findings in the scan are treated as errors - if strict && totalWarnings > 0 { - return fmt.Errorf("strict mode: poutine found %d security warnings/errors - workflows must have no poutine findings in strict mode", totalWarnings) - } - // In non-strict mode, findings are logged but not treated as errors - return nil - } - // Other exit codes are actual errors - return fmt.Errorf("poutine failed with exit code %d", exitCode) - } - // Non-ExitError errors (e.g., command not found) - return fmt.Errorf("poutine failed: %w", err) + if err := runPoutineOnDirectoryHandleError(err, totalWarnings, strict); err != nil { + return err } return nil } -// runPoutineOnFile runs the poutine security scanner on a single .lock.yml file using Docker -// This is a wrapper that filters the directory scan results to a single file for backward compatibility -func runPoutineOnFile(lockFile string, verbose bool, strict bool) error { - poutineLog.Printf("Running poutine security scanner: file=%s, strict=%v", lockFile, strict) - - // Find git root to get the absolute path for Docker volume mount +func runPoutineOnDirectoryCommand() (string, *exec.Cmd, error) { gitRoot, err := gitutil.FindGitRoot() if err != nil { - return fmt.Errorf("failed to find git root: %w", err) + return "", nil, fmt.Errorf("failed to find git root: %w", err) } - - // Validate gitRoot is an absolute path (security: ensure trusted path from git) if !filepath.IsAbs(gitRoot) { - return fmt.Errorf("git root is not an absolute path: %s", gitRoot) + return "", nil, fmt.Errorf("git root is not an absolute path: %s", gitRoot) } - - // Ensure poutine config exists with custom runner configuration if err := ensurePoutineConfig(gitRoot); err != nil { - return fmt.Errorf("failed to ensure poutine config: %w", err) + return "", nil, fmt.Errorf("failed to ensure poutine config: %w", err) } + // #nosec G204 -- gitRoot comes from git rev-parse (trusted source) and is validated as absolute path + cmd := exec.Command("docker", "run", "--rm", "-v", gitRoot+":/workdir", "-w", "/workdir", "ghcr.io/boostsecurityio/poutine:latest", "analyze_local", ".", "--format", "json", "--quiet") + return gitRoot, cmd, nil +} +func runPoutineOnDirectoryVerboseCommand(verbose bool, gitRoot string) { + if !verbose { + return + } + dockerCmd := fmt.Sprintf("docker run --rm -v \"%s:/workdir\" -w /workdir ghcr.io/boostsecurityio/poutine:latest analyze_local . --format json --quiet", gitRoot) + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Run poutine directly: "+dockerCmd)) +} + +func runPoutineOnDirectoryHandleError(err error, totalWarnings int, strict bool) error { + if err == nil { + return nil + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitCode := exitErr.ExitCode() + poutineLog.Printf("Poutine exited with code %d (warnings=%d)", exitCode, totalWarnings) + if exitCode == 1 { + if strict && totalWarnings > 0 { + return fmt.Errorf("strict mode: poutine found %d security warnings/errors - workflows must have no poutine findings in strict mode", totalWarnings) + } + return nil + } + return fmt.Errorf("poutine failed with exit code %d", exitCode) + } + return fmt.Errorf("poutine failed: %w", err) +} + +// runPoutineOnFile runs the poutine security scanner on a single .lock.yml file using Docker +// This is a wrapper that filters the directory scan results to a single file for backward compatibility +func runPoutineOnFile(lockFile string, verbose bool, strict bool) error { + poutineLog.Printf("Running poutine security scanner: file=%s, strict=%v", lockFile, strict) + + gitRoot, err := runPoutineOnFileGitRoot() + if err != nil { + return err + } // Get the relative path from git root relPath, err := filepath.Rel(gitRoot, lockFile) if err != nil { return fmt.Errorf("failed to get relative path: %w", err) } - // Build the Docker command with JSON output for easier parsing - // docker run --rm -v "$(pwd)":/workdir -w /workdir ghcr.io/boostsecurityio/poutine:latest analyze_local . --format json - // #nosec G204 -- gitRoot comes from git rev-parse (trusted source) and is validated as absolute path - // exec.Command with separate args (not shell execution) prevents command injection - cmd := exec.Command( - "docker", - "run", - "--rm", - "-v", gitRoot+":/workdir", - "-w", "/workdir", - "ghcr.io/boostsecurityio/poutine:latest", - "analyze_local", - ".", - "--format", "json", - "--quiet", // Disable progress output - ) + cmd := runPoutineOnFileCommand(gitRoot) // Always show that poutine is running (regular verbosity) fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Running poutine security scanner")) // In verbose mode, also show the command that users can run directly - if verbose { - dockerCmd := fmt.Sprintf("docker run --rm -v \"%s:/workdir\" -w /workdir ghcr.io/boostsecurityio/poutine:latest analyze_local . --format json --quiet", - gitRoot) - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Run poutine directly: "+dockerCmd)) - } + runPoutineOnFileVerboseCommand(verbose, gitRoot) // Capture output var stdout, stderr bytes.Buffer @@ -243,163 +205,200 @@ func runPoutineOnFile(lockFile string, verbose bool, strict bool) error { } // Check if the error is due to findings or actual failure + if err := runPoutineOnFileHandleError(err, totalWarnings, strict, lockFile); err != nil { + return err + } + + return nil +} + +func runPoutineOnFileGitRoot() (string, error) { + gitRoot, err := gitutil.FindGitRoot() if err != nil { - // poutine exits with non-zero code when findings are present - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - exitCode := exitErr.ExitCode() - poutineLog.Printf("Poutine exited with code %d (warnings=%d)", exitCode, totalWarnings) - // Exit code 1 typically indicates findings in the repository - // In non-strict mode, we allow this even if we don't have findings - // specific to the current file (poutine scans the whole directory) - if exitCode == 1 { - // In strict mode, any findings in the scan are treated as errors - if strict && totalWarnings > 0 { - return fmt.Errorf("strict mode: poutine found %d security warnings/errors in %s - workflows must have no poutine findings in strict mode", totalWarnings, filepath.Base(lockFile)) - } - // In non-strict mode, findings are logged but not treated as errors - return nil + return "", fmt.Errorf("failed to find git root: %w", err) + } + if !filepath.IsAbs(gitRoot) { + return "", fmt.Errorf("git root is not an absolute path: %s", gitRoot) + } + if err := ensurePoutineConfig(gitRoot); err != nil { + return "", fmt.Errorf("failed to ensure poutine config: %w", err) + } + return gitRoot, nil +} + +func runPoutineOnFileCommand(gitRoot string) *exec.Cmd { + // #nosec G204 -- gitRoot comes from git rev-parse (trusted source) and is validated as absolute path + return exec.Command("docker", "run", "--rm", "-v", gitRoot+":/workdir", "-w", "/workdir", "ghcr.io/boostsecurityio/poutine:latest", "analyze_local", ".", "--format", "json", "--quiet") +} + +func runPoutineOnFileVerboseCommand(verbose bool, gitRoot string) { + if !verbose { + return + } + dockerCmd := fmt.Sprintf("docker run --rm -v \"%s:/workdir\" -w /workdir ghcr.io/boostsecurityio/poutine:latest analyze_local . --format json --quiet", gitRoot) + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Run poutine directly: "+dockerCmd)) +} + +func runPoutineOnFileHandleError(err error, totalWarnings int, strict bool, lockFile string) error { + if err == nil { + return nil + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitCode := exitErr.ExitCode() + poutineLog.Printf("Poutine exited with code %d (warnings=%d)", exitCode, totalWarnings) + if exitCode == 1 { + if strict && totalWarnings > 0 { + return fmt.Errorf("strict mode: poutine found %d security warnings/errors in %s - workflows must have no poutine findings in strict mode", totalWarnings, filepath.Base(lockFile)) } - // Other exit codes are actual errors - return fmt.Errorf("poutine failed with exit code %d on %s", exitCode, filepath.Base(lockFile)) + return nil } - // Non-ExitError errors (e.g., command not found) - return fmt.Errorf("poutine failed on %s: %w", filepath.Base(lockFile), err) + return fmt.Errorf("poutine failed with exit code %d on %s", exitCode, filepath.Base(lockFile)) } - - return nil + return fmt.Errorf("poutine failed on %s: %w", filepath.Base(lockFile), err) } // parseAndDisplayPoutineOutput parses poutine JSON output and displays it in the desired format // Returns the total number of warnings found for the specific file func parseAndDisplayPoutineOutput(stdout, targetFile string, verbose bool) (int, error) { // Parse JSON output from stdout + output, ok, err := parseAndDisplayPoutineOutputParse(stdout) + if err != nil || !ok { + return 0, err + } + + // Filter findings to only those relevant to the target file + relevantFindings := parseAndDisplayPoutineOutputRelevant(output.Findings, targetFile) + totalWarnings := len(relevantFindings) + + // Skip files with 0 warnings + if totalWarnings == 0 { + return 0, nil + } + + // Read file content for context display + fileLines := parseAndDisplayPoutineOutputFileLines(targetFile) + + // Display detailed findings using CompilerError format + for _, finding := range relevantFindings { + parseAndDisplayPoutineOutputFinding(finding, output.Rules, fileLines) + } + + return totalWarnings, nil +} + +func parseAndDisplayPoutineOutputParse(stdout string) (poutineOutput, bool, error) { var output poutineOutput if stdout == "" { - return 0, nil // No output means no findings + return output, false, nil } - trimmed := strings.TrimSpace(stdout) if !strings.HasPrefix(trimmed, "{") { - // Non-JSON output, likely an error if trimmed != "" { - return 0, fmt.Errorf("unexpected poutine output format: %s", trimmed) + return output, false, fmt.Errorf("unexpected poutine output format: %s", trimmed) } - return 0, nil + return output, false, nil } - if err := json.Unmarshal([]byte(stdout), &output); err != nil { - return 0, fmt.Errorf("failed to parse poutine JSON output: %w", err) + return output, false, fmt.Errorf("failed to parse poutine JSON output: %w", err) } + return output, true, nil +} - // Filter findings to only those relevant to the target file +func parseAndDisplayPoutineOutputRelevant(findings []poutineFinding, targetFile string) []poutineFinding { var relevantFindings []poutineFinding - for _, finding := range output.Findings { + for _, finding := range findings { if finding.Meta.Path == targetFile { relevantFindings = append(relevantFindings, finding) } } + return relevantFindings +} - totalWarnings := len(relevantFindings) - - // Skip files with 0 warnings - if totalWarnings == 0 { - return 0, nil - } - - // Read file content for context display +func parseAndDisplayPoutineOutputFileLines(targetFile string) []string { fileContent, err := os.ReadFile(targetFile) - var fileLines []string - if err == nil { - fileLines = strings.Split(string(fileContent), "\n") + if err != nil { + return nil } + return strings.Split(string(fileContent), "\n") +} - // Display detailed findings using CompilerError format - for _, finding := range relevantFindings { - // Get rule details - ruleInfo := output.Rules[finding.RuleID] - severity := ruleInfo.Level - if severity == "" { - severity = "warning" // Default to warning if not specified - } - - title := ruleInfo.Title - if title == "" { - title = finding.RuleID - } +func parseAndDisplayPoutineOutputFinding(finding poutineFinding, rules map[string]struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Level string `json:"level"` +}, fileLines []string) { + ruleInfo := rules[finding.RuleID] + severity, title := parseAndDisplayPoutineOutputRule(finding, ruleInfo.Level, ruleInfo.Title) + lineNum := parseAndDisplayPoutineOutputLine(finding) + context := parseAndDisplayPoutineOutputContext(fileLines, lineNum) + compilerErr := console.CompilerError{ + Position: console.ErrorPosition{File: finding.Meta.Path, Line: lineNum, Column: 1}, + Type: parseAndDisplayPoutineOutputErrorType(severity), + Message: parseAndDisplayPoutineOutputMessage(finding, severity, title), + Context: context, + } + fmt.Fprint(os.Stderr, console.FormatError(compilerErr)) +} - // Get line number (poutine uses 1-based indexing) - lineNum := finding.Meta.Line - if lineNum == 0 { - lineNum = 1 // Default to line 1 if not specified - } +func parseAndDisplayPoutineOutputRule(finding poutineFinding, severity string, title string) (string, string) { + if severity == "" { + severity = "warning" + } + if title == "" { + title = finding.RuleID + } + return severity, title +} - // Create context lines around the error - var context []string - if len(fileLines) > 0 && lineNum > 0 && lineNum <= len(fileLines) { - startLine := max(1, lineNum-2) - endLine := min(len(fileLines), lineNum+2) +func parseAndDisplayPoutineOutputLine(finding poutineFinding) int { + if finding.Meta.Line == 0 { + return 1 + } + return finding.Meta.Line +} - for i := startLine; i <= endLine; i++ { - if i-1 < len(fileLines) { - context = append(context, fileLines[i-1]) - } +func parseAndDisplayPoutineOutputContext(fileLines []string, lineNum int) []string { + var context []string + if len(fileLines) > 0 && lineNum > 0 && lineNum <= len(fileLines) { + startLine := max(1, lineNum-2) + endLine := min(len(fileLines), lineNum+2) + for i := startLine; i <= endLine; i++ { + if i-1 < len(fileLines) { + context = append(context, fileLines[i-1]) } } + } + return context +} - // Map severity to error type - errorType := "warning" - switch severity { - case "error": - errorType = "error" - case "note": - errorType = "info" - } - - // Build message with details - message := fmt.Sprintf("[%s] %s: %s", severity, finding.RuleID, title) - if finding.Meta.Details != "" { - message = fmt.Sprintf("%s - %s", message, finding.Meta.Details) - } - - // Create and format CompilerError - compilerErr := console.CompilerError{ - Position: console.ErrorPosition{ - File: finding.Meta.Path, - Line: lineNum, - Column: 1, // poutine doesn't provide column info - }, - Type: errorType, - Message: message, - Context: context, - } - - fmt.Fprint(os.Stderr, console.FormatError(compilerErr)) +func parseAndDisplayPoutineOutputErrorType(severity string) string { + switch severity { + case "error": + return "error" + case "note": + return "info" + default: + return "warning" } +} - return totalWarnings, nil +func parseAndDisplayPoutineOutputMessage(finding poutineFinding, severity string, title string) string { + message := fmt.Sprintf("[%s] %s: %s", severity, finding.RuleID, title) + if finding.Meta.Details != "" { + message = fmt.Sprintf("%s - %s", message, finding.Meta.Details) + } + return message } // parseAndDisplayPoutineOutputForDirectory parses poutine JSON output and displays all findings // Returns the total number of warnings found across all files func parseAndDisplayPoutineOutputForDirectory(stdout string, verbose bool, gitRoot string) (int, error) { // Parse JSON output from stdout - var output poutineOutput - if stdout == "" { - return 0, nil // No output means no findings - } - - trimmed := strings.TrimSpace(stdout) - if !strings.HasPrefix(trimmed, "{") { - // Non-JSON output, likely an error - if trimmed != "" { - return 0, fmt.Errorf("unexpected poutine output format: %s", trimmed) - } - return 0, nil - } - - if err := json.Unmarshal([]byte(stdout), &output); err != nil { - return 0, fmt.Errorf("failed to parse poutine JSON output: %w", err) + output, ok, err := parseAndDisplayPoutineOutputParse(stdout) + if err != nil || !ok { + return 0, err } // Display all findings (no filtering by file) @@ -411,115 +410,87 @@ func parseAndDisplayPoutineOutputForDirectory(stdout string, verbose bool, gitRo } // Group findings by file for better readability - findingsByFile := make(map[string][]poutineFinding) - for _, finding := range output.Findings { - findingsByFile[finding.Meta.Path] = append(findingsByFile[finding.Meta.Path], finding) - } + findingsByFile := parseAndDisplayPoutineOutputForDirectoryByFile(output.Findings) // Display findings for each file for filePath, findings := range findingsByFile { - // Validate and sanitize file path to prevent path traversal - cleanPath := filepath.Clean(filePath) - - // Convert to absolute path if relative - absPath := cleanPath - if !filepath.IsAbs(cleanPath) { - absPath = filepath.Join(gitRoot, cleanPath) - } - - // Ensure the file is within gitRoot to prevent path traversal - absGitRoot, err := filepath.Abs(gitRoot) - if err != nil { - poutineLog.Printf("Failed to get absolute path for git root: %v", err) - continue - } - - absPath, err = filepath.Abs(absPath) - if err != nil { - poutineLog.Printf("Failed to get absolute path for %s: %v", filePath, err) - continue - } - - // Check if the resolved path is within gitRoot - relPath, err := filepath.Rel(absGitRoot, absPath) - if err != nil || strings.HasPrefix(relPath, "..") { - poutineLog.Printf("Skipping file outside git root: %s", filePath) - continue - } - - // Read file content for context display - // #nosec G304 -- absPath is validated through: 1) filepath.Clean() normalization, - // 2) absolute path resolution, and 3) filepath.Rel() check ensuring it's within gitRoot - // (lines 414-441). Path traversal attacks are prevented by the boundary validation. - fileContent, err := os.ReadFile(absPath) - var fileLines []string - if err == nil { - fileLines = strings.Split(string(fileContent), "\n") - } - - // Display detailed findings using CompilerError format - for _, finding := range findings { - // Get rule details - ruleInfo := output.Rules[finding.RuleID] - severity := ruleInfo.Level - if severity == "" { - severity = "warning" // Default to warning if not specified - } - - title := ruleInfo.Title - if title == "" { - title = finding.RuleID - } - - // Get line number (poutine uses 1-based indexing) - lineNum := finding.Meta.Line - if lineNum == 0 { - lineNum = 1 // Default to line 1 if not specified - } + parseAndDisplayPoutineOutputForDirectoryFile(filePath, findings, output.Rules, gitRoot) + } - // Create context lines around the error - var context []string - if len(fileLines) > 0 && lineNum > 0 && lineNum <= len(fileLines) { - startLine := max(1, lineNum-2) - endLine := min(len(fileLines), lineNum+2) - - for i := startLine; i <= endLine; i++ { - if i-1 < len(fileLines) { - context = append(context, fileLines[i-1]) - } - } - } + return totalWarnings, nil +} - // Map severity to error type - errorType := "warning" - switch severity { - case "error": - errorType = "error" - case "note": - errorType = "info" - } +func parseAndDisplayPoutineOutputForDirectoryByFile(findings []poutineFinding) map[string][]poutineFinding { + findingsByFile := make(map[string][]poutineFinding) + for _, finding := range findings { + findingsByFile[finding.Meta.Path] = append(findingsByFile[finding.Meta.Path], finding) + } + return findingsByFile +} - // Build message with details - message := fmt.Sprintf("[%s] %s: %s", severity, finding.RuleID, title) - if finding.Meta.Details != "" { - message = fmt.Sprintf("%s - %s", message, finding.Meta.Details) - } +func parseAndDisplayPoutineOutputForDirectoryFile(filePath string, findings []poutineFinding, rules map[string]struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Level string `json:"level"` +}, gitRoot string) { + absPath, ok := parseAndDisplayPoutineOutputForDirectoryPath(filePath, gitRoot) + if !ok { + return + } + fileLines := parseAndDisplayPoutineOutputForDirectoryFileLines(absPath) + for _, finding := range findings { + parseAndDisplayPoutineOutputForDirectoryFinding(finding, rules, fileLines) + } +} - // Create and format CompilerError - compilerErr := console.CompilerError{ - Position: console.ErrorPosition{ - File: finding.Meta.Path, - Line: lineNum, - Column: 1, // poutine doesn't provide column info - }, - Type: errorType, - Message: message, - Context: context, - } +func parseAndDisplayPoutineOutputForDirectoryPath(filePath string, gitRoot string) (string, bool) { + cleanPath := filepath.Clean(filePath) + absPath := cleanPath + if !filepath.IsAbs(cleanPath) { + absPath = filepath.Join(gitRoot, cleanPath) + } + absGitRoot, err := filepath.Abs(gitRoot) + if err != nil { + poutineLog.Printf("Failed to get absolute path for git root: %v", err) + return "", false + } + absPath, err = filepath.Abs(absPath) + if err != nil { + poutineLog.Printf("Failed to get absolute path for %s: %v", filePath, err) + return "", false + } + relPath, err := filepath.Rel(absGitRoot, absPath) + if err != nil || strings.HasPrefix(relPath, "..") { + poutineLog.Printf("Skipping file outside git root: %s", filePath) + return "", false + } + return absPath, true +} - fmt.Fprint(os.Stderr, console.FormatError(compilerErr)) - } +func parseAndDisplayPoutineOutputForDirectoryFileLines(absPath string) []string { + // #nosec G304 -- absPath is validated through filepath.Clean(), absolute path resolution, and filepath.Rel() boundary checks. + fileContent, err := os.ReadFile(absPath) + if err != nil { + return nil } + return strings.Split(string(fileContent), "\n") +} - return totalWarnings, nil +func parseAndDisplayPoutineOutputForDirectoryFinding(finding poutineFinding, rules map[string]struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Level string `json:"level"` +}, fileLines []string) { + ruleInfo := rules[finding.RuleID] + severity, title := parseAndDisplayPoutineOutputRule(finding, ruleInfo.Level, ruleInfo.Title) + lineNum := parseAndDisplayPoutineOutputLine(finding) + compilerErr := console.CompilerError{ + Position: console.ErrorPosition{File: finding.Meta.Path, Line: lineNum, Column: 1}, + Type: parseAndDisplayPoutineOutputErrorType(severity), + Message: parseAndDisplayPoutineOutputMessage(finding, severity, title), + Context: parseAndDisplayPoutineOutputContext(fileLines, lineNum), + } + fmt.Fprint(os.Stderr, console.FormatError(compilerErr)) } diff --git a/pkg/cli/pr_automerge.go b/pkg/cli/pr_automerge.go index 4c4cf796d02..8400346b8b1 100644 --- a/pkg/cli/pr_automerge.go +++ b/pkg/cli/pr_automerge.go @@ -56,14 +56,7 @@ func AutoMergePullRequestsCreatedAfter(repoSlug string, createdAfter time.Time, } // Filter PRs to only those created after the specified time - var eligiblePRs []PullRequest - for _, pr := range prs { - if pr.CreatedAt.After(createdAfter) { - eligiblePRs = append(eligiblePRs, pr) - } else if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Skipping PR #%d: created at %s (before workflow start time)", pr.Number, pr.CreatedAt.Format(time.RFC3339)))) - } - } + eligiblePRs := autoMergePullRequestsCreatedAfterEligible(prs, createdAfter, verbose) if len(eligiblePRs) == 0 { prAutomergeLog.Print("No eligible PRs found for auto-merge") @@ -77,37 +70,55 @@ func AutoMergePullRequestsCreatedAfter(repoSlug string, createdAfter time.Time, fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d pull request(s) created after workflow start time", len(eligiblePRs)))) for _, pr := range eligiblePRs { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Processing PR #%d: %s (draft: %t, mergeable: %s, created: %s)", - pr.Number, pr.Title, pr.IsDraft, pr.Mergeable, pr.CreatedAt.Format(time.RFC3339)))) - } + autoMergePullRequestsCreatedAfterProcess(repoSlug, pr, verbose) + } - // Convert from draft to non-draft if necessary - if pr.IsDraft { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Converting PR #%d from draft to ready for review", pr.Number))) - if output, err := workflow.RunGHCombined("Converting draft to ready...", "pr", "ready", strconv.Itoa(pr.Number), "--repo", repoSlug); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to convert PR #%d from draft: %v (output: %s)", pr.Number, err, string(output)))) - continue - } - } + return nil +} - // Check if PR is mergeable - if pr.Mergeable != "MERGEABLE" { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("PR #%d is not mergeable (status: %s), skipping auto-merge", pr.Number, pr.Mergeable))) - continue +func autoMergePullRequestsCreatedAfterEligible(prs []PullRequest, createdAfter time.Time, verbose bool) []PullRequest { + var eligiblePRs []PullRequest + for _, pr := range prs { + if pr.CreatedAt.After(createdAfter) { + eligiblePRs = append(eligiblePRs, pr) + } else if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Skipping PR #%d: created at %s (before workflow start time)", pr.Number, pr.CreatedAt.Format(time.RFC3339)))) } + } + return eligiblePRs +} - // Auto-merge the PR - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Auto-merging PR #%d", pr.Number))) - if output, err := workflow.RunGHCombined("Auto-merging pull request...", "pr", "merge", strconv.Itoa(pr.Number), "--repo", repoSlug, "--auto", "--squash"); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to auto-merge PR #%d: %v (output: %s)", pr.Number, err, string(output)))) - continue - } +func autoMergePullRequestsCreatedAfterProcess(repoSlug string, pr PullRequest, verbose bool) { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Processing PR #%d: %s (draft: %t, mergeable: %s, created: %s)", + pr.Number, pr.Title, pr.IsDraft, pr.Mergeable, pr.CreatedAt.Format(time.RFC3339)))) + } + if pr.IsDraft && !autoMergePullRequestsCreatedAfterReady(repoSlug, pr) { + return + } + if pr.Mergeable != "MERGEABLE" { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("PR #%d is not mergeable (status: %s), skipping auto-merge", pr.Number, pr.Mergeable))) + return + } + autoMergePullRequestsCreatedAfterMerge(repoSlug, pr) +} - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Successfully enabled auto-merge for PR #%d", pr.Number))) +func autoMergePullRequestsCreatedAfterReady(repoSlug string, pr PullRequest) bool { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Converting PR #%d from draft to ready for review", pr.Number))) + if output, err := workflow.RunGHCombined("Converting draft to ready...", "pr", "ready", strconv.Itoa(pr.Number), "--repo", repoSlug); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to convert PR #%d from draft: %v (output: %s)", pr.Number, err, string(output)))) + return false } + return true +} - return nil +func autoMergePullRequestsCreatedAfterMerge(repoSlug string, pr PullRequest) { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Auto-merging PR #%d", pr.Number))) + if output, err := workflow.RunGHCombined("Auto-merging pull request...", "pr", "merge", strconv.Itoa(pr.Number), "--repo", repoSlug, "--auto", "--squash"); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to auto-merge PR #%d: %v (output: %s)", pr.Number, err, string(output)))) + return + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Successfully enabled auto-merge for PR #%d", pr.Number))) } // AutoMergePullRequestsLegacy is the legacy function that auto-merges all open PRs (used by trial command for backward compatibility) diff --git a/pkg/cli/pr_command.go b/pkg/cli/pr_command.go index 98cae012438..2ada76b5a96 100644 --- a/pkg/cli/pr_command.go +++ b/pkg/cli/pr_command.go @@ -253,7 +253,9 @@ func createPatchFromPR(sourceOwner, sourceRepo string, prInfo *PRInfo, verbose b } return patchFile, nil -} // applyPatchToRepo applies a patch to the target repository and returns the branch name +} + +// applyPatchToRepo applies a patch to the target repository and returns the branch name func applyPatchToRepo(patchFile string, prInfo *PRInfo, targetOwner, targetRepo string, verbose bool) (string, error) { // Get current branch to restore later currentBranch, err := getCurrentBranch() @@ -261,26 +263,9 @@ func applyPatchToRepo(patchFile string, prInfo *PRInfo, targetOwner, targetRepo return "", fmt.Errorf("failed to get current branch: %w", err) } - // Get the default branch of the target repository - defaultBranchOutput, err := workflow.RunGH("Fetching default branch...", "api", fmt.Sprintf("/repos/%s/%s", targetOwner, targetRepo), "--jq", ".default_branch") + _, err = applyPatchToRepoCheckoutDefault(targetOwner, targetRepo, verbose) if err != nil { - return "", fmt.Errorf("failed to get default branch: %w", err) - } - defaultBranch := strings.TrimSpace(string(defaultBranchOutput)) - - // Ensure we're on the latest version of the default branch - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Checking out and updating %s branch...", defaultBranch))) - } - - cmd := exec.Command("git", "checkout", defaultBranch) - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("failed to checkout default branch %s: %w", defaultBranch, err) - } - - cmd = exec.Command("git", "pull", "origin", defaultBranch) - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("failed to pull latest %s: %w", defaultBranch, err) + return "", err } // Create a new branch for the transfer based on the updated default branch @@ -296,16 +281,7 @@ func applyPatchToRepo(patchFile string, prInfo *PRInfo, targetOwner, targetRepo // Apply the patch if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Applying patch...")) - - // Show some info about the patch file - patchContent, err := os.ReadFile(patchFile) - if err == nil { - lines := strings.Split(string(patchContent), "\n") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Patch file has %d lines", len(lines)))) - if len(lines) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("First line: "+lines[0])) - } - } + applyPatchToRepoVerbosePatchInfo(patchFile) } // Check if patch looks like a mailbox format (starts with "From ") @@ -318,86 +294,127 @@ func applyPatchToRepo(patchFile string, prInfo *PRInfo, targetOwner, targetRepo isMailboxFormat := strings.HasPrefix(string(patchContent), "From ") if isMailboxFormat { - // Try git am for mailbox format patches - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Applying mailbox format patch with git am...")) - } + appliedWithAm = applyPatchToRepoApplyMailbox(patchFile, verbose) + } - cmd = exec.Command("git", "am", patchFile) - if err := cmd.Run(); err == nil { - appliedWithAm = true - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Successfully applied patch with git am")) - } - } else { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("git am failed, trying git apply...")) - } - // Reset any partial am state - _ = exec.Command("git", "am", "--abort").Run() + if !appliedWithAm { + if err := applyPatchToRepoApplyDiff(patchFile, currentBranch, branchName, verbose); err != nil { + return "", err } } + // If we didn't use git am, we need to stage and commit manually if !appliedWithAm { - // Try git apply for standard diff format or as fallback - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Applying patch with git apply...")) + if err := applyPatchToRepoCommit(prInfo); err != nil { + return "", err } + } else if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Applied patch using git am (includes commit)")) + } - cmd = exec.Command("git", "apply", "--3way", patchFile) - if err := cmd.Run(); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("3-way merge failed, trying with whitespace options...")) - } + return branchName, nil +} - // Try with --ignore-space-change and --ignore-whitespace - cmd = exec.Command("git", "apply", "--ignore-space-change", "--ignore-whitespace", patchFile) - if err := cmd.Run(); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Standard apply failed, trying with --reject to see what failed...")) - - // Try with --reject to see which parts fail - rejectCmd := exec.Command("git", "apply", "--reject", patchFile) - rejectOutput, _ := rejectCmd.CombinedOutput() - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Patch rejection details:")) - fmt.Fprintln(os.Stderr, string(rejectOutput)) - } - - // Try to reset back to original branch and clean up - _ = exec.Command("git", "checkout", currentBranch).Run() - _ = exec.Command("git", "branch", "-D", branchName).Run() - return "", fmt.Errorf("failed to apply patch: %w. You may need to resolve conflicts manually", err) - } - } +func applyPatchToRepoCheckoutDefault(targetOwner string, targetRepo string, verbose bool) (string, error) { + defaultBranchOutput, err := workflow.RunGH("Fetching default branch...", "api", fmt.Sprintf("/repos/%s/%s", targetOwner, targetRepo), "--jq", ".default_branch") + if err != nil { + return "", fmt.Errorf("failed to get default branch: %w", err) + } + defaultBranch := strings.TrimSpace(string(defaultBranchOutput)) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Checking out and updating %s branch...", defaultBranch))) + } + cmd := exec.Command("git", "checkout", defaultBranch) + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to checkout default branch %s: %w", defaultBranch, err) + } + cmd = exec.Command("git", "pull", "origin", defaultBranch) + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to pull latest %s: %w", defaultBranch, err) + } + return defaultBranch, nil +} + +func applyPatchToRepoVerbosePatchInfo(patchFile string) { + patchContent, err := os.ReadFile(patchFile) + if err != nil { + return + } + lines := strings.Split(string(patchContent), "\n") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Patch file has %d lines", len(lines)))) + if len(lines) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("First line: "+lines[0])) + } +} +func applyPatchToRepoApplyMailbox(patchFile string, verbose bool) bool { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Applying mailbox format patch with git am...")) + } + cmd := exec.Command("git", "am", patchFile) + if err := cmd.Run(); err == nil { if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Successfully applied patch with git apply")) - } - } // If we didn't use git am, we need to stage and commit manually - if !appliedWithAm { - // Stage all changes - cmd = exec.Command("git", "add", ".") - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("failed to stage changes: %w", err) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Successfully applied patch with git am")) } + return true + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("git am failed, trying git apply...")) + } + _ = exec.Command("git", "am", "--abort").Run() + return false +} - // Create commit with meaningful message - commitMsg := fmt.Sprintf("Transfer PR #%d from %s\n\n%s", prInfo.Number, prInfo.SourceRepo, prInfo.Title) - if prInfo.Body != "" { - commitMsg += "\n\n" + prInfo.Body - } - commitMsg += fmt.Sprintf("\n\nOriginal-PR: %s#%d", prInfo.SourceRepo, prInfo.Number) - commitMsg += "\nOriginal-Author: " + prInfo.AuthorLogin +func applyPatchToRepoApplyDiff(patchFile string, currentBranch string, branchName string, verbose bool) error { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Applying patch with git apply...")) + } + cmd := exec.Command("git", "apply", "--3way", patchFile) + if err := cmd.Run(); err != nil { + return applyPatchToRepoApplyDiffFallback(patchFile, currentBranch, branchName, verbose) + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Successfully applied patch with git apply")) + } + return nil +} - cmd = exec.Command("git", "commit", "-m", commitMsg) - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("failed to commit changes: %w", err) +func applyPatchToRepoApplyDiffFallback(patchFile string, currentBranch string, branchName string, verbose bool) error { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("3-way merge failed, trying with whitespace options...")) + } + cmd := exec.Command("git", "apply", "--ignore-space-change", "--ignore-whitespace", patchFile) + if err := cmd.Run(); err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Standard apply failed, trying with --reject to see what failed...")) + rejectCmd := exec.Command("git", "apply", "--reject", patchFile) + rejectOutput, _ := rejectCmd.CombinedOutput() + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Patch rejection details:")) + fmt.Fprintln(os.Stderr, string(rejectOutput)) } - } else if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Applied patch using git am (includes commit)")) + _ = exec.Command("git", "checkout", currentBranch).Run() + _ = exec.Command("git", "branch", "-D", branchName).Run() + return fmt.Errorf("failed to apply patch: %w. You may need to resolve conflicts manually", err) } + return nil +} - return branchName, nil +func applyPatchToRepoCommit(prInfo *PRInfo) error { + cmd := exec.Command("git", "add", ".") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to stage changes: %w", err) + } + commitMsg := fmt.Sprintf("Transfer PR #%d from %s\n\n%s", prInfo.Number, prInfo.SourceRepo, prInfo.Title) + if prInfo.Body != "" { + commitMsg += "\n\n" + prInfo.Body + } + commitMsg += fmt.Sprintf("\n\nOriginal-PR: %s#%d", prInfo.SourceRepo, prInfo.Number) + commitMsg += "\nOriginal-Author: " + prInfo.AuthorLogin + cmd = exec.Command("git", "commit", "-m", commitMsg) + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to commit changes: %w", err) + } + return nil } // createTransferPR creates a new PR in the target repository @@ -412,66 +429,107 @@ func createTransferPR(targetOwner, targetRepo string, prInfo *PRInfo, branchName var needsFork bool if !hasWriteAccess { - needsFork = true - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No write access to target repository, using fork workflow...")) - } - - forkOwner, forkRepo, err = createForkIfNeeded(targetOwner, targetRepo, verbose) + forkOwner, forkRepo, err = createTransferPRFork(targetOwner, targetRepo, verbose) if err != nil { - return fmt.Errorf("failed to create fork: %w", err) + return err } + needsFork = true + } - // Add fork as remote if not already present - remoteName := "fork" - githubHost := getGitHubHost() - forkRepoURL := fmt.Sprintf("%s/%s/%s.git", githubHost, forkOwner, forkRepo) - - // Check if fork remote exists - checkRemoteCmd := exec.Command("git", "remote", "get-url", remoteName) - if checkRemoteCmd.Run() != nil { - // Remote doesn't exist, add it - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Adding fork remote: "+forkRepoURL)) - } - addRemoteCmd := exec.Command("git", "remote", "add", remoteName, forkRepoURL) - if err := addRemoteCmd.Run(); err != nil { - return fmt.Errorf("failed to add fork remote: %w", err) - } - } + // Push the branch + if err := createTransferPRPush(branchName, needsFork, verbose); err != nil { + return err + } - // Also ensure target repository is set as upstream remote if not already present - upstreamRemote := "upstream" - targetRepoURL := fmt.Sprintf("https://github.com/%s/%s.git", targetOwner, targetRepo) - - // Check if upstream remote exists and points to the right repo - checkUpstreamCmd := exec.Command("git", "remote", "get-url", upstreamRemote) - upstreamOutput, err := checkUpstreamCmd.Output() - if err != nil || strings.TrimSpace(string(upstreamOutput)) != targetRepoURL { - // Upstream doesn't exist or points to wrong repo, add/update it - if err != nil { - // Remote doesn't exist, add it - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Adding upstream remote: "+targetRepoURL)) - } - addUpstreamCmd := exec.Command("git", "remote", "add", upstreamRemote, targetRepoURL) - if err := addUpstreamCmd.Run(); err != nil { - return fmt.Errorf("failed to add upstream remote: %w", err) - } - } else { - // Remote exists but points to wrong repo, update it - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updating upstream remote: "+targetRepoURL)) - } - setUpstreamCmd := exec.Command("git", "remote", "set-url", upstreamRemote, targetRepoURL) - if err := setUpstreamCmd.Run(); err != nil { - return fmt.Errorf("failed to update upstream remote: %w", err) - } - } + // Create PR body with original info + prBody := createTransferPRBody(prInfo) + + // Create the PR + repoFlag := fmt.Sprintf("%s/%s", targetOwner, targetRepo) + headRef := createTransferPRHeadRef(forkOwner, branchName, needsFork) + + output, err := workflow.RunGH("Creating pull request...", "pr", "create", + "--repo", repoFlag, + "--title", prInfo.Title, + "--body", prBody, + "--head", headRef) + if err != nil { + return fmt.Errorf("failed to create PR: %w", err) + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("PR created successfully!")) + if needsFork { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("PR created from fork %s/%s to %s/%s", forkOwner, forkRepo, targetOwner, targetRepo))) + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("URL: "+strings.TrimSpace(string(output)))) + + return nil +} + +func createTransferPRFork(targetOwner string, targetRepo string, verbose bool) (string, string, error) { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No write access to target repository, using fork workflow...")) + } + forkOwner, forkRepo, err := createForkIfNeeded(targetOwner, targetRepo, verbose) + if err != nil { + return "", "", fmt.Errorf("failed to create fork: %w", err) + } + if err := createTransferPRForkRemote(forkOwner, forkRepo, verbose); err != nil { + return "", "", err + } + if err := createTransferPRUpstreamRemote(targetOwner, targetRepo, verbose); err != nil { + return "", "", err + } + return forkOwner, forkRepo, nil +} + +func createTransferPRForkRemote(forkOwner string, forkRepo string, verbose bool) error { + remoteName := "fork" + githubHost := getGitHubHost() + forkRepoURL := fmt.Sprintf("%s/%s/%s.git", githubHost, forkOwner, forkRepo) + checkRemoteCmd := exec.Command("git", "remote", "get-url", remoteName) + if checkRemoteCmd.Run() == nil { + return nil + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Adding fork remote: "+forkRepoURL)) + } + addRemoteCmd := exec.Command("git", "remote", "add", remoteName, forkRepoURL) + if err := addRemoteCmd.Run(); err != nil { + return fmt.Errorf("failed to add fork remote: %w", err) + } + return nil +} + +func createTransferPRUpstreamRemote(targetOwner string, targetRepo string, verbose bool) error { + upstreamRemote := "upstream" + targetRepoURL := fmt.Sprintf("https://github.com/%s/%s.git", targetOwner, targetRepo) + checkUpstreamCmd := exec.Command("git", "remote", "get-url", upstreamRemote) + upstreamOutput, err := checkUpstreamCmd.Output() + if err == nil && strings.TrimSpace(string(upstreamOutput)) == targetRepoURL { + return nil + } + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Adding upstream remote: "+targetRepoURL)) + } + addUpstreamCmd := exec.Command("git", "remote", "add", upstreamRemote, targetRepoURL) + if err := addUpstreamCmd.Run(); err != nil { + return fmt.Errorf("failed to add upstream remote: %w", err) } + return nil + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updating upstream remote: "+targetRepoURL)) + } + setUpstreamCmd := exec.Command("git", "remote", "set-url", upstreamRemote, targetRepoURL) + if err := setUpstreamCmd.Run(); err != nil { + return fmt.Errorf("failed to update upstream remote: %w", err) } + return nil +} - // Push the branch +func createTransferPRPush(branchName string, needsFork bool, verbose bool) error { if verbose { if needsFork { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Pushing branch to fork...")) @@ -479,54 +537,35 @@ func createTransferPR(targetOwner, targetRepo string, prInfo *PRInfo, branchName fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Pushing branch to remote...")) } } - - var pushCmd *exec.Cmd + remote := "origin" if needsFork { - pushCmd = exec.Command("git", "push", "-u", "fork", branchName) - } else { - pushCmd = exec.Command("git", "push", "-u", "origin", branchName) + remote = "fork" } - + pushCmd := exec.Command("git", "push", "-u", remote, branchName) if err := pushCmd.Run(); err != nil { if needsFork { return fmt.Errorf("failed to push branch to fork: %w", err) } return fmt.Errorf("failed to push branch: %w", err) } + return nil +} - // Create PR body with original info +func createTransferPRBody(prInfo *PRInfo) string { prBody := prInfo.Body if prBody != "" { prBody += "\n\n---\n\n" } prBody += fmt.Sprintf("**Transferred from:** %s#%d\n", prInfo.SourceRepo, prInfo.Number) prBody += "**Original Author:** @" + prInfo.AuthorLogin + return prBody +} - // Create the PR - repoFlag := fmt.Sprintf("%s/%s", targetOwner, targetRepo) - var headRef string - if needsFork { - headRef = fmt.Sprintf("%s:%s", forkOwner, branchName) - } else { - headRef = branchName - } - - output, err := workflow.RunGH("Creating pull request...", "pr", "create", - "--repo", repoFlag, - "--title", prInfo.Title, - "--body", prBody, - "--head", headRef) - if err != nil { - return fmt.Errorf("failed to create PR: %w", err) - } - - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("PR created successfully!")) +func createTransferPRHeadRef(forkOwner string, branchName string, needsFork bool) string { if needsFork { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("PR created from fork %s/%s to %s/%s", forkOwner, forkRepo, targetOwner, targetRepo))) + return fmt.Sprintf("%s:%s", forkOwner, branchName) } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("URL: "+strings.TrimSpace(string(output)))) - - return nil + return branchName } // transferPR is the main function that orchestrates the PR transfer @@ -549,28 +588,9 @@ func transferPR(prURL, targetRepo string, verbose bool) error { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Source: %s/%s PR #%d", sourceOwner, sourceRepoName, prNumber))) } - // Determine target repository - var targetOwner, targetRepoName string - if targetRepo != "" { - repoSpec, err := parseRepoSpec(targetRepo) - if err != nil { - return fmt.Errorf("invalid target repository format: %w", err) - } - parts := strings.SplitN(repoSpec.RepoSlug, "/", 2) - if len(parts) != 2 { - return errors.New("invalid target repository format, expected: owner/repo") - } - targetOwner, targetRepoName = parts[0], parts[1] - } else { - // Use current repository as target - slug, err := GetCurrentRepoSlug() - if err != nil { - return fmt.Errorf("failed to determine target repository: %w", err) - } - targetOwner, targetRepoName, err = repoutil.SplitRepoSlug(slug) - if err != nil { - return fmt.Errorf("failed to parse target repository: %w", err) - } + targetOwner, targetRepoName, err := transferPRTarget(targetRepo) + if err != nil { + return err } prLog.Printf("Determined target repository: %s/%s", targetOwner, targetRepoName) @@ -586,129 +606,94 @@ func transferPR(prURL, targetRepo string, verbose bool) error { } // Ensure we're in the correct git repository - var workingDir string - var needsCleanup bool - - if targetRepo != "" { - // Check if we're already in the target repository - if isGitRepo() { - slug, err := GetCurrentRepoSlug() - if err == nil { - currentOwner, currentRepoName, err := repoutil.SplitRepoSlug(slug) - if err == nil && currentOwner == targetOwner && currentRepoName == targetRepoName { - // We're already in the target repo - workingDir = "." - } else { - // We need to clone the target repository - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Cloning target repository %s/%s...", targetOwner, targetRepoName))) - } - tempDir, err := os.MkdirTemp("", "gh-aw-pr-transfer-repo-") - if err != nil { - return fmt.Errorf("failed to create temp directory for repo: %w", err) - } - - cloneCmd := workflow.ExecGH("repo", "clone", fmt.Sprintf("%s/%s", targetOwner, targetRepoName), tempDir) - if err := cloneCmd.Run(); err != nil { - // Clean up temporary directory on error - if rmErr := os.RemoveAll(tempDir); rmErr != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("failed to clean up temporary directory %s: %v", tempDir, rmErr))) - } - return fmt.Errorf("failed to clone target repository: %w", err) - } - - workingDir = tempDir - needsCleanup = true - - // Change to the cloned repository directory - if err := os.Chdir(tempDir); err != nil { - // Clean up temporary directory on error - if rmErr := os.RemoveAll(tempDir); rmErr != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("failed to clean up temporary directory %s: %v", tempDir, rmErr))) - } - return fmt.Errorf("failed to change to cloned repository directory: %w", err) - } - } - } else { - // Error getting current repo, clone anyway - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Cloning target repository %s/%s...", targetOwner, targetRepoName))) - } - tempDir, err := os.MkdirTemp("", "gh-aw-pr-transfer-repo-") - if err != nil { - return fmt.Errorf("failed to create temp directory for repo: %w", err) - } - - cloneCmd := workflow.ExecGH("repo", "clone", fmt.Sprintf("%s/%s", targetOwner, targetRepoName), tempDir) - if err := cloneCmd.Run(); err != nil { - // Clean up temporary directory on error - if rmErr := os.RemoveAll(tempDir); rmErr != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("failed to clean up temporary directory %s: %v", tempDir, rmErr))) - } - return fmt.Errorf("failed to clone target repository: %w", err) - } - - workingDir = tempDir - needsCleanup = true - - // Change to the cloned repository directory - if err := os.Chdir(tempDir); err != nil { - // Clean up temporary directory on error - if rmErr := os.RemoveAll(tempDir); rmErr != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("failed to clean up temporary directory %s: %v", tempDir, rmErr))) - } - return fmt.Errorf("failed to change to cloned repository directory: %w", err) - } - } - } else { - // We're not in a git repository and need to clone - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Cloning target repository %s/%s...", targetOwner, targetRepoName))) - } - tempDir, err := os.MkdirTemp("", "gh-aw-pr-transfer-repo-") - if err != nil { - return fmt.Errorf("failed to create temp directory for repo: %w", err) - } + workingDir, needsCleanup, err := transferPRWorkingDir(targetRepo, targetOwner, targetRepoName, verbose) + if err != nil { + return err + } - cloneCmd := workflow.ExecGH("repo", "clone", fmt.Sprintf("%s/%s", targetOwner, targetRepoName), tempDir) - if err := cloneCmd.Run(); err != nil { - // Clean up temporary directory on error - if rmErr := os.RemoveAll(tempDir); rmErr != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("failed to clean up temporary directory %s: %v", tempDir, rmErr))) - } - return fmt.Errorf("failed to clone target repository: %w", err) - } + // Cleanup function + defer func() { + transferPRCleanup(workingDir, needsCleanup, verbose) + }() - workingDir = tempDir - needsCleanup = true + return transferPRCreate(targetOwner, targetRepoName, sourceOwner, sourceRepoName, prNumber, verbose) +} - // Change to the cloned repository directory - if err := os.Chdir(tempDir); err != nil { - // Clean up temporary directory on error - if rmErr := os.RemoveAll(tempDir); rmErr != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("failed to clean up temporary directory %s: %v", tempDir, rmErr))) - } - return fmt.Errorf("failed to change to cloned repository directory: %w", err) - } +func transferPRTarget(targetRepo string) (string, string, error) { + if targetRepo != "" { + repoSpec, err := parseRepoSpec(targetRepo) + if err != nil { + return "", "", fmt.Errorf("invalid target repository format: %w", err) } - } else { - // Using current repository as target - if !isGitRepo() { - return errors.New("not in a git repository") + parts := strings.SplitN(repoSpec.RepoSlug, "/", 2) + if len(parts) != 2 { + return "", "", errors.New("invalid target repository format, expected: owner/repo") } - workingDir = "." + return parts[0], parts[1], nil + } + slug, err := GetCurrentRepoSlug() + if err != nil { + return "", "", fmt.Errorf("failed to determine target repository: %w", err) + } + owner, repo, err := repoutil.SplitRepoSlug(slug) + if err != nil { + return "", "", fmt.Errorf("failed to parse target repository: %w", err) } + return owner, repo, nil +} - // Cleanup function - defer func() { - if needsCleanup && workingDir != "" { - // Clean up temporary directory when done - if err := os.RemoveAll(workingDir); err != nil && verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("failed to clean up temporary directory %s: %v", workingDir, err))) +func transferPRWorkingDir(targetRepo string, targetOwner string, targetRepoName string, verbose bool) (string, bool, error) { + if targetRepo == "" { + if !isGitRepo() { + return "", false, errors.New("not in a git repository") + } + return ".", false, nil + } + if isGitRepo() { + slug, err := GetCurrentRepoSlug() + if err == nil { + currentOwner, currentRepoName, err := repoutil.SplitRepoSlug(slug) + if err == nil && currentOwner == targetOwner && currentRepoName == targetRepoName { + return ".", false, nil } } - }() + } + tempDir, err := transferPRCloneTarget(targetOwner, targetRepoName, verbose) + if err != nil { + return "", false, err + } + return tempDir, true, nil +} + +func transferPRCloneTarget(targetOwner string, targetRepoName string, verbose bool) (string, error) { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Cloning target repository %s/%s...", targetOwner, targetRepoName))) + } + tempDir, err := os.MkdirTemp("", "gh-aw-pr-transfer-repo-") + if err != nil { + return "", fmt.Errorf("failed to create temp directory for repo: %w", err) + } + cloneCmd := workflow.ExecGH("repo", "clone", fmt.Sprintf("%s/%s", targetOwner, targetRepoName), tempDir) + if err := cloneCmd.Run(); err != nil { + transferPRCleanup(tempDir, true, verbose) + return "", fmt.Errorf("failed to clone target repository: %w", err) + } + if err := os.Chdir(tempDir); err != nil { + transferPRCleanup(tempDir, true, verbose) + return "", fmt.Errorf("failed to change to cloned repository directory: %w", err) + } + return tempDir, nil +} + +func transferPRCleanup(workingDir string, needsCleanup bool, verbose bool) { + if needsCleanup && workingDir != "" { + if err := os.RemoveAll(workingDir); err != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("failed to clean up temporary directory %s: %v", workingDir, err))) + } + } +} +func transferPRCreate(targetOwner string, targetRepoName string, sourceOwner string, sourceRepoName string, prNumber int, verbose bool) error { // Fetch PR information if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Fetching PR details...")) diff --git a/pkg/cli/preconditions.go b/pkg/cli/preconditions.go index fd757f9b32e..c91edf97e8f 100644 --- a/pkg/cli/preconditions.go +++ b/pkg/cli/preconditions.go @@ -74,8 +74,11 @@ func checkActionsEnabledShared(repoSlug string, verbose bool) error { return nil } - // Check allowed actions setting - switch permissions.AllowedActions { + return checkActionsEnabledSharedAllowed(permissions.AllowedActions, permissions.SelectedActionsURL, verbose) +} + +func checkActionsEnabledSharedAllowed(allowedActions string, selectedActionsURL string, verbose bool) error { + switch allowedActions { case "all": // All actions allowed - good to go if verbose { @@ -95,7 +98,7 @@ func checkActionsEnabledShared(repoSlug string, verbose bool) error { return errors.New("repository action permissions prevent agentic workflows from running") case "selected": // Selected actions - need to check if GitHub-owned actions are allowed - if err := checkSelectedActionsPermissions(permissions.SelectedActionsURL, verbose); err != nil { + if err := checkSelectedActionsPermissions(selectedActionsURL, verbose); err != nil { return err } default: diff --git a/pkg/cli/project_command.go b/pkg/cli/project_command.go index 72413ed93d1..33aa273012b 100644 --- a/pkg/cli/project_command.go +++ b/pkg/cli/project_command.go @@ -114,99 +114,114 @@ Project Setup: func RunProjectNew(ctx context.Context, config ProjectConfig) error { projectLog.Printf("Creating project: title=%s, owner=%s, repo=%s", config.Title, config.Owner, config.Repo) - // Resolve owner type - ownerType := "org" - ownerLogin := config.Owner - if config.Owner == "@me" { - ownerType = "user" - // Get current user - currentUser, err := getCurrentUser(ctx) - if err != nil { - return fmt.Errorf("failed to get current user: %w", err) - } - ownerLogin = currentUser - console.LogVerbose(config.Verbose, "Resolved @me to user: "+ownerLogin) + if err := runProjectNewResolveOwner(ctx, &config); err != nil { + return err } - - config.OwnerType = ownerType - config.Owner = ownerLogin - - // Validate owner exists if err := validateOwner(ctx, config.OwnerType, config.Owner, config.Verbose); err != nil { return fmt.Errorf("owner validation failed: %w", err) } - // Get owner ID ownerId, err := getOwnerNodeId(ctx, config.OwnerType, config.Owner, config.Verbose) if err != nil { return fmt.Errorf("failed to get owner ID: %w", err) } - // Create project fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Creating project '%s' for %s %s...", config.Title, config.OwnerType, config.Owner))) - project, err := createProject(ctx, ownerId, config.Title, config.Verbose) if err != nil { return fmt.Errorf("failed to create project: %w", err) } + projectID, err := runProjectNewProjectID(project) + if err != nil { + return err + } + runProjectNewLinkRepository(ctx, config, projectID) + + projectURL, projectNumber, err := runProjectNewProjectDetails(project) + if err != nil { + return err + } + runProjectNewSetupProject(ctx, config, projectURL, projectNumber) + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Created project #%v: %s", project["number"], config.Title))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf(" URL: %s", project["url"]))) + return nil +} + +func runProjectNewResolveOwner(ctx context.Context, config *ProjectConfig) error { + ownerType := "org" + ownerLogin := config.Owner + if config.Owner == "@me" { + ownerType = "user" + currentUser, err := getCurrentUser(ctx) + if err != nil { + return fmt.Errorf("failed to get current user: %w", err) + } + ownerLogin = currentUser + console.LogVerbose(config.Verbose, "Resolved @me to user: "+ownerLogin) + } + config.OwnerType = ownerType + config.Owner = ownerLogin + return nil +} + +func runProjectNewProjectID(project map[string]any) (string, error) { projectID, ok := project["id"].(string) if !ok || projectID == "" { - return errors.New("failed to get project ID from response") + return "", errors.New("failed to get project ID from response") } + return projectID, nil +} - // Link to repository if specified - if config.Repo != "" { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Linking project to repository %s...", config.Repo))) - if err := linkProjectToRepo(ctx, projectID, config.Repo, config.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to link project to repository: %v", err))) - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Project linked to repository")) - } +func runProjectNewLinkRepository(ctx context.Context, config ProjectConfig, projectID string) { + if config.Repo == "" { + return } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Linking project to repository %s...", config.Repo))) + if err := linkProjectToRepo(ctx, projectID, config.Repo, config.Verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to link project to repository: %v", err))) + } else { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Project linked to repository")) + } +} - // Create views and fields if requested +func runProjectNewProjectDetails(project map[string]any) (string, int, error) { projectURL, ok := project["url"].(string) if !ok || projectURL == "" { - return errors.New("failed to get project URL from response") + return "", 0, errors.New("failed to get project URL from response") } - projectNumberFloat, ok := project["number"].(float64) if !ok || projectNumberFloat <= 0 { - return errors.New("failed to get valid project number from response") + return "", 0, errors.New("failed to get valid project number from response") } - projectNumber := int(projectNumberFloat) - - if config.WithProjectSetup { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Creating standard project views...")) - if err := createStandardViews(ctx, projectURL, config.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to create views: %v", err))) - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Created standard views")) - } + return projectURL, int(projectNumberFloat), nil +} - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Creating custom fields...")) - if err := createStandardFields(ctx, projectURL, projectNumber, config.Owner, config.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to create fields: %v", err))) - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Created custom fields")) - } +func runProjectNewSetupProject(ctx context.Context, config ProjectConfig, projectURL string, projectNumber int) { + if !config.WithProjectSetup { + return } - - if config.WithProjectSetup { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Enhancing Status field...")) - if err := ensureStatusOption(ctx, projectURL, "Review Required", config.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to update Status field: %v", err))) - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Added 'Review Required' status option")) - } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Creating standard project views...")) + if err := createStandardViews(ctx, projectURL, config.Verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to create views: %v", err))) + } else { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Created standard views")) } - // Output success - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Created project #%v: %s", project["number"], config.Title))) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf(" URL: %s", project["url"]))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Creating custom fields...")) + if err := createStandardFields(ctx, projectURL, projectNumber, config.Owner, config.Verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to create fields: %v", err))) + } else { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Created custom fields")) + } - return nil + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Enhancing Status field...")) + if err := ensureStatusOption(ctx, projectURL, "Review Required", config.Verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to update Status field: %v", err))) + } else { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Added 'Review Required' status option")) + } } // getCurrentUser gets the current authenticated user's login @@ -325,29 +340,31 @@ func createProject(ctx context.Context, ownerId, title string, verbose bool) (ma return nil, fmt.Errorf("GraphQL mutation failed: %w", err) } - // Parse response + project, err := createProjectParseResponse(output) + if err != nil { + return nil, err + } + console.LogVerbose(verbose, fmt.Sprintf("✓ Project created: #%v", project["number"])) + return project, nil +} + +func createProjectParseResponse(output []byte) (map[string]any, error) { var response map[string]any if err := json.Unmarshal(output, &response); err != nil { return nil, fmt.Errorf("failed to parse GraphQL response: %w", err) } - - // Extract project data data, ok := response["data"].(map[string]any) if !ok { return nil, errors.New("invalid response: missing 'data' field") } - createResult, ok := data["createProjectV2"].(map[string]any) if !ok { return nil, errors.New("invalid response: missing 'createProjectV2' field") } - project, ok := createResult["projectV2"].(map[string]any) if !ok { return nil, errors.New("invalid response: missing 'projectV2' field") } - - console.LogVerbose(verbose, fmt.Sprintf("✓ Project created: #%v", project["number"])) return project, nil } @@ -356,38 +373,11 @@ func linkProjectToRepo(ctx context.Context, projectId, repoSlug string, verbose projectLog.Printf("Linking project %s to repository %s", projectId, repoSlug) console.LogVerbose(verbose, "Linking project to repository: "+repoSlug) - // Parse repo slug - parts := strings.Split(repoSlug, "/") - if len(parts) != 2 { - return fmt.Errorf("invalid repository format. Expected 'owner/repo', got '%s'", repoSlug) - } - repoOwner := parts[0] - repoName := parts[1] - - // Get repository ID - repoIdQuery := `query($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { id } }` - repoIdBody := map[string]any{ - "query": repoIdQuery, - "variables": map[string]any{ - "owner": repoOwner, - "name": repoName, - }, - } - repoIdJSON, err := json.Marshal(repoIdBody) - if err != nil { - return fmt.Errorf("failed to marshal repository ID GraphQL request: %w", err) - } - output, err := workflow.RunGHInputContext(ctx, "Getting repository ID...", bytes.NewReader(repoIdJSON), "api", "graphql", "--input", "-", "--jq", ".data.repository.id") + repoId, err := linkProjectToRepoRepositoryID(ctx, repoSlug) if err != nil { - return fmt.Errorf("repository '%s' not found: %w", repoSlug, err) + return err } - repoId := strings.TrimSpace(string(output)) - if repoId == "" { - return errors.New("failed to get repository ID") - } - - // Link project to repository mutation := `mutation($projectId: ID!, $repositoryId: ID!) { linkProjectV2ToRepository(input: { projectId: $projectId, repositoryId: $repositoryId }) { repository { @@ -395,7 +385,6 @@ func linkProjectToRepo(ctx context.Context, projectId, repoSlug string, verbose } } }` - mutationBody := map[string]any{ "query": mutation, "variables": map[string]any{ @@ -407,7 +396,6 @@ func linkProjectToRepo(ctx context.Context, projectId, repoSlug string, verbose if err != nil { return fmt.Errorf("failed to marshal GraphQL request: %w", err) } - _, err = workflow.RunGHInputContext(ctx, "Linking project to repository...", bytes.NewReader(mutationJSON), "api", "graphql", "--input", "-") if err != nil { return fmt.Errorf("failed to link project to repository: %w", err) @@ -417,6 +405,34 @@ func linkProjectToRepo(ctx context.Context, projectId, repoSlug string, verbose return nil } +func linkProjectToRepoRepositoryID(ctx context.Context, repoSlug string) (string, error) { + parts := strings.Split(repoSlug, "/") + if len(parts) != 2 { + return "", fmt.Errorf("invalid repository format. Expected 'owner/repo', got '%s'", repoSlug) + } + repoIdQuery := `query($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { id } }` + repoIdBody := map[string]any{ + "query": repoIdQuery, + "variables": map[string]any{ + "owner": parts[0], + "name": parts[1], + }, + } + repoIdJSON, err := json.Marshal(repoIdBody) + if err != nil { + return "", fmt.Errorf("failed to marshal repository ID GraphQL request: %w", err) + } + output, err := workflow.RunGHInputContext(ctx, "Getting repository ID...", bytes.NewReader(repoIdJSON), "api", "graphql", "--input", "-", "--jq", ".data.repository.id") + if err != nil { + return "", fmt.Errorf("repository '%s' not found: %w", repoSlug, err) + } + repoId := strings.TrimSpace(string(output)) + if repoId == "" { + return "", errors.New("failed to get repository ID") + } + return repoId, nil +} + // escapeGraphQLString escapes special characters in GraphQL strings func escapeGraphQLString(s string) string { s = strings.ReplaceAll(s, "\\", "\\\\") @@ -640,104 +656,95 @@ type statusFieldInfo struct { // getStatusField retrieves the Status field information for a project func getStatusField(ctx context.Context, info projectURLInfo, verbose bool) (statusFieldInfo, error) { - var query string - var jqProjectID, jqFields string - + query, jqProjectID, jqFields := getStatusFieldQuery(info) projectNumberArg := "number=" + strconv.Itoa(info.projectNumber) + + projectID, fieldsOutput, err := getStatusFieldRaw(query, jqProjectID, jqFields, info.ownerLogin, projectNumberArg) + if err != nil { + return statusFieldInfo{}, err + } + + var fields []map[string]any + if err := json.Unmarshal(fieldsOutput, &fields); err != nil { + return statusFieldInfo{}, fmt.Errorf("failed to parse fields: %w", err) + } + return getStatusFieldFind(projectID, fields) +} + +func getStatusFieldQuery(info projectURLInfo) (string, string, string) { if info.scope == "orgs" { - query = `query($login: String!, $number: Int!) { + return `query($login: String!, $number: Int!) { organization(login: $login) { projectV2(number: $number) { id fields(first: 100) { - nodes { - ... on ProjectV2SingleSelectField { - id - name - options { name color description } - } - } + nodes { ... on ProjectV2SingleSelectField { id name options { name color description } } } } } } - }` - jqProjectID = ".data.organization.projectV2.id" - jqFields = ".data.organization.projectV2.fields.nodes" - } else { - query = `query($login: String!, $number: Int!) { - user(login: $login) { - projectV2(number: $number) { - id - fields(first: 100) { - nodes { - ... on ProjectV2SingleSelectField { - id - name - options { name color description } - } - } - } + }`, ".data.organization.projectV2.id", ".data.organization.projectV2.fields.nodes" + } + return `query($login: String!, $number: Int!) { + user(login: $login) { + projectV2(number: $number) { + id + fields(first: 100) { + nodes { ... on ProjectV2SingleSelectField { id name options { name color description } } } } } - }` - jqProjectID = ".data.user.projectV2.id" - jqFields = ".data.user.projectV2.fields.nodes" - } + } + }`, ".data.user.projectV2.id", ".data.user.projectV2.fields.nodes" +} - // Get project ID - projectIDOutput, err := projectCommandRunGH("Getting project info...", "api", "graphql", "-f", "query="+query, "-f", "login="+info.ownerLogin, "-F", projectNumberArg, "--jq", jqProjectID) +func getStatusFieldRaw(query, jqProjectID, jqFields, ownerLogin, projectNumberArg string) (string, []byte, error) { + projectIDOutput, err := projectCommandRunGH("Getting project info...", "api", "graphql", "-f", "query="+query, "-f", "login="+ownerLogin, "-F", projectNumberArg, "--jq", jqProjectID) if err != nil { - return statusFieldInfo{}, fmt.Errorf("failed to get project ID: %w", err) + return "", nil, fmt.Errorf("failed to get project ID: %w", err) } projectID := strings.TrimSpace(string(projectIDOutput)) - // Get fields - fieldsOutput, err := projectCommandRunGH("Getting project fields...", "api", "graphql", "-f", "query="+query, "-f", "login="+info.ownerLogin, "-F", projectNumberArg, "--jq", jqFields) + fieldsOutput, err := projectCommandRunGH("Getting project fields...", "api", "graphql", "-f", "query="+query, "-f", "login="+ownerLogin, "-F", projectNumberArg, "--jq", jqFields) if err != nil { - return statusFieldInfo{}, fmt.Errorf("failed to get project fields: %w", err) - } - - // Parse fields to find Status field - var fields []map[string]any - if err := json.Unmarshal(fieldsOutput, &fields); err != nil { - return statusFieldInfo{}, fmt.Errorf("failed to parse fields: %w", err) + return "", nil, fmt.Errorf("failed to get project fields: %w", err) } + return projectID, fieldsOutput, nil +} +func getStatusFieldFind(projectID string, fields []map[string]any) (statusFieldInfo, error) { for _, field := range fields { if fieldName, ok := field["name"].(string); ok && fieldName == "Status" { fieldID, _ := field["id"].(string) if fieldID == "" { continue } - - // Parse options - var options []singleSelectOption - if optionsData, ok := field["options"].([]any); ok { - for _, optData := range optionsData { - if optMap, ok := optData.(map[string]any); ok { - opt := singleSelectOption{ - Name: getString(optMap, "name"), - Color: getString(optMap, "color"), - } - if desc := getString(optMap, "description"); desc != "" { - opt.Description = desc - } - options = append(options, opt) - } - } - } - return statusFieldInfo{ projectID: projectID, fieldID: fieldID, - options: options, + options: getStatusFieldOptions(field), }, nil } } - return statusFieldInfo{}, errors.New("status field not found in project") } +func getStatusFieldOptions(field map[string]any) []singleSelectOption { + var options []singleSelectOption + optionsData, ok := field["options"].([]any) + if !ok { + return options + } + for _, optData := range optionsData { + if optMap, ok := optData.(map[string]any); ok { + opt := singleSelectOption{Name: getString(optMap, "name"), Color: getString(optMap, "color")} + if desc := getString(optMap, "description"); desc != "" { + opt.Description = desc + } + options = append(options, opt) + } + } + return options +} + // getString safely extracts a string value from a map func getString(m map[string]any, key string) string { if val, ok := m[key].(string); ok { diff --git a/pkg/cli/redacted_domains.go b/pkg/cli/redacted_domains.go index f03fbe35425..9cfbd5f45a2 100644 --- a/pkg/cli/redacted_domains.go +++ b/pkg/cli/redacted_domains.go @@ -81,43 +81,48 @@ func parseRedactedDomainsLog(logPath string, verbose bool) (*RedactedDomainsAnal func analyzeRedactedDomains(runDir string, verbose bool) (*RedactedDomainsAnalysis, error) { redactedDomainsLog.Printf("Analyzing redacted domains in: %s", runDir) - // The file could be in several locations depending on artifact extraction: - // 1. Directly in the run directory as "redacted-urls.log" - // 2. In an "agent_outputs" subdirectory as "redacted-urls.log" - // 3. Following the original path structure: agent_outputs/tmp/gh-aw/redacted-urls.log - - // Check for redacted-urls.log directly in the run directory - directPath := filepath.Join(runDir, "redacted-urls.log") - if _, err := os.Stat(directPath); err == nil { - redactedDomainsLog.Printf("Found redacted-urls.log at direct path: %s", directPath) + if path, message := analyzeRedactedDomainsKnownPath(runDir); path != "" { + redactedDomainsLog.Printf("Found redacted-urls.log at %s: %s", message, path) if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found redacted-urls.log in run directory")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(message)) } - return parseRedactedDomainsLog(directPath, verbose) + return parseRedactedDomainsLog(path, verbose) } - // Check for redacted-urls.log in agent_outputs directory - agentOutputsPath := filepath.Join(runDir, "agent_outputs", "redacted-urls.log") - if _, err := os.Stat(agentOutputsPath); err == nil { - redactedDomainsLog.Printf("Found redacted-urls.log in agent_outputs: %s", agentOutputsPath) + foundPath := analyzeRedactedDomainsWalk(runDir) + if foundPath != "" { + redactedDomainsLog.Printf("Found redacted-urls.log via recursive search: %s", foundPath) if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found redacted-urls.log in agent_outputs directory")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found redacted-urls.log at "+foundPath)) } - return parseRedactedDomainsLog(agentOutputsPath, verbose) + return parseRedactedDomainsLog(foundPath, verbose) } - // Check for the full path structure that mirrors the upload path - // agent_outputs/tmp/gh-aw/redacted-urls.log - fullPath := filepath.Join(runDir, "agent_outputs", "tmp", "gh-aw", "redacted-urls.log") - if _, err := os.Stat(fullPath); err == nil { - redactedDomainsLog.Printf("Found redacted-urls.log at full path: %s", fullPath) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found redacted-urls.log at full artifact path")) + redactedDomainsLog.Print("No redacted-urls.log found") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No redacted-urls.log found in "+runDir)) + } + return nil, nil +} + +func analyzeRedactedDomainsKnownPath(runDir string) (string, string) { + candidates := []struct { + path string + message string + }{ + {filepath.Join(runDir, "redacted-urls.log"), "Found redacted-urls.log in run directory"}, + {filepath.Join(runDir, "agent_outputs", "redacted-urls.log"), "Found redacted-urls.log in agent_outputs directory"}, + {filepath.Join(runDir, "agent_outputs", "tmp", "gh-aw", "redacted-urls.log"), "Found redacted-urls.log at full artifact path"}, + } + for _, candidate := range candidates { + if _, err := os.Stat(candidate.path); err == nil { + return candidate.path, candidate.message } - return parseRedactedDomainsLog(fullPath, verbose) } + return "", "" +} - // Fallback: search recursively for redacted-urls.log +func analyzeRedactedDomainsWalk(runDir string) string { var foundPath string if walkErr := filepath.Walk(runDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -135,19 +140,5 @@ func analyzeRedactedDomains(runDir string, verbose bool) (*RedactedDomainsAnalys }); walkErr != nil && !errors.Is(walkErr, errWalkStop) { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", runDir, walkErr))) } - - if foundPath != "" { - redactedDomainsLog.Printf("Found redacted-urls.log via recursive search: %s", foundPath) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found redacted-urls.log at "+foundPath)) - } - return parseRedactedDomainsLog(foundPath, verbose) - } - - // No redacted domains log found - this is not an error, just means no URLs were redacted - redactedDomainsLog.Print("No redacted-urls.log found") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No redacted-urls.log found in "+runDir)) - } - return nil, nil + return foundPath } diff --git a/pkg/cli/remove_command.go b/pkg/cli/remove_command.go index 634e3a98082..0d49b874b56 100644 --- a/pkg/cli/remove_command.go +++ b/pkg/cli/remove_command.go @@ -20,122 +20,141 @@ var removeLog = logger.New("cli:remove_command") // RemoveWorkflows removes workflows matching a pattern func RemoveWorkflows(pattern string, keepOrphans bool, workflowDir string) error { removeLog.Printf("Removing workflows: pattern=%q, keepOrphans=%v, workflowDir=%q", pattern, keepOrphans, workflowDir) - workflowsDir := workflowDir - if workflowsDir == "" { - workflowsDir = getWorkflowsDir() + workflowsDir := RemoveWorkflowsDir(workflowDir) + mdFiles, err := RemoveWorkflowsMarkdownFiles(workflowsDir) + if err != nil || len(mdFiles) == 0 { + return err } - if _, err := os.Stat(workflowsDir); os.IsNotExist(err) { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No .github/workflows directory found.")) + if pattern == "" { + RemoveWorkflowsPrintAvailable(mdFiles) + return nil + } + + filesToRemove := RemoveWorkflowsMatchingFiles(mdFiles, pattern) + if len(filesToRemove) == 0 { + removeLog.Printf("No workflows matched pattern: %q", pattern) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No workflows found matching pattern: "+pattern)) return nil } + removeLog.Printf("Found %d workflows to remove", len(filesToRemove)) + + orphanedIncludes := RemoveWorkflowsPreviewOrphans(filesToRemove, keepOrphans) + RemoveWorkflowsPrintPlan(filesToRemove, orphanedIncludes) + confirmed, err := RemoveWorkflowsConfirm() + if err != nil || !confirmed { + return err + } - // Find all markdown files in .github/workflows + removedFiles := RemoveWorkflowsDeleteFiles(filesToRemove) + RemoveWorkflowsCleanupOrphans(removedFiles, keepOrphans) + if len(removedFiles) > 0 && isGitRepo() { + stageWorkflowChanges() + } + return nil +} + +func RemoveWorkflowsDir(workflowDir string) string { + if workflowDir != "" { + return workflowDir + } + return getWorkflowsDir() +} + +func RemoveWorkflowsMarkdownFiles(workflowsDir string) ([]string, error) { + if _, err := os.Stat(workflowsDir); os.IsNotExist(err) { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No .github/workflows directory found.")) + return nil, nil + } mdFiles, err := filepath.Glob(filepath.Join(workflowsDir, "*.md")) if err != nil { - return fmt.Errorf("failed to find workflow files: %w", err) + return nil, fmt.Errorf("failed to find workflow files: %w", err) } - - // Filter out README.md files mdFiles = filterWorkflowFiles(mdFiles) - removeLog.Printf("Found %d workflow files", len(mdFiles)) if len(mdFiles) == 0 { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No workflow files found to remove.")) - return nil } + return mdFiles, nil +} - var filesToRemove []string - - // If no pattern specified, list all files for user to see - if pattern == "" { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Available workflows to remove:")) - for _, file := range mdFiles { - workflowName, _ := extractWorkflowNameFromFile(file) - base := filepath.Base(file) - name := normalizeWorkflowID(base) - if workflowName != "" { - fmt.Fprintf(os.Stderr, " %-20s - %s\n", name, workflowName) - } else { - fmt.Fprintf(os.Stderr, " %s\n", name) - } +func RemoveWorkflowsPrintAvailable(mdFiles []string) { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Available workflows to remove:")) + for _, file := range mdFiles { + workflowName, _ := extractWorkflowNameFromFile(file) + name := normalizeWorkflowID(filepath.Base(file)) + if workflowName != "" { + fmt.Fprintf(os.Stderr, " %-20s - %s\n", name, workflowName) + } else { + fmt.Fprintf(os.Stderr, " %s\n", name) } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("\nUsage: %s remove ", string(constants.CLIExtensionPrefix)))) - return nil } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("\nUsage: %s remove ", string(constants.CLIExtensionPrefix)))) +} - // Find matching files by workflow name or filename +func RemoveWorkflowsMatchingFiles(mdFiles []string, pattern string) []string { + var filesToRemove []string for _, file := range mdFiles { - base := filepath.Base(file) - filename := normalizeWorkflowID(base) + filename := normalizeWorkflowID(filepath.Base(file)) workflowName, _ := extractWorkflowNameFromFile(file) - - // Check if pattern matches filename or workflow name if strings.Contains(strings.ToLower(filename), strings.ToLower(pattern)) || strings.Contains(strings.ToLower(workflowName), strings.ToLower(pattern)) { filesToRemove = append(filesToRemove, file) } } + return filesToRemove +} - if len(filesToRemove) == 0 { - removeLog.Printf("No workflows matched pattern: %q", pattern) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No workflows found matching pattern: "+pattern)) +func RemoveWorkflowsPreviewOrphans(filesToRemove []string, keepOrphans bool) []string { + if keepOrphans { return nil } - - removeLog.Printf("Found %d workflows to remove", len(filesToRemove)) - - // Preview orphaned includes that would be removed (if orphan removal is enabled) - var orphanedIncludes []string - if !keepOrphans { - var err error - orphanedIncludes, err = previewOrphanedIncludes(filesToRemove, false) - if err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to preview orphaned includes: %v", err))) - orphanedIncludes = []string{} // Continue with empty list - } + orphanedIncludes, err := previewOrphanedIncludes(filesToRemove, false) + if err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to preview orphaned includes: %v", err))) + return []string{} } + return orphanedIncludes +} - // Show what will be removed +func RemoveWorkflowsPrintPlan(filesToRemove, orphanedIncludes []string) { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("The following workflows will be removed:")) for _, file := range filesToRemove { - workflowName, _ := extractWorkflowNameFromFile(file) - if workflowName != "" { - fmt.Fprintf(os.Stderr, " %s - %s\n", filepath.Base(file), workflowName) - } else { - fmt.Fprintf(os.Stderr, " %s\n", filepath.Base(file)) - } - - // Also check for corresponding .lock.yml file in .github/workflows - lockFile := stringutil.MarkdownToLockFile(file) - if fileutil.FileExists(lockFile) { - fmt.Fprintf(os.Stderr, " %s (compiled workflow)\n", filepath.Base(lockFile)) - } + RemoveWorkflowsPrintPlanFile(file) } - - // Show orphaned includes that will also be removed if len(orphanedIncludes) > 0 { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("\nThe following orphaned include files will also be removed (suppress with --keep-orphans):")) for _, include := range orphanedIncludes { fmt.Fprintf(os.Stderr, " %s (orphaned include)\n", include) } } +} + +func RemoveWorkflowsPrintPlanFile(file string) { + workflowName, _ := extractWorkflowNameFromFile(file) + if workflowName != "" { + fmt.Fprintf(os.Stderr, " %s - %s\n", filepath.Base(file), workflowName) + } else { + fmt.Fprintf(os.Stderr, " %s\n", filepath.Base(file)) + } + lockFile := stringutil.MarkdownToLockFile(file) + if fileutil.FileExists(lockFile) { + fmt.Fprintf(os.Stderr, " %s (compiled workflow)\n", filepath.Base(lockFile)) + } +} - // Ask for confirmation - confirmed, err := console.ConfirmAction( - "Are you sure you want to remove these workflows?", - "Yes, remove", - "No, cancel", - ) +func RemoveWorkflowsConfirm() (bool, error) { + confirmed, err := console.ConfirmAction("Are you sure you want to remove these workflows?", "Yes, remove", "No, cancel") if err != nil { - return fmt.Errorf("failed to get confirmation: %w", err) + return false, fmt.Errorf("failed to get confirmation: %w", err) } if !confirmed { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Operation cancelled.")) - return nil } + return confirmed, nil +} - // Remove the files +func RemoveWorkflowsDeleteFiles(filesToRemove []string) []string { var removedFiles []string for _, file := range filesToRemove { if err := os.Remove(file); err != nil { @@ -144,40 +163,37 @@ func RemoveWorkflows(pattern string, keepOrphans bool, workflowDir string) error fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Removed: "+filepath.Base(file))) removedFiles = append(removedFiles, file) } - - // Also remove corresponding .lock.yml file - lockFile := stringutil.MarkdownToLockFile(file) - if fileutil.FileExists(lockFile) { - if err := os.Remove(lockFile); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove %s: %v", lockFile, err))) - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Removed: "+filepath.Base(lockFile))) - } - } + RemoveWorkflowsDeleteLockFile(file) } + return removedFiles +} - // Clean up orphaned include files (if orphan removal is enabled) - if len(removedFiles) > 0 && !keepOrphans { - if err := cleanupOrphanedIncludes(false); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to clean up orphaned includes: %v", err))) - } +func RemoveWorkflowsDeleteLockFile(file string) { + lockFile := stringutil.MarkdownToLockFile(file) + if !fileutil.FileExists(lockFile) { + return } - - // Stage changes to git if in a git repository - if len(removedFiles) > 0 && isGitRepo() { - stageWorkflowChanges() + if err := os.Remove(lockFile); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove %s: %v", lockFile, err))) + } else { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Removed: "+filepath.Base(lockFile))) } +} - return nil +func RemoveWorkflowsCleanupOrphans(removedFiles []string, keepOrphans bool) { + if len(removedFiles) == 0 || keepOrphans { + return + } + if err := cleanupOrphanedIncludes(false); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to clean up orphaned includes: %v", err))) + } } // cleanupOrphanedIncludes removes include files that are no longer used by any workflow func cleanupOrphanedIncludes(verbose bool) error { removeLog.Print("Cleaning up orphaned include files") - // Get all remaining markdown files - mdFiles, err := getMarkdownWorkflowFiles("") + usedIncludes, err := cleanupOrphanedIncludesUsed(verbose) if err != nil { - // No markdown files means we can clean up all includes removeLog.Print("No markdown files found, cleaning up all includes") if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No markdown files found, cleaning up all includes")) @@ -185,10 +201,20 @@ func cleanupOrphanedIncludes(verbose bool) error { return cleanupAllIncludes(verbose) } - // Collect all include dependencies from remaining workflows - usedIncludes := make(map[string]struct { - }) + allIncludes, err := getAllIncludeFiles() + if err != nil { + return fmt.Errorf("failed to scan include files: %w", err) + } + cleanupOrphanedIncludesRemove(allIncludes, usedIncludes, verbose) + return nil +} +func cleanupOrphanedIncludesUsed(verbose bool) (map[string]struct{}, error) { + mdFiles, err := getMarkdownWorkflowFiles("") + if err != nil { + return nil, err + } + usedIncludes := make(map[string]struct{}) for _, mdFile := range mdFiles { content, err := os.ReadFile(mdFile) if err != nil { @@ -197,8 +223,6 @@ func cleanupOrphanedIncludes(verbose bool) error { } continue } - - // Find includes used by this workflow includes, err := findIncludesInContent(string(content)) if err != nil { if verbose { @@ -206,94 +230,72 @@ func cleanupOrphanedIncludes(verbose bool) error { } continue } - for _, include := range includes { - usedIncludes[include] = struct { - }{} + usedIncludes[include] = struct{}{} } } + return usedIncludes, nil +} - // Find all include files in the workflows directory - // Only consider files in subdirectories (like shared/) as potential include files - // Root-level .md files are workflow files, not include files +func cleanupOrphanedIncludesRemove(allIncludes []string, usedIncludes map[string]struct{}, verbose bool) { workflowsDir := constants.GetWorkflowDir() - var allIncludes []string - - err = filepath.Walk(workflowsDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err + for _, include := range allIncludes { + if setutil.Contains(usedIncludes, include) { + continue } - - if !info.IsDir() && strings.HasSuffix(info.Name(), ".md") { - relPath, err := filepath.Rel(workflowsDir, path) - if err != nil { - return err - } - - // Only consider files in subdirectories as potential include files - // Root-level .md files are workflow files, not include files - if strings.Contains(relPath, string(filepath.Separator)) { - allIncludes = append(allIncludes, relPath) + includePath := filepath.Join(workflowsDir, include) + if err := os.Remove(includePath); err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove orphaned include %s: %v", include, err))) } + } else { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Removed orphaned include: "+include)) } + } +} - return nil - }) +// previewOrphanedIncludes returns a list of include files that would become orphaned if the specified files were removed +func previewOrphanedIncludes(filesToRemove []string, verbose bool) ([]string, error) { + allMdFiles, err := getMarkdownWorkflowFiles("") + if err != nil { + return nil, err + } + remainingFiles := previewOrphanedIncludesRemainingFiles(allMdFiles, filesToRemove) + if len(remainingFiles) == 0 { + return getAllIncludeFiles() + } + usedIncludes := previewOrphanedIncludesUsed(remainingFiles, verbose) + allIncludes, err := getAllIncludeFiles() if err != nil { - return fmt.Errorf("failed to scan include files: %w", err) + return nil, err } - // Remove unused includes + var orphanedIncludes []string for _, include := range allIncludes { if !setutil.Contains(usedIncludes, include) { - includePath := filepath.Join(workflowsDir, include) - if err := os.Remove(includePath); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove orphaned include %s: %v", include, err))) - } - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Removed orphaned include: "+include)) - } + orphanedIncludes = append(orphanedIncludes, include) } } - - return nil + return orphanedIncludes, nil } -// previewOrphanedIncludes returns a list of include files that would become orphaned if the specified files were removed -func previewOrphanedIncludes(filesToRemove []string, verbose bool) ([]string, error) { - // Get all current markdown files - allMdFiles, err := getMarkdownWorkflowFiles("") - if err != nil { - return nil, err - } - - // Create a map of files to remove for quick lookup - removeMap := make(map[string]struct { - }) +func previewOrphanedIncludesRemainingFiles(allMdFiles, filesToRemove []string) []string { + removeMap := make(map[string]struct{}) for _, file := range filesToRemove { - removeMap[file] = struct { - }{} + removeMap[file] = struct{}{} } - - // Get the files that would remain after removal var remainingFiles []string for _, file := range allMdFiles { if !setutil.Contains(removeMap, file) { remainingFiles = append(remainingFiles, file) } } + return remainingFiles +} - // If no files remain, all include files would be orphaned - if len(remainingFiles) == 0 { - return getAllIncludeFiles() - } - - // Collect all include dependencies from remaining workflows - usedIncludes := make(map[string]struct { - }) - +func previewOrphanedIncludesUsed(remainingFiles []string, verbose bool) map[string]struct{} { + usedIncludes := make(map[string]struct{}) for _, mdFile := range remainingFiles { content, err := os.ReadFile(mdFile) if err != nil { @@ -302,8 +304,6 @@ func previewOrphanedIncludes(filesToRemove []string, verbose bool) ([]string, er } continue } - - // Find includes used by this workflow includes, err := findIncludesInContent(string(content)) if err != nil { if verbose { @@ -311,27 +311,11 @@ func previewOrphanedIncludes(filesToRemove []string, verbose bool) ([]string, er } continue } - for _, include := range includes { - usedIncludes[include] = struct { - }{} - } - } - - // Find all include files and check which ones would be orphaned - allIncludes, err := getAllIncludeFiles() - if err != nil { - return nil, err - } - - var orphanedIncludes []string - for _, include := range allIncludes { - if !setutil.Contains(usedIncludes, include) { - orphanedIncludes = append(orphanedIncludes, include) + usedIncludes[include] = struct{}{} } } - - return orphanedIncludes, nil + return usedIncludes } // getAllIncludeFiles returns all include files in .github/workflows subdirectories @@ -424,69 +408,55 @@ func findIncludesInContent(content string) ([]string, error) { // parseIncludePath extracts the file path from @include/@import/{{#import}} directive lines // without allocating a regex submatch slice or a directive struct. // Returns an empty string if the line is not a recognised directive. -// Section references (e.g. file.md#Section) are stripped from the returned path. func parseIncludePath(line string) string { trimmed := strings.TrimSpace(line) if trimmed == "" { return "" } - - // Fast path: the vast majority of lines are not directives. - // Checking the first byte avoids three full HasPrefix comparisons. if trimmed[0] != '@' && trimmed[0] != '{' { return "" } var rest string - switch { case strings.HasPrefix(trimmed, "@include"): rest = trimmed[len("@include"):] case strings.HasPrefix(trimmed, "@import"): rest = trimmed[len("@import"):] case strings.HasPrefix(trimmed, "{{#import"): - rest = trimmed[len("{{#import"):] - // Skip optional marker '?' - if rest != "" && rest[0] == '?' { - rest = rest[1:] - } - // Skip optional whitespace, then an optional single colon, then optional whitespace - // (mirrors the regex \s*:?\s* in IncludeDirectivePattern) - rest = strings.TrimSpace(rest) - if rest != "" && rest[0] == ':' { - rest = strings.TrimSpace(rest[1:]) - } - // Extract path up to closing "}}" and require only whitespace after it. - before, after, ok := strings.Cut(rest, "}}") - if !ok || strings.TrimSpace(after) != "" { - return "" - } - path := strings.TrimSpace(before) - if path == "" { - return "" - } - // Strip section reference (file.md#Section → file.md) - if filePath, _, ok := strings.Cut(path, "#"); ok { - return filePath - } - return path + return parseIncludePathBraceDirective(trimmed[len("{{#import"):]) default: return "" } - // Handle @include and @import: skip optional marker '?' if rest != "" && rest[0] == '?' { rest = rest[1:] } - // Require at least one whitespace character after the directive keyword if rest == "" || (rest[0] != ' ' && rest[0] != '\t') { return "" } - path := strings.TrimSpace(rest) + return parseIncludePathStripSection(strings.TrimSpace(rest)) +} + +func parseIncludePathBraceDirective(rest string) string { + if rest != "" && rest[0] == '?' { + rest = rest[1:] + } + rest = strings.TrimSpace(rest) + if rest != "" && rest[0] == ':' { + rest = strings.TrimSpace(rest[1:]) + } + before, after, ok := strings.Cut(rest, "}}") + if !ok || strings.TrimSpace(after) != "" { + return "" + } + return parseIncludePathStripSection(strings.TrimSpace(before)) +} + +func parseIncludePathStripSection(path string) string { if path == "" { return "" } - // Strip section reference (file.md#Section → file.md) if filePath, _, ok := strings.Cut(path, "#"); ok { return filePath } diff --git a/pkg/cli/resources.go b/pkg/cli/resources.go index 8611500a86f..46a9e3a9f6e 100644 --- a/pkg/cli/resources.go +++ b/pkg/cli/resources.go @@ -68,15 +68,54 @@ func extractResources(content string) ([]string, error) { // For non-Markdown resource files: if the target already exists and force is false, an error // is returned regardless of origin (non-markdown files have no source tracking). func fetchAndSaveRemoteResources(ctx context.Context, content string, spec *WorkflowSpec, targetDir string, verbose bool, force bool, tracker *FileTracker) error { - if spec.RepoSlug == "" { + owner, repo, ref, ok := fetchAndSaveRemoteResourcesRepoInfo(ctx, spec) + if !ok { + return nil + } + resourcePaths, err := extractResources(content) + if err != nil || len(resourcePaths) == 0 { + return err + } + + absTargetDir, err := filepath.Abs(targetDir) + if err != nil { + remoteWorkflowLog.Printf("Failed to resolve absolute path for target directory %s: %v", targetDir, err) return nil } + state := fetchAndSaveRemoteResourcesContext{ + owner: owner, repo: repo, ref: ref, workflowBaseDir: getParentDir(spec.WorkflowPath), + targetDir: targetDir, absTargetDir: absTargetDir, spec: spec, + verbose: verbose, force: force, tracker: tracker, + } + for _, resourcePath := range resourcePaths { + if err := fetchAndSaveRemoteResourcesOne(ctx, state, resourcePath); err != nil { + return err + } + } + return nil +} + +type fetchAndSaveRemoteResourcesContext struct { + owner string + repo string + ref string + workflowBaseDir string + targetDir string + absTargetDir string + spec *WorkflowSpec + verbose bool + force bool + tracker *FileTracker +} +func fetchAndSaveRemoteResourcesRepoInfo(ctx context.Context, spec *WorkflowSpec) (string, string, string, bool) { + if spec.RepoSlug == "" { + return "", "", "", false + } parts := strings.SplitN(spec.RepoSlug, "/", 2) if len(parts) != 2 { - return nil + return "", "", "", false } - owner, repo := parts[0], parts[1] ref := spec.Version if ref == "" { defaultBranch, err := getRepoDefaultBranch(ctx, spec.RepoSlug) @@ -88,145 +127,130 @@ func fetchAndSaveRemoteResources(ctx context.Context, content string, spec *Work } spec.Version = ref } + return parts[0], parts[1], ref, true +} - resourcePaths, err := extractResources(content) - if err != nil { - return err - } - if len(resourcePaths) == 0 { +func fetchAndSaveRemoteResourcesOne(ctx context.Context, state fetchAndSaveRemoteResourcesContext, resourcePath string) error { + if strings.Contains(resourcePath, "..") { + if state.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping resource with unsafe path: %q", resourcePath))) + } return nil } - // Resources are resolved relative to the source workflow's directory in the remote repo. - workflowBaseDir := getParentDir(spec.WorkflowPath) - - // Pre-compute the absolute target directory for path-traversal boundary checks. - absTargetDir, err := filepath.Abs(targetDir) - if err != nil { - remoteWorkflowLog.Printf("Failed to resolve absolute path for target directory %s: %v", targetDir, err) + remoteFilePath, targetPath, ok := fetchAndSaveRemoteResourcesPaths(state, resourcePath) + if !ok { return nil } - - for _, resourcePath := range resourcePaths { - // Early rejection of path traversal patterns. This is a fast first-pass check; - // the filepath.Rel boundary check below is the authoritative security control. - if strings.Contains(resourcePath, "..") { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping resource with unsafe path: %q", resourcePath))) - } - continue - } - - // Resolve the remote file path - var remoteFilePath string - if rest, ok := strings.CutPrefix(resourcePath, "/"); ok { - remoteFilePath = rest - } else if workflowBaseDir != "" { - remoteFilePath = path.Join(workflowBaseDir, resourcePath) - } else { - remoteFilePath = resourcePath - } - remoteFilePath = path.Clean(remoteFilePath) - - // Derive the local relative path by stripping the workflow base dir prefix - localRelPath := remoteFilePath - if workflowBaseDir != "" && strings.HasPrefix(remoteFilePath, workflowBaseDir+"/") { - localRelPath = remoteFilePath[len(workflowBaseDir)+1:] - } - localRelPath = filepath.Clean(filepath.FromSlash(localRelPath)) - localRelPath = strings.TrimLeft(localRelPath, string(filepath.Separator)) - if localRelPath == "" || localRelPath == "." { - continue - } - targetPath := filepath.Join(targetDir, localRelPath) - - // Belt-and-suspenders: verify the resolved path stays inside targetDir - absTargetPath, absErr := filepath.Abs(targetPath) - if absErr != nil { - remoteWorkflowLog.Printf("Failed to resolve absolute path for resource %s: %v", resourcePath, absErr) - continue - } - if rel, relErr := filepath.Rel(absTargetDir, absTargetPath); relErr != nil || strings.HasPrefix(rel, "..") { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Refusing to write resource outside target directory: %q", resourcePath))) - } - continue + fileExists, err := fetchAndSaveRemoteResourcesCheckExisting(state, resourcePath, targetPath) + if err != nil || fileExists && !state.force && strings.HasSuffix(strings.ToLower(targetPath), ".md") && readSourceRepoFromFile(targetPath) == state.spec.RepoSlug { + return err + } + fileContent, ok := fetchAndSaveRemoteResourcesDownload(ctx, state, remoteFilePath) + if !ok { + return nil + } + if strings.HasSuffix(strings.ToLower(remoteFilePath), ".md") { + depSourceString := state.spec.RepoSlug + "/" + remoteFilePath + "@" + state.ref + if updated, srcErr := addSourceToWorkflow(string(fileContent), depSourceString); srcErr == nil { + fileContent = []byte(updated) } + } + return fetchAndSaveRemoteResourcesWrite(state, remoteFilePath, targetPath, fileContent, fileExists) +} - // Check whether the target file already exists. - fileExists := false - if fileutil.FileExists(targetPath) { - fileExists = true - if !force { - isMarkdown := strings.HasSuffix(strings.ToLower(targetPath), ".md") - if isMarkdown { - // For markdown files, allow same-source overwrites. - existingSourceRepo := readSourceRepoFromFile(targetPath) - if existingSourceRepo == spec.RepoSlug { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Resource file from same source already exists, skipping: "+targetPath)) - } - continue - } - return fmt.Errorf( - "resource %q already exists at %s (existing source: %q, installing from: %q); remove the file or use --force to overwrite", - resourcePath, targetPath, sourceRepoLabel(existingSourceRepo), spec.RepoSlug, - ) - } - // Non-markdown files have no source tracking — always conflict. - return fmt.Errorf( - "resource %q already exists at %s; remove the file or use --force to overwrite", - resourcePath, targetPath, - ) - } +func fetchAndSaveRemoteResourcesPaths(state fetchAndSaveRemoteResourcesContext, resourcePath string) (string, string, bool) { + var remoteFilePath string + if rest, ok := strings.CutPrefix(resourcePath, "/"); ok { + remoteFilePath = rest + } else if state.workflowBaseDir != "" { + remoteFilePath = path.Join(state.workflowBaseDir, resourcePath) + } else { + remoteFilePath = resourcePath + } + remoteFilePath = path.Clean(remoteFilePath) + localRelPath := fetchAndSaveRemoteResourcesLocalPath(state.workflowBaseDir, remoteFilePath) + if localRelPath == "" || localRelPath == "." { + return "", "", false + } + targetPath := filepath.Join(state.targetDir, localRelPath) + absTargetPath, absErr := filepath.Abs(targetPath) + if absErr != nil { + remoteWorkflowLog.Printf("Failed to resolve absolute path for resource %s: %v", resourcePath, absErr) + return "", "", false + } + if rel, relErr := filepath.Rel(state.absTargetDir, absTargetPath); relErr != nil || strings.HasPrefix(rel, "..") { + if state.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Refusing to write resource outside target directory: %q", resourcePath))) } + return "", "", false + } + return remoteFilePath, targetPath, true +} - // Download from source repository - fileContent, err := parser.DownloadFileFromGitHub(ctx, owner, repo, remoteFilePath, ref) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch resource %s: %v", remoteFilePath, err))) - } - continue - } +func fetchAndSaveRemoteResourcesLocalPath(workflowBaseDir, remoteFilePath string) string { + localRelPath := remoteFilePath + if workflowBaseDir != "" && strings.HasPrefix(remoteFilePath, workflowBaseDir+"/") { + localRelPath = remoteFilePath[len(workflowBaseDir)+1:] + } + localRelPath = filepath.Clean(filepath.FromSlash(localRelPath)) + return strings.TrimLeft(localRelPath, string(filepath.Separator)) +} - // For markdown resources, embed the source field for future conflict detection. - if strings.HasSuffix(strings.ToLower(remoteFilePath), ".md") { - depSourceString := spec.RepoSlug + "/" + remoteFilePath + "@" + ref - if updated, srcErr := addSourceToWorkflow(string(fileContent), depSourceString); srcErr == nil { - fileContent = []byte(updated) +func fetchAndSaveRemoteResourcesCheckExisting(state fetchAndSaveRemoteResourcesContext, resourcePath, targetPath string) (bool, error) { + if !fileutil.FileExists(targetPath) { + return false, nil + } + if state.force { + return true, nil + } + isMarkdown := strings.HasSuffix(strings.ToLower(targetPath), ".md") + if isMarkdown { + existingSourceRepo := readSourceRepoFromFile(targetPath) + if existingSourceRepo == state.spec.RepoSlug { + if state.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Resource file from same source already exists, skipping: "+targetPath)) } + return true, nil } + return true, fmt.Errorf("resource %q already exists at %s (existing source: %q, installing from: %q); remove the file or use --force to overwrite", resourcePath, targetPath, sourceRepoLabel(existingSourceRepo), state.spec.RepoSlug) + } + return true, fmt.Errorf("resource %q already exists at %s; remove the file or use --force to overwrite", resourcePath, targetPath) +} - // Create parent directory if needed - if err := os.MkdirAll(filepath.Dir(targetPath), constants.DirPermPublic); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for resource %s: %v", remoteFilePath, err))) - } - continue +func fetchAndSaveRemoteResourcesDownload(ctx context.Context, state fetchAndSaveRemoteResourcesContext, remoteFilePath string) ([]byte, bool) { + fileContent, err := parser.DownloadFileFromGitHub(ctx, state.owner, state.repo, remoteFilePath, state.ref) + if err != nil { + if state.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to fetch resource %s: %v", remoteFilePath, err))) } + return nil, false + } + return fileContent, true +} - // Write the file - if err := os.WriteFile(targetPath, fileContent, constants.FilePermSensitive); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write resource %s: %v", remoteFilePath, err))) - } - continue +func fetchAndSaveRemoteResourcesWrite(state fetchAndSaveRemoteResourcesContext, remoteFilePath, targetPath string, fileContent []byte, fileExists bool) error { + if err := os.MkdirAll(filepath.Dir(targetPath), constants.DirPermPublic); err != nil { + if state.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to create directory for resource %s: %v", remoteFilePath, err))) } - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched resource: "+targetPath)) + return nil + } + if err := os.WriteFile(targetPath, fileContent, constants.FilePermSensitive); err != nil { + if state.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to write resource %s: %v", remoteFilePath, err))) } - - // Track the file - if tracker != nil { - if fileExists { - tracker.TrackModified(targetPath) - } else { - tracker.TrackCreated(targetPath) - } + return nil + } + if state.verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched resource: "+targetPath)) + } + if state.tracker != nil { + if fileExists { + state.tracker.TrackModified(targetPath) + } else { + state.tracker.TrackCreated(targetPath) } } - return nil } diff --git a/pkg/cli/retry.go b/pkg/cli/retry.go index c33ac192d3f..8ca9cad58dc 100644 --- a/pkg/cli/retry.go +++ b/pkg/cli/retry.go @@ -37,89 +37,87 @@ type RepeatOptions struct { // with graceful signal handling for shutdown. func ExecuteWithRepeat(options RepeatOptions) error { retryLog.Printf("Executing function with repeat count: %d", options.RepeatCount) - // Run the function once if err := options.ExecuteFunc(); err != nil { retryLog.Printf("Initial execution failed: %v", err) return err } - - // If no repeat specified, we're done if options.RepeatCount <= 0 { retryLog.Print("No repeat requested, execution complete") return nil } + output := ExecuteWithRepeatOutput(options) + ctx := ExecuteWithRepeatContext(options) + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sigChan) + + runCleanup := func() { + if options.CleanupFunc != nil { + retryLog.Print("Executing cleanup function") + options.CleanupFunc() + } + } + for i := 1; i <= options.RepeatCount; i++ { + if err := ExecuteWithRepeatIteration(ctx, sigChan, output, options, i, runCleanup); err != nil { + return err + } + } + retryLog.Printf("Completed all %d iterations successfully", options.RepeatCount) + return nil +} + +func ExecuteWithRepeatOutput(options RepeatOptions) *os.File { retryLog.Printf("Starting repeat mode for %d iterations", options.RepeatCount) - // Set up repeat mode output := os.Stdout if options.UseStderr { output = os.Stderr } - - // Use provided start message or default startMsg := options.StartMessage if startMsg == "" { startMsg = fmt.Sprintf("Repeating %d more times. Press Ctrl+C to stop.", options.RepeatCount) } fmt.Fprintln(output, console.FormatInfoMessage(startMsg)) + return output +} - // Use provided context or fall back to background context +func ExecuteWithRepeatContext(options RepeatOptions) context.Context { ctx := options.Ctx if ctx == nil { ctx = context.Background() } + return ctx +} - // Set up signal handling for graceful shutdown - // Signal channel provides a fallback when no context is provided or for direct OS signals - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - defer signal.Stop(sigChan) - - // runCleanup executes the optional cleanup function - runCleanup := func() { - if options.CleanupFunc != nil { - retryLog.Print("Executing cleanup function") - options.CleanupFunc() - } +func ExecuteWithRepeatIteration(ctx context.Context, sigChan <-chan os.Signal, output *os.File, options RepeatOptions, i int, runCleanup func()) error { + select { + case <-ctx.Done(): + retryLog.Printf("Context cancelled at iteration %d/%d", i, options.RepeatCount) + fmt.Fprintln(output, console.FormatInfoMessage("Received interrupt signal, stopping repeat...")) + runCleanup() + return ctx.Err() + case <-sigChan: + retryLog.Printf("Interrupt signal received at iteration %d/%d", i, options.RepeatCount) + fmt.Fprintln(output, console.FormatInfoMessage("Received interrupt signal, stopping repeat...")) + runCleanup() + return context.Canceled + default: + retryLog.Printf("Starting iteration %d/%d", i, options.RepeatCount) + ExecuteWithRepeatRunIteration(output, options, i) + return nil } +} - // Run the specified number of additional times - for i := 1; i <= options.RepeatCount; i++ { - select { - case <-ctx.Done(): - retryLog.Printf("Context cancelled at iteration %d/%d", i, options.RepeatCount) - fmt.Fprintln(output, console.FormatInfoMessage("Received interrupt signal, stopping repeat...")) - runCleanup() - return ctx.Err() - - case <-sigChan: - retryLog.Printf("Interrupt signal received at iteration %d/%d", i, options.RepeatCount) - fmt.Fprintln(output, console.FormatInfoMessage("Received interrupt signal, stopping repeat...")) - runCleanup() - return context.Canceled - - default: - retryLog.Printf("Starting iteration %d/%d", i, options.RepeatCount) - // Use provided repeat message or default - repeatMsg := options.RepeatMessage - if repeatMsg == "" { - repeatMsg = fmt.Sprintf("Running repetition %d/%d", i, options.RepeatCount) - } else { - // If message contains timestamp placeholder, replace it with current time - if strings.Contains(repeatMsg, "%s") { - repeatMsg = fmt.Sprintf(repeatMsg, time.Now().Format("2006-01-02 15:04:05")) - } - } - fmt.Fprintln(output, console.FormatInfoMessage(repeatMsg)) - - if err := options.ExecuteFunc(); err != nil { - retryLog.Printf("Error during iteration %d: %v", i, err) - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Error during repeat %d/%d: %v", i, options.RepeatCount, err))) - // Continue running on error during repeat - } - } +func ExecuteWithRepeatRunIteration(output *os.File, options RepeatOptions, i int) { + repeatMsg := options.RepeatMessage + if repeatMsg == "" { + repeatMsg = fmt.Sprintf("Running repetition %d/%d", i, options.RepeatCount) + } else if strings.Contains(repeatMsg, "%s") { + repeatMsg = fmt.Sprintf(repeatMsg, time.Now().Format("2006-01-02 15:04:05")) + } + fmt.Fprintln(output, console.FormatInfoMessage(repeatMsg)) + if err := options.ExecuteFunc(); err != nil { + retryLog.Printf("Error during iteration %d: %v", i, err) + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Error during repeat %d/%d: %v", i, options.RepeatCount, err))) } - - retryLog.Printf("Completed all %d iterations successfully", options.RepeatCount) - return nil } diff --git a/pkg/cli/run_interactive.go b/pkg/cli/run_interactive.go index 904b2321eca..2f2ee6da647 100644 --- a/pkg/cli/run_interactive.go +++ b/pkg/cli/run_interactive.go @@ -30,79 +30,106 @@ type WorkflowOption struct { // RunWorkflowInteractively runs a workflow in interactive mode func RunWorkflowInteractively(ctx context.Context, verbose bool, repoOverride string, refOverride string, autoMergePRs bool, push bool, engineOverride string, dryRun bool) error { runInteractiveLog.Print("Starting interactive workflow run") - - // Check if running in CI environment if IsRunningInCI() { return errors.New("interactive mode cannot be used in CI environments") } - if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Starting interactive workflow run...")) } - // Step 1: Find workflows with workflow_dispatch trigger - workflows, err := findRunnableWorkflows(verbose) + selectedWorkflow, inputValues, err := runWorkflowInteractivelySelect(ctx, verbose) if err != nil { - return fmt.Errorf("failed to find runnable workflows: %w", err) + return err + } + if !confirmExecution(ctx, selectedWorkflow, inputValues) { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Workflow execution cancelled")) + return nil } - if len(workflows) == 0 { - return errors.New("no runnable workflows found. Workflows must have 'workflow_dispatch' trigger") + cmdStr := buildCommandString(selectedWorkflow.Name, inputValues, repoOverride, refOverride, autoMergePRs, push, engineOverride) + runWorkflowInteractivelyPrintCommand(cmdStr) + if err := runWorkflowInteractivelyExecute(runWorkflowInteractivelyExecuteParams{ + Ctx: ctx, + WorkflowName: selectedWorkflow.Name, + InputValues: inputValues, + Verbose: verbose, + RepoOverride: repoOverride, + RefOverride: refOverride, + AutoMergePRs: autoMergePRs, + Push: push, + EngineOverride: engineOverride, + DryRun: dryRun, + }); err != nil { + return err } + runWorkflowInteractivelyPrintSuccess(cmdStr) + return nil +} - // Step 2: Let user select a workflow +func runWorkflowInteractivelySelect(ctx context.Context, verbose bool) (*WorkflowOption, []string, error) { + workflows, err := findRunnableWorkflows(verbose) + if err != nil { + return nil, nil, fmt.Errorf("failed to find runnable workflows: %w", err) + } + if len(workflows) == 0 { + return nil, nil, errors.New("no runnable workflows found. Workflows must have 'workflow_dispatch' trigger") + } selectedWorkflow, err := selectWorkflow(ctx, workflows) if err != nil { - return fmt.Errorf("workflow selection cancelled or failed: %w", err) + return nil, nil, fmt.Errorf("workflow selection cancelled or failed: %w", err) } - runInteractiveLog.Printf("Selected workflow: %s", selectedWorkflow.Name) - - // Step 3: Show workflow information showWorkflowInfo(selectedWorkflow) - - // Step 4: Collect workflow inputs if needed inputValues, err := collectWorkflowInputs(ctx, selectedWorkflow) if err != nil { - return fmt.Errorf("failed to collect workflow inputs: %w", err) - } - - // Step 5: Confirm execution - if !confirmExecution(ctx, selectedWorkflow, inputValues) { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Workflow execution cancelled")) - return nil + return nil, nil, fmt.Errorf("failed to collect workflow inputs: %w", err) } + return selectedWorkflow, inputValues, nil +} - // Step 6: Build command string for display - cmdStr := buildCommandString(selectedWorkflow.Name, inputValues, repoOverride, refOverride, autoMergePRs, push, engineOverride) +func runWorkflowInteractivelyPrintCommand(cmdStr string) { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("\nRunning workflow...")) fmt.Fprintln(os.Stderr, console.FormatCommandMessage("Equivalent command: "+cmdStr)) fmt.Fprintln(os.Stderr, "") +} - // Step 7: Execute the workflow - err = RunWorkflowOnGitHub(ctx, selectedWorkflow.Name, RunOptions{ +type runWorkflowInteractivelyExecuteParams struct { + Ctx context.Context + WorkflowName string + InputValues []string + Verbose bool + RepoOverride string + RefOverride string + AutoMergePRs bool + Push bool + EngineOverride string + DryRun bool +} + +func runWorkflowInteractivelyExecute(p runWorkflowInteractivelyExecuteParams) error { + err := RunWorkflowOnGitHub(p.Ctx, p.WorkflowName, RunOptions{ Enable: false, - EngineOverride: engineOverride, - RepoOverride: repoOverride, - RefOverride: refOverride, - AutoMergePRs: autoMergePRs, - Push: push, - Inputs: inputValues, - Verbose: verbose, - DryRun: dryRun, + EngineOverride: p.EngineOverride, + RepoOverride: p.RepoOverride, + RefOverride: p.RefOverride, + AutoMergePRs: p.AutoMergePRs, + Push: p.Push, + Inputs: p.InputValues, + Verbose: p.Verbose, + DryRun: p.DryRun, }) if err != nil { return fmt.Errorf("failed to run workflow: %w", err) } + return nil +} - // Show success message with command to run again +func runWorkflowInteractivelyPrintSuccess(cmdStr string) { fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Workflow dispatched successfully!")) fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, console.FormatInfoMessage("To run this workflow again, use:")) fmt.Fprintln(os.Stderr, console.FormatCommandMessage(cmdStr)) - - return nil } // findRunnableWorkflows finds all workflows that support workflow_dispatch @@ -266,57 +293,18 @@ func collectWorkflowInputs(ctx context.Context, wf *WorkflowOption) ([]string, e // collectInputsWithMap collects inputs using a map to properly capture values func collectInputsWithMap(ctx context.Context, inputs map[string]*workflow.InputDefinition) ([]string, error) { - // Create a map to store string values for the form - inputValues := make(map[string]string) - // Create a map to track the string pointers we'll pass to huh inputPtrs := make(map[string]*string) var formGroups []*huh.Group - - // Create input fields for each workflow input for name, input := range inputs { - inputName := name - inputDef := input - - // Initialize with default value (convert any to string) - defaultStr := "" - if inputDef.Default != nil { - defaultStr = fmt.Sprintf("%v", inputDef.Default) - } - inputValues[inputName] = defaultStr - - // Create a string variable for this input that huh can update - valueStr := defaultStr - inputPtrs[inputName] = &valueStr - - // Create input field that updates the string variable - field := huh.NewInput(). - Title(fmt.Sprintf("Enter value for '%s'", inputName)). - Value(inputPtrs[inputName]) - - if inputDef.Description != "" { - field = field.Description(inputDef.Description) - } - - if inputDef.Required { - field = field.Validate(func(s string) error { - if s == "" { - return errors.New("this input is required") - } - return nil - }) - } - - group := huh.NewGroup(field) + group := collectInputsWithMapGroup(name, input, inputPtrs) formGroups = append(formGroups, group) } - // Show the form form := console.NewForm(formGroups...) if err := form.RunWithContext(ctx); err != nil { return nil, fmt.Errorf("input collection cancelled: %w", err) } - // Collect the final values from the pointers var result []string for name, valuePtr := range inputPtrs { value := *valuePtr @@ -324,11 +312,34 @@ func collectInputsWithMap(ctx context.Context, inputs map[string]*workflow.Input result = append(result, fmt.Sprintf("%s=%s", name, value)) } } - runInteractiveLog.Printf("Collected %d input values", len(result)) return result, nil } +func collectInputsWithMapGroup(name string, input *workflow.InputDefinition, inputPtrs map[string]*string) *huh.Group { + inputName := name + inputDef := input + defaultStr := "" + if inputDef.Default != nil { + defaultStr = fmt.Sprintf("%v", inputDef.Default) + } + valueStr := defaultStr + inputPtrs[inputName] = &valueStr + field := huh.NewInput().Title(fmt.Sprintf("Enter value for '%s'", inputName)).Value(inputPtrs[inputName]) + if inputDef.Description != "" { + field = field.Description(inputDef.Description) + } + if inputDef.Required { + field = field.Validate(func(s string) error { + if s == "" { + return errors.New("this input is required") + } + return nil + }) + } + return huh.NewGroup(field) +} + // confirmExecution asks the user to confirm workflow execution func confirmExecution(ctx context.Context, wf *WorkflowOption, inputs []string) bool { runInteractiveLog.Print("Requesting execution confirmation") @@ -378,57 +389,47 @@ type RunWorkflowOptions struct { // since the workflow name is already known. It will still collect inputs if the workflow has them. func RunSpecificWorkflowInteractively(ctx context.Context, opts RunWorkflowOptions) error { runInteractiveLog.Printf("Running specific workflow interactively: %s", opts.WorkflowName) - - // Find the workflow file - workflowsDir := constants.GetWorkflowDir() - mdFile := filepath.Join(workflowsDir, opts.WorkflowName+".md") - - // Check if file exists - if _, err := os.Stat(mdFile); os.IsNotExist(err) { - return fmt.Errorf("workflow file not found: %s", mdFile) - } - - // Get workflow inputs - inputs, err := getWorkflowInputs(mdFile) + wf, err := runSpecificWorkflowInteractivelyOption(opts.WorkflowName) if err != nil { - runInteractiveLog.Printf("Failed to get inputs for workflow %s: %v", opts.WorkflowName, err) - // Continue without inputs - they might not be required - inputs = nil + return err } - - // Create workflow option for display - wf := &WorkflowOption{ - Name: opts.WorkflowName, - Description: buildWorkflowDescription(inputs), - FilePath: mdFile, - Inputs: inputs, - } - - // Show workflow info if there are inputs - if len(inputs) > 0 { + if len(wf.Inputs) > 0 { showWorkflowInfo(wf) } - // Collect workflow inputs if needed inputValues, err := collectWorkflowInputs(ctx, wf) if err != nil { return fmt.Errorf("failed to collect workflow inputs: %w", err) } - - // Confirm execution (skip if no inputs were collected - user already confirmed they want to run) if len(inputValues) > 0 && !confirmExecution(ctx, wf, inputValues) { fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Workflow execution cancelled")) return nil } - // Build command string for display cmdStr := buildCommandString(opts.WorkflowName, inputValues, opts.RepoOverride, opts.RefOverride, opts.AutoMergePRs, opts.Push, opts.EngineOverride) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("\nRunning workflow...")) - fmt.Fprintln(os.Stderr, console.FormatCommandMessage("Equivalent command: "+cmdStr)) - fmt.Fprintln(os.Stderr, "") + runWorkflowInteractivelyPrintCommand(cmdStr) + if err := runSpecificWorkflowInteractivelyExecute(ctx, opts, inputValues); err != nil { + return err + } + return nil +} + +func runSpecificWorkflowInteractivelyOption(workflowName string) (*WorkflowOption, error) { + workflowsDir := constants.GetWorkflowDir() + mdFile := filepath.Join(workflowsDir, workflowName+".md") + if _, err := os.Stat(mdFile); os.IsNotExist(err) { + return nil, fmt.Errorf("workflow file not found: %s", mdFile) + } + inputs, err := getWorkflowInputs(mdFile) + if err != nil { + runInteractiveLog.Printf("Failed to get inputs for workflow %s: %v", workflowName, err) + inputs = nil + } + return &WorkflowOption{Name: workflowName, Description: buildWorkflowDescription(inputs), FilePath: mdFile, Inputs: inputs}, nil +} - // Execute the workflow - err = RunWorkflowOnGitHub(ctx, opts.WorkflowName, RunOptions{ +func runSpecificWorkflowInteractivelyExecute(ctx context.Context, opts RunWorkflowOptions, inputValues []string) error { + err := RunWorkflowOnGitHub(ctx, opts.WorkflowName, RunOptions{ Enable: false, EngineOverride: opts.EngineOverride, RepoOverride: opts.RepoOverride, @@ -443,7 +444,6 @@ func RunSpecificWorkflowInteractively(ctx context.Context, opts RunWorkflowOptio if err != nil { return fmt.Errorf("failed to run workflow: %w", err) } - return nil } diff --git a/pkg/cli/run_push.go b/pkg/cli/run_push.go index c6633e9d07a..17eab36af6d 100644 --- a/pkg/cli/run_push.go +++ b/pkg/cli/run_push.go @@ -29,44 +29,60 @@ var runPushLog = logger.New("cli:run_push") // regardless of the frontmatter hash status. func collectWorkflowFiles(ctx context.Context, workflowPath string, verbose bool, approve bool) ([]string, error) { runPushLog.Printf("Collecting files for workflow: %s", workflowPath) + files, visited, absWorkflowPath, lockFilePath, err := collectWorkflowFilesInit(workflowPath) + if err != nil { + return nil, err + } - files := make(map[string]struct { // Use map to avoid duplicates - }) - visited := make(map[string]struct { - }) + collectWorkflowFilesLogLockStatus(absWorkflowPath, lockFilePath, verbose, err) + if err := recompileWorkflow(ctx, absWorkflowPath, verbose, approve); err != nil { + runPushLog.Printf("Failed to recompile workflow: %v", err) + return nil, fmt.Errorf("failed to recompile workflow: %w", err) + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Workflow compiled successfully")) + } + runPushLog.Printf("Recompilation completed successfully") + collectWorkflowFilesAddLock(files, lockFilePath, verbose) - // Get absolute path for the workflow + runPushLog.Printf("Starting import collection for %s", absWorkflowPath) + if err := collectImports(absWorkflowPath, files, visited, verbose); err != nil { + runPushLog.Printf("Failed to collect imports: %v", err) + return nil, fmt.Errorf("failed to collect imports: %w", err) + } + result := sliceutil.MapKeys(files) + sort.Strings(result) + runPushLog.Printf("Collected %d files total", len(result)) + return result, nil +} + +func collectWorkflowFilesInit(workflowPath string) (map[string]struct{}, map[string]struct{}, string, string, error) { + files := make(map[string]struct{}) + visited := make(map[string]struct{}) absWorkflowPath, err := filepath.Abs(workflowPath) if err != nil { runPushLog.Printf("Failed to get absolute path for %s: %v", workflowPath, err) - return nil, fmt.Errorf("failed to get absolute path for workflow: %w", err) + return nil, nil, "", "", fmt.Errorf("failed to get absolute path for workflow: %w", err) } runPushLog.Printf("Resolved absolute workflow path: %s", absWorkflowPath) - - // Validate the absolute path absWorkflowPath, err = fileutil.ValidateAbsolutePath(absWorkflowPath) if err != nil { runPushLog.Printf("Invalid workflow path: %v", err) - return nil, fmt.Errorf("invalid workflow path: %w", err) + return nil, nil, "", "", fmt.Errorf("invalid workflow path: %w", err) } - - // Add the workflow .md file - files[absWorkflowPath] = struct { - }{} + files[absWorkflowPath] = struct{}{} runPushLog.Printf("Added workflow file: %s", absWorkflowPath) - - // Check lock file and log hash status for observability lockFilePath := stringutil.MarkdownToLockFile(absWorkflowPath) runPushLog.Printf("Checking lock file: %s", lockFilePath) + return files, visited, absWorkflowPath, lockFilePath, nil +} - // Always recompile, but check and log hash status for observability +func collectWorkflowFilesLogLockStatus(absWorkflowPath, lockFilePath string, verbose bool, err error) { if fileutil.FileExists(lockFilePath) { runPushLog.Printf("Lock file exists: %s", lockFilePath) - // Check frontmatter hash for observability runPushLog.Print("Checking frontmatter hash for observability") if hashMismatch, err := checkFrontmatterHashMismatch(absWorkflowPath, lockFilePath); err != nil { runPushLog.Printf("Error checking frontmatter hash: %v", err) - // Don't fail, just log the error } else if hashMismatch { runPushLog.Print("Lock file frontmatter hash changed (will recompile)") if verbose { @@ -79,7 +95,6 @@ func collectWorkflowFiles(ctx context.Context, workflowPath string, verbose bool } } } else if os.IsNotExist(err) { - // Lock file doesn't exist runPushLog.Printf("Lock file not found: %s", lockFilePath) if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Lock file not found, compiling workflow...")) @@ -90,45 +105,17 @@ func collectWorkflowFiles(ctx context.Context, workflowPath string, verbose bool fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Compiling workflow...")) } } - - // Always recompile (hash check is for observability only) runPushLog.Printf("Always recompiling workflow: %s", absWorkflowPath) - if err := recompileWorkflow(ctx, absWorkflowPath, verbose, approve); err != nil { - runPushLog.Printf("Failed to recompile workflow: %v", err) - return nil, fmt.Errorf("failed to recompile workflow: %w", err) - } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Workflow compiled successfully")) - } - runPushLog.Printf("Recompilation completed successfully") +} - // Add the corresponding .lock.yml file +func collectWorkflowFilesAddLock(files map[string]struct{}, lockFilePath string, verbose bool) { if fileutil.FileExists(lockFilePath) { - files[lockFilePath] = struct { - }{} + files[lockFilePath] = struct{}{} runPushLog.Printf("Added lock file: %s", lockFilePath) } else if verbose { runPushLog.Printf("Lock file not found after compilation: %s", lockFilePath) fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Lock file not found after compilation: "+lockFilePath)) } - - // Collect transitive closure of imported files - runPushLog.Printf("Starting import collection for %s", absWorkflowPath) - if err := collectImports(absWorkflowPath, files, visited, verbose); err != nil { - runPushLog.Printf("Failed to collect imports: %v", err) - return nil, fmt.Errorf("failed to collect imports: %w", err) - } - runPushLog.Printf("Import collection completed") - - // Convert map to slice - result := sliceutil.MapKeys(files) - - // Sort files for stable output - sort.Strings(result) - runPushLog.Printf("Sorted %d files for stable output", len(result)) - - runPushLog.Printf("Collected %d files total", len(result)) - return result, nil } // recompileWorkflow compiles a workflow using CompileWorkflows @@ -230,154 +217,145 @@ func checkLockFileStatus(workflowPath string) (*LockFileStatus, error) { } // collectImports recursively collects all imported files (transitive closure) -func collectImports(workflowPath string, files map[string]struct { -}, visited map[string]struct { -}, verbose bool) error { - // Avoid processing the same file multiple times +func collectImports(workflowPath string, files map[string]struct{}, visited map[string]struct{}, verbose bool) error { if setutil.Contains(visited, workflowPath) { runPushLog.Printf("Skipping already visited file: %s", workflowPath) return nil } - visited[workflowPath] = struct { - }{} - + visited[workflowPath] = struct{}{} runPushLog.Printf("Processing imports for: %s", workflowPath) - // Read the workflow file content, err := os.ReadFile(workflowPath) if err != nil { runPushLog.Printf("Failed to read workflow file %s: %v", workflowPath, err) return fmt.Errorf("failed to read workflow file %s: %w", workflowPath, err) } runPushLog.Printf("Read %d bytes from %s", len(content), workflowPath) - - // Extract frontmatter to get imports field result, err := parser.ExtractFrontmatterFromContent(string(content)) if err != nil { - // No frontmatter is okay - might be a simple file runPushLog.Printf("No frontmatter in %s, skipping imports extraction: %v", workflowPath, err) return nil } runPushLog.Printf("Extracted frontmatter from %s", workflowPath) - - // Get imports from frontmatter importsField, exists := result.Frontmatter["imports"] if !exists { runPushLog.Printf("No imports field in %s", workflowPath) return nil } - runPushLog.Printf("Found imports field in %s", workflowPath) - // Parse imports field - can be array of strings or objects with path, - // or an object with 'aw' subfield for agentic workflow paths. workflowDir := filepath.Dir(workflowPath) - runPushLog.Printf("Workflow directory: %s", workflowDir) - var imports []string - - // extractPathsFromArray extracts file paths from a []any import list. - extractPathsFromArray := func(items []any) []string { - var paths []string - for i, item := range items { - switch importItem := item.(type) { - case string: - runPushLog.Printf("Import %d: string format: %s", i, importItem) - paths = append(paths, importItem) - case map[string]any: - if pathValue, hasPath := importItem["path"]; hasPath { - if pathStr, ok := pathValue.(string); ok { - runPushLog.Printf("Import %d: object format with path: %s", i, pathStr) - paths = append(paths, pathStr) - } else { - runPushLog.Printf("Import %d: object has path but not string type", i) - } - } else { - runPushLog.Printf("Import %d: object missing path field", i) - } - default: - runPushLog.Printf("Import %d: unknown type: %T", i, importItem) - } - } - return paths + imports := collectImportsParseImports(importsField, workflowPath, workflowDir) + if err := collectImportsProcess(imports, workflowDir, files, visited, verbose); err != nil { + return err } + runPushLog.Printf("Finished processing imports for: %s", workflowPath) + return nil +} +func collectImportsParseImports(importsField any, workflowPath, workflowDir string) []string { + runPushLog.Printf("Found imports field in %s", workflowPath) + runPushLog.Printf("Workflow directory: %s", workflowDir) switch v := importsField.(type) { case []any: runPushLog.Printf("Parsing imports as []any with %d items", len(v)) - imports = extractPathsFromArray(v) + return collectImportsExtractPathsFromArray(v) case []string: runPushLog.Printf("Parsing imports as []string with %d items", len(v)) - imports = v + return v case map[string]any: - // Object form: extract paths from the 'aw' subfield. - // The 'apm-packages' subfield contains package names, not file paths. - if awAny, hasAW := v["aw"]; hasAW { - switch aw := awAny.(type) { - case []any: - runPushLog.Printf("Parsing imports.aw as []any with %d items", len(aw)) - imports = extractPathsFromArray(aw) - case []string: - runPushLog.Printf("Parsing imports.aw as []string with %d items", len(aw)) - imports = aw - } - } + return collectImportsParseMap(v) default: runPushLog.Printf("Imports field has unexpected type: %T", v) + return nil } +} - runPushLog.Printf("Found %d imports in %s", len(imports), workflowPath) - - // Process each import - for i, importPath := range imports { - runPushLog.Printf("Processing import %d/%d: %s", i+1, len(imports), importPath) - - // Resolve the import path - resolvedPath := resolveImportPath(importPath, workflowDir, importPathRunPushOpts) - if resolvedPath == "" { - runPushLog.Printf("Could not resolve import path: %s", importPath) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Could not resolve import: "+importPath)) - } - continue +func collectImportsParseMap(v map[string]any) []string { + if awAny, hasAW := v["aw"]; hasAW { + switch aw := awAny.(type) { + case []any: + runPushLog.Printf("Parsing imports.aw as []any with %d items", len(aw)) + return collectImportsExtractPathsFromArray(aw) + case []string: + runPushLog.Printf("Parsing imports.aw as []string with %d items", len(aw)) + return aw } - runPushLog.Printf("Resolved import path: %s -> %s", importPath, resolvedPath) + } + return nil +} - // Get absolute path - var absImportPath string - if filepath.IsAbs(resolvedPath) { - absImportPath = resolvedPath - runPushLog.Printf("Import path is absolute: %s", absImportPath) - } else { - absImportPath = filepath.Join(workflowDir, resolvedPath) - runPushLog.Printf("Joined relative path: %s + %s = %s", workflowDir, resolvedPath, absImportPath) +func collectImportsExtractPathsFromArray(items []any) []string { + var paths []string + for i, item := range items { + switch importItem := item.(type) { + case string: + runPushLog.Printf("Import %d: string format: %s", i, importItem) + paths = append(paths, importItem) + case map[string]any: + paths = collectImportsAppendObjectPath(paths, i, importItem) + default: + runPushLog.Printf("Import %d: unknown type: %T", i, importItem) } + } + return paths +} - // Check if file exists - if _, err := os.Stat(absImportPath); err != nil { - runPushLog.Printf("Import file not found: %s (error: %v)", absImportPath, err) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Import file not found: "+absImportPath)) - } - continue +func collectImportsAppendObjectPath(paths []string, i int, importItem map[string]any) []string { + if pathValue, hasPath := importItem["path"]; hasPath { + if pathStr, ok := pathValue.(string); ok { + runPushLog.Printf("Import %d: object format with path: %s", i, pathStr) + return append(paths, pathStr) } - runPushLog.Printf("Import file exists: %s", absImportPath) - - // Add the import file - files[absImportPath] = struct { - }{} - runPushLog.Printf("Added import file: %s", absImportPath) + runPushLog.Printf("Import %d: object has path but not string type", i) + } else { + runPushLog.Printf("Import %d: object missing path field", i) + } + return paths +} - // Recursively collect imports from this file - runPushLog.Printf("Recursively collecting imports from: %s", absImportPath) - if err := collectImports(absImportPath, files, visited, verbose); err != nil { - runPushLog.Printf("Failed to recursively collect imports from %s: %v", absImportPath, err) +func collectImportsProcess(imports []string, workflowDir string, files map[string]struct{}, visited map[string]struct{}, verbose bool) error { + runPushLog.Printf("Found %d imports", len(imports)) + for i, importPath := range imports { + if err := collectImportsProcessOne(importPath, i, len(imports), workflowDir, files, visited, verbose); err != nil { return err } } - - runPushLog.Printf("Finished processing imports for: %s", workflowPath) return nil } +func collectImportsProcessOne(importPath string, i, total int, workflowDir string, files map[string]struct{}, visited map[string]struct{}, verbose bool) error { + runPushLog.Printf("Processing import %d/%d: %s", i+1, total, importPath) + resolvedPath := resolveImportPath(importPath, workflowDir, importPathRunPushOpts) + if resolvedPath == "" { + runPushLog.Printf("Could not resolve import path: %s", importPath) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Could not resolve import: "+importPath)) + } + return nil + } + absImportPath := collectImportsAbsolutePath(resolvedPath, workflowDir) + if _, err := os.Stat(absImportPath); err != nil { + runPushLog.Printf("Import file not found: %s (error: %v)", absImportPath, err) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Import file not found: "+absImportPath)) + } + return nil + } + files[absImportPath] = struct{}{} + runPushLog.Printf("Added import file: %s", absImportPath) + return collectImports(absImportPath, files, visited, verbose) +} + +func collectImportsAbsolutePath(resolvedPath, workflowDir string) string { + if filepath.IsAbs(resolvedPath) { + runPushLog.Printf("Import path is absolute: %s", resolvedPath) + return resolvedPath + } + absImportPath := filepath.Join(workflowDir, resolvedPath) + runPushLog.Printf("Joined relative path: %s + %s = %s", workflowDir, resolvedPath, absImportPath) + return absImportPath +} + // pushWorkflowFiles commits and pushes the workflow files to the repository func pushWorkflowFiles(ctx context.Context, workflowName string, files []string, refOverride string, verbose bool) error { if ctx == nil { @@ -385,88 +363,116 @@ func pushWorkflowFiles(ctx context.Context, workflowName string, files []string, } runPushLog.Printf("Pushing %d files for workflow: %s", len(files), workflowName) runPushLog.Printf("Files to push: %v", files) + pushWorkflowFilesPrintStaging(files, verbose) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Staging %d files for commit", len(files)))) - for _, file := range files { - fmt.Fprintf(os.Stderr, " - %s\n", file) + statusOutput, err := pushWorkflowFilesStageAndStatus(ctx, files, verbose) + if err != nil { + return err + } + if strings.TrimSpace(string(statusOutput)) == "" { + runPushLog.Printf("No staged changes detected") + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No changes to commit")) } + runPushLog.Print("No changes to commit") + return nil + } + if err := pushWorkflowFilesVerifyRef(refOverride, verbose); err != nil { + return err } - // Stage all files + stagedFiles := strings.Split(strings.TrimSpace(string(statusOutput)), "\n") + runPushLog.Printf("Found %d staged files: %v", len(stagedFiles), stagedFiles) + if err := pushWorkflowFilesCheckExtra(files, stagedFiles); err != nil { + return err + } + commitMessage := "Updated agentic workflow " + workflowName + if err := pushWorkflowFilesConfirm(files, commitMessage); err != nil { + return err + } + return pushWorkflowFilesCommitAndPush(ctx, commitMessage, verbose) +} + +func pushWorkflowFilesPrintStaging(files []string, verbose bool) { + if !verbose { + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Staging %d files for commit", len(files)))) + for _, file := range files { + fmt.Fprintf(os.Stderr, " - %s\n", file) + } +} + +func pushWorkflowFilesStageAndStatus(ctx context.Context, files []string, verbose bool) ([]byte, error) { gitArgs := append([]string{"add"}, files...) runPushLog.Printf("Executing git command: git %v", gitArgs) cmd := exec.CommandContext(ctx, "git", gitArgs...) if output, err := cmd.CombinedOutput(); err != nil { runPushLog.Printf("Failed to stage files: %v, output: %s", err, string(output)) - return fmt.Errorf("failed to stage files: %w\nOutput: %s", err, string(output)) + return nil, fmt.Errorf("failed to stage files: %w\nOutput: %s", err, string(output)) } runPushLog.Printf("Successfully staged %d files", len(files)) - if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Files staged successfully")) } - // Check if there are any staged files in git (after we've staged our files) runPushLog.Printf("Checking staged files with git diff --cached --name-only") statusCmd := exec.CommandContext(ctx, "git", "diff", "--cached", "--name-only") statusOutput, err := statusCmd.CombinedOutput() if err != nil { runPushLog.Printf("Failed to check git status: %v, output: %s", err, string(statusOutput)) - return fmt.Errorf("failed to check git status: %w\nOutput: %s", err, string(statusOutput)) + return nil, fmt.Errorf("failed to check git status: %w\nOutput: %s", err, string(statusOutput)) } runPushLog.Printf("Git status output: %s", string(statusOutput)) + return statusOutput, nil +} - // Check if there are no staged changes (nothing to commit) - if strings.TrimSpace(string(statusOutput)) == "" { - runPushLog.Printf("No staged changes detected") - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No changes to commit")) - } - runPushLog.Print("No changes to commit") +func pushWorkflowFilesVerifyRef(refOverride string, verbose bool) error { + if refOverride == "" { return nil } + runPushLog.Printf("Checking if current branch matches --ref value: %s", refOverride) + currentBranch, err := getCurrentBranch() + if err != nil { + runPushLog.Printf("Failed to determine current branch: %v", err) + return fmt.Errorf("failed to determine current branch: %w", err) + } + if currentBranch != refOverride { + runPushLog.Printf("Current branch (%s) does not match --ref value (%s)", currentBranch, refOverride) + return fmt.Errorf("--push requires the current branch (%s) to match the --ref value (%s). Switching branches is not supported. Please checkout the target branch first", currentBranch, refOverride) + } + runPushLog.Printf("Current branch matches --ref value: %s", currentBranch) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Verified current branch matches --ref: "+currentBranch)) + } + return nil +} - // Now that we know there are changes to commit, check that current branch matches --ref value if specified - // This happens after we've determined there are actual changes, so we don't fail unnecessarily - if refOverride != "" { - runPushLog.Printf("Checking if current branch matches --ref value: %s", refOverride) - currentBranch, err := getCurrentBranch() - if err != nil { - runPushLog.Printf("Failed to determine current branch: %v", err) - return fmt.Errorf("failed to determine current branch: %w", err) - } - runPushLog.Printf("Current branch: %s", currentBranch) - - if currentBranch != refOverride { - runPushLog.Printf("Current branch (%s) does not match --ref value (%s)", currentBranch, refOverride) - return fmt.Errorf("--push requires the current branch (%s) to match the --ref value (%s). Switching branches is not supported. Please checkout the target branch first", currentBranch, refOverride) - } - - runPushLog.Printf("Current branch matches --ref value: %s", currentBranch) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Verified current branch matches --ref: "+currentBranch)) +func pushWorkflowFilesCheckExtra(files, stagedFiles []string) error { + ourFiles := pushWorkflowFilesMap(files) + var extraStagedFiles []string + for _, stagedFile := range stagedFiles { + if pushWorkflowFilesContainsStaged(ourFiles, stagedFile) { + continue } + runPushLog.Printf("Extra staged file detected: %s", stagedFile) + extraStagedFiles = append(extraStagedFiles, stagedFile) } + if len(extraStagedFiles) == 0 { + runPushLog.Printf("No extra staged files detected - all staged files are part of our workflow") + return nil + } + pushWorkflowFilesPrintExtra(extraStagedFiles) + return errors.New("git has staged files not part of workflow - commit or unstage them before using --push") +} - // Get the list of staged files - stagedFiles := strings.Split(strings.TrimSpace(string(statusOutput)), "\n") - runPushLog.Printf("Found %d staged files: %v", len(stagedFiles), stagedFiles) - - // Check if there are staged files beyond what we just staged - // Convert our files list to a map for quick lookup +func pushWorkflowFilesMap(files []string) map[string]struct{} { runPushLog.Printf("Building map of our files for comparison") - ourFiles := make(map[string]struct { - }) + ourFiles := make(map[string]struct{}) for _, file := range files { - // Normalize the path - absPath, err := filepath.Abs(file) - if err == nil { - // Validate the absolute path - validPath, validErr := fileutil.ValidateAbsolutePath(absPath) - if validErr == nil { - ourFiles[validPath] = struct { - }{} + if absPath, err := filepath.Abs(file); err == nil { + if validPath, validErr := fileutil.ValidateAbsolutePath(absPath); validErr == nil { + ourFiles[validPath] = struct{}{} runPushLog.Printf("Added to our files map: %s (absolute: %s)", file, validPath) } else { runPushLog.Printf("Failed to validate path for %s: %v", absPath, validErr) @@ -474,58 +480,43 @@ func pushWorkflowFiles(ctx context.Context, workflowName string, files []string, } else { runPushLog.Printf("Failed to get absolute path for %s: %v", file, err) } - ourFiles[file] = struct { - }{} + ourFiles[file] = struct{}{} runPushLog.Printf("Added to our files map: %s", file) } + return ourFiles +} - // Check if there are any staged files that aren't in our list - runPushLog.Printf("Checking for extra staged files not in our list") - var extraStagedFiles []string - for _, stagedFile := range stagedFiles { - runPushLog.Printf("Checking staged file: %s", stagedFile) - // Try both absolute and relative paths - absStagedPath, err := filepath.Abs(stagedFile) - if err == nil { - // Validate the staged path - validPath, validErr := fileutil.ValidateAbsolutePath(absStagedPath) - if validErr == nil && setutil.Contains(ourFiles, validPath) { - runPushLog.Printf("Staged file %s matches our file %s (absolute)", stagedFile, validPath) - continue - } - } - if setutil.Contains(ourFiles, stagedFile) { - runPushLog.Printf("Staged file %s matches our file (relative)", stagedFile) - continue +func pushWorkflowFilesContainsStaged(ourFiles map[string]struct{}, stagedFile string) bool { + runPushLog.Printf("Checking staged file: %s", stagedFile) + if absStagedPath, err := filepath.Abs(stagedFile); err == nil { + if validPath, validErr := fileutil.ValidateAbsolutePath(absStagedPath); validErr == nil && setutil.Contains(ourFiles, validPath) { + runPushLog.Printf("Staged file %s matches our file %s (absolute)", stagedFile, validPath) + return true } - runPushLog.Printf("Extra staged file detected: %s", stagedFile) - extraStagedFiles = append(extraStagedFiles, stagedFile) } + if setutil.Contains(ourFiles, stagedFile) { + runPushLog.Printf("Staged file %s matches our file (relative)", stagedFile) + return true + } + return false +} - // If there are extra staged files that we didn't stage, give up - if len(extraStagedFiles) > 0 { - runPushLog.Printf("Found %d extra staged files not in our list, refusing to proceed: %v", len(extraStagedFiles), extraStagedFiles) - - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatErrorMessage("Cannot proceed: there are already staged files in git that are not part of this workflow")) - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Extra staged files:")) - for _, file := range extraStagedFiles { - fmt.Fprintf(os.Stderr, " - %s\n", file) - } - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please commit or unstage these files before using --push")) - fmt.Fprintln(os.Stderr, "") - - return errors.New("git has staged files not part of workflow - commit or unstage them before using --push") +func pushWorkflowFilesPrintExtra(extraStagedFiles []string) { + runPushLog.Printf("Found %d extra staged files not in our list, refusing to proceed: %v", len(extraStagedFiles), extraStagedFiles) + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatErrorMessage("Cannot proceed: there are already staged files in git that are not part of this workflow")) + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Extra staged files:")) + for _, file := range extraStagedFiles { + fmt.Fprintf(os.Stderr, " - %s\n", file) } - runPushLog.Printf("No extra staged files detected - all staged files are part of our workflow") + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please commit or unstage these files before using --push")) + fmt.Fprintln(os.Stderr, "") +} - // Create commit message - commitMessage := "Updated agentic workflow " + workflowName +func pushWorkflowFilesConfirm(files []string, commitMessage string) error { runPushLog.Printf("Creating commit with message: %s", commitMessage) - - // Show what will be committed and ask for confirmation using console helper fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Ready to commit and push the following files:")) for _, file := range files { @@ -534,52 +525,38 @@ func pushWorkflowFiles(ctx context.Context, workflowName string, files []string, fmt.Fprintln(os.Stderr, "") fmt.Fprintf(os.Stderr, console.FormatInfoMessage("Commit message: %s\n"), commitMessage) fmt.Fprintln(os.Stderr, "") - - // Ask for confirmation using console helper - runPushLog.Printf("Requesting user confirmation for commit and push") - confirmed, err := console.ConfirmAction( - "Do you want to commit and push these changes?", - "Yes, commit and push", - "No, cancel", - ) + confirmed, err := console.ConfirmAction("Do you want to commit and push these changes?", "Yes, commit and push", "No, cancel") if err != nil { runPushLog.Printf("Confirmation failed: %v", err) return fmt.Errorf("confirmation failed: %w", err) } - if !confirmed { runPushLog.Print("Push cancelled by user") return errors.New("push cancelled by user") } runPushLog.Printf("User confirmed - proceeding with commit and push") + return nil +} - // Commit the changes +func pushWorkflowFilesCommitAndPush(ctx context.Context, commitMessage string, verbose bool) error { runPushLog.Printf("Executing git commit with message: %s", commitMessage) - cmd = exec.CommandContext(ctx, "git", "commit", "-m", commitMessage) + cmd := exec.CommandContext(ctx, "git", "commit", "-m", commitMessage) if output, err := cmd.CombinedOutput(); err != nil { runPushLog.Printf("Failed to commit: %v, output: %s", err, string(output)) return fmt.Errorf("failed to commit changes: %w\nOutput: %s", err, string(output)) } - runPushLog.Printf("Commit successful") - if verbose { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Changes committed successfully")) } - - // Push the changes runPushLog.Print("Pushing changes to remote") - runPushLog.Printf("Executing git push") cmd = exec.CommandContext(ctx, "git", "push") if output, err := cmd.CombinedOutput(); err != nil { runPushLog.Printf("Failed to push: %v, output: %s", err, string(output)) return fmt.Errorf("failed to push changes: %w\nOutput: %s", err, string(output)) } - runPushLog.Printf("Push to remote successful") - if verbose { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Changes pushed to remote")) } - runPushLog.Print("Push completed successfully") return nil } diff --git a/pkg/cli/run_workflow_execution.go b/pkg/cli/run_workflow_execution.go index c6bb5231cac..5385fa5ccba 100644 --- a/pkg/cli/run_workflow_execution.go +++ b/pkg/cli/run_workflow_execution.go @@ -58,151 +58,162 @@ type WorkflowRunResult struct { // RunWorkflowOnGitHub runs an agentic workflow on GitHub Actions func RunWorkflowOnGitHub(ctx context.Context, workflowIdOrName string, opts RunOptions) error { executionLog.Printf("Starting workflow run: workflow=%s, enable=%v, engineOverride=%s, repo=%s, ref=%s, push=%v, wait=%v, inputs=%v", workflowIdOrName, opts.Enable, opts.EngineOverride, opts.RepoOverride, opts.RefOverride, opts.Push, opts.WaitForCompletion, opts.Inputs) + if err := runWorkflowOnGitHubValidateStart(ctx, workflowIdOrName, opts); err != nil { + return err + } + if err := runWorkflowOnGitHubValidateWorkflow(workflowIdOrName, opts); err != nil { + return err + } + + wasDisabled, workflowID, err := runWorkflowOnGitHubEnableWorkflow(workflowIdOrName, opts) + if err != nil { + return err + } + lockFileName, lockFilePath, err := runWorkflowOnGitHubResolveLockFile(workflowIdOrName, opts) + if err != nil { + return err + } + if err := runWorkflowOnGitHubPrepareLocal(ctx, workflowIdOrName, lockFileName, lockFilePath, opts); err != nil { + return err + } + args, ref := runWorkflowOnGitHubBuildArgs(lockFileName, opts) + workflowStartTime := time.Now() + + if opts.DryRun { + return runWorkflowOnGitHubDryRun(workflowIdOrName, lockFileName, ref, opts, wasDisabled, workflowID) + } + runInfo, runErr, err := runWorkflowOnGitHubTrigger(ctx, workflowIdOrName, lockFileName, args, ref, opts, wasDisabled, workflowID) + if err != nil { + return err + } + if err := runWorkflowOnGitHubAfterTrigger(ctx, workflowIdOrName, runInfo, runErr, workflowStartTime, opts); err != nil { + return err + } + if opts.Enable && wasDisabled && workflowID != 0 { + restoreWorkflowState(workflowIdOrName, workflowID, opts.RepoOverride, opts.Verbose) + } + return nil +} - // Check context cancellation at the start +func runWorkflowOnGitHubValidateStart(ctx context.Context, workflowIdOrName string, opts RunOptions) error { select { case <-ctx.Done(): fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) return ctx.Err() default: } - if workflowIdOrName == "" { return errors.New("workflow name or ID is required") } - - // Validate input format early before attempting workflow validation for _, input := range opts.Inputs { if !strings.Contains(input, "=") { return fmt.Errorf("invalid input format '%s': expected key=value", input) } - // Check that key (before '=') is not empty parts := strings.SplitN(input, "=", 2) if parts[0] == "" { return fmt.Errorf("invalid input format '%s': key cannot be empty", input) } } - if opts.Verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Running workflow on GitHub Actions: "+workflowIdOrName)) } - - // Check if gh CLI is available if !isGHCLIAvailable() { return errors.New("GitHub CLI (gh) is required but not available") } + return nil +} - // Validate workflow exists and is runnable +func runWorkflowOnGitHubValidateWorkflow(workflowIdOrName string, opts RunOptions) error { if opts.RepoOverride != "" { executionLog.Printf("Validating remote workflow: %s in repo %s", workflowIdOrName, opts.RepoOverride) - // For remote repositories, use remote validation if err := validateRemoteWorkflow(workflowIdOrName, opts.RepoOverride, opts.Verbose); err != nil { return fmt.Errorf("failed to validate remote workflow: %w", err) } - // Note: We skip local runnable check for remote workflows as we assume they are properly configured - } else { - executionLog.Printf("Validating local workflow: %s", workflowIdOrName) - // For local workflows, use existing local validation - workflowFile, err := resolveWorkflowFile(workflowIdOrName, opts.Verbose) - if err != nil { - // Return error directly without wrapping - it already contains formatted message with suggestions - return err - } - - // Check if the workflow is runnable (has workflow_dispatch trigger) - runnable, err := IsRunnable(workflowFile) - if err != nil { - return fmt.Errorf("failed to check if workflow %s is runnable: %w", workflowFile, err) - } - - if !runnable { - return fmt.Errorf("workflow '%s' cannot be run on GitHub Actions - it must have 'workflow_dispatch' trigger", workflowIdOrName) - } - executionLog.Printf("Workflow is runnable: %s", workflowFile) - - // Validate workflow inputs - if err := validateWorkflowInputs(workflowFile, opts.Inputs); err != nil { - return fmt.Errorf("%w", err) - } - - // Check if the workflow file has local modifications - if status, err := checkWorkflowFileStatus(workflowFile); err == nil && status != nil { - var warnings []string - - if status.IsModified { - warnings = append(warnings, "The workflow file has unstaged changes") - } - if status.IsStaged { - warnings = append(warnings, "The workflow file has staged changes") - } - if status.HasUnpushedCommits { - warnings = append(warnings, "The workflow file has unpushed commits") - } - - if len(warnings) > 0 { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(strings.Join(warnings, ", "))) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("These changes will not be reflected in the GitHub Actions run")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Consider pushing your changes before running the workflow")) - } - } + return nil + } + executionLog.Printf("Validating local workflow: %s", workflowIdOrName) + workflowFile, err := resolveWorkflowFile(workflowIdOrName, opts.Verbose) + if err != nil { + return err + } + runnable, err := IsRunnable(workflowFile) + if err != nil { + return fmt.Errorf("failed to check if workflow %s is runnable: %w", workflowFile, err) } + if !runnable { + return fmt.Errorf("workflow '%s' cannot be run on GitHub Actions - it must have 'workflow_dispatch' trigger", workflowIdOrName) + } + executionLog.Printf("Workflow is runnable: %s", workflowFile) + if err := validateWorkflowInputs(workflowFile, opts.Inputs); err != nil { + return fmt.Errorf("%w", err) + } + runWorkflowOnGitHubWarnLocalStatus(workflowFile) + return nil +} - // Handle --enable flag logic: check workflow state and enable if needed - var wasDisabled bool - var workflowID int64 - if opts.Enable { - // Get current workflow status - wf, err := getWorkflowStatus(workflowIdOrName, opts.RepoOverride, opts.Verbose) - if err != nil { - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not check workflow status: %v", err))) - } - } +func runWorkflowOnGitHubWarnLocalStatus(workflowFile string) { + status, err := checkWorkflowFileStatus(workflowFile) + if err != nil || status == nil { + return + } + var warnings []string + if status.IsModified { + warnings = append(warnings, "The workflow file has unstaged changes") + } + if status.IsStaged { + warnings = append(warnings, "The workflow file has staged changes") + } + if status.HasUnpushedCommits { + warnings = append(warnings, "The workflow file has unpushed commits") + } + if len(warnings) > 0 { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(strings.Join(warnings, ", "))) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("These changes will not be reflected in the GitHub Actions run")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Consider pushing your changes before running the workflow")) + } +} - // If we successfully got workflow status, check if it needs enabling - if err == nil { - workflowID = wf.ID - if wf.State == "disabled_manually" { - wasDisabled = true - executionLog.Printf("Workflow %s is disabled, temporarily enabling for this run (id=%d)", workflowIdOrName, wf.ID) - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow '%s' is disabled, enabling it temporarily...", workflowIdOrName))) - } - // Enable the workflow - enableArgs := []string{"workflow", "enable", strconv.FormatInt(wf.ID, 10)} - if opts.RepoOverride != "" { - enableArgs = append(enableArgs, "--repo", opts.RepoOverride) - } - cmd := workflow.ExecGH(enableArgs...) - if err := cmd.Run(); err != nil { - executionLog.Printf("Failed to enable workflow %s: %v", workflowIdOrName, err) - return fmt.Errorf("failed to enable workflow '%s': %w", workflowIdOrName, err) - } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Enabled workflow: "+workflowIdOrName)) - } else { - executionLog.Printf("Workflow %s is already enabled (state=%s)", workflowIdOrName, wf.State) - } +func runWorkflowOnGitHubEnableWorkflow(workflowIdOrName string, opts RunOptions) (bool, int64, error) { + if !opts.Enable { + return false, 0, nil + } + wf, err := getWorkflowStatus(workflowIdOrName, opts.RepoOverride, opts.Verbose) + if err != nil { + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not check workflow status: %v", err))) } + return false, 0, nil + } + if wf.State != "disabled_manually" { + executionLog.Printf("Workflow %s is already enabled (state=%s)", workflowIdOrName, wf.State) + return false, wf.ID, nil + } + executionLog.Printf("Workflow %s is disabled, temporarily enabling for this run (id=%d)", workflowIdOrName, wf.ID) + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow '%s' is disabled, enabling it temporarily...", workflowIdOrName))) + } + enableArgs := []string{"workflow", "enable", strconv.FormatInt(wf.ID, 10)} + if opts.RepoOverride != "" { + enableArgs = append(enableArgs, "--repo", opts.RepoOverride) } + if err := workflow.ExecGH(enableArgs...).Run(); err != nil { + executionLog.Printf("Failed to enable workflow %s: %v", workflowIdOrName, err) + return false, 0, fmt.Errorf("failed to enable workflow '%s': %w", workflowIdOrName, err) + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Enabled workflow: "+workflowIdOrName)) + return true, wf.ID, nil +} - // Normalize workflow ID to handle both \"workflow-name\" and \".github/workflows/workflow-name.md\" formats +func runWorkflowOnGitHubResolveLockFile(workflowIdOrName string, opts RunOptions) (string, string, error) { normalizedID := normalizeWorkflowID(workflowIdOrName) - - // Construct lock file name from normalized ID (same for both local and remote) lockFileName := normalizedID + ".lock.yml" - - // For local workflows, validate the workflow exists and check for lock file var lockFilePath string if opts.RepoOverride == "" { - // For local workflows, validate the workflow exists locally workflowsDir := getWorkflowsDir() - _, _, err := readWorkflowFile(normalizedID+".md", workflowsDir) if err != nil { - return fmt.Errorf("failed to find workflow in local %s: %w", workflowsDir, err) + return "", "", fmt.Errorf("failed to find workflow in local %s: %w", workflowsDir, err) } - - // Check if the lock file exists in the workflows directory lockFilePath = filepath.Join(constants.GetWorkflowDir(), lockFileName) if _, err := os.Stat(lockFilePath); os.IsNotExist(err) { executionLog.Printf("Lock file not found: %s (workflow must be compiled first)", lockFilePath) @@ -210,104 +221,89 @@ func RunWorkflowOnGitHub(ctx context.Context, workflowIdOrName string, opts RunO fmt.Sprintf("Run '%s compile' to compile all workflows", string(constants.CLIExtensionPrefix)), fmt.Sprintf("Run '%s compile %s' to compile this specific workflow", string(constants.CLIExtensionPrefix), normalizedID), } - errMsg := console.FormatErrorWithSuggestions( - fmt.Sprintf("workflow lock file '%s' not found in %s", lockFileName, constants.GetWorkflowDir()), - suggestions, - ) - return errors.New(errMsg) + errMsg := console.FormatErrorWithSuggestions(fmt.Sprintf("workflow lock file '%s' not found in %s", lockFileName, constants.GetWorkflowDir()), suggestions) + return "", "", errors.New(errMsg) } executionLog.Printf("Found lock file: %s", lockFilePath) } + return lockFileName, lockFilePath, nil +} - // Recompile workflow if engine override is provided (only for local workflows) +func runWorkflowOnGitHubPrepareLocal(ctx context.Context, workflowIdOrName, lockFileName, lockFilePath string, opts RunOptions) error { if opts.EngineOverride != "" && opts.RepoOverride == "" { - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Recompiling workflow with engine override: "+opts.EngineOverride)) - } - - workflowMarkdownPath := stringutil.LockFileToMarkdown(lockFilePath) - config := CompileConfig{ - MarkdownFiles: []string{workflowMarkdownPath}, - Verbose: opts.Verbose, - EngineOverride: opts.EngineOverride, - Validate: true, - Watch: false, - WorkflowDir: "", - SkipInstructions: false, - NoEmit: false, - Purge: false, - TrialMode: false, - TrialLogicalRepoSlug: "", - Strict: false, - } - if _, err := CompileWorkflows(ctx, config); err != nil { - return fmt.Errorf("failed to recompile workflow with engine override: %w", err) - } - - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Successfully recompiled workflow with engine: "+opts.EngineOverride)) - } - } else if opts.EngineOverride != "" && opts.RepoOverride != "" { - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Note: Engine override ignored for remote repository workflows")) + if err := runWorkflowOnGitHubRecompileWithEngine(ctx, lockFilePath, opts); err != nil { + return err } + } else if opts.EngineOverride != "" && opts.RepoOverride != "" && opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Note: Engine override ignored for remote repository workflows")) } - if opts.Verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Using lock file: "+lockFileName)) } - - // Check for missing or outdated lock files (when not using --push) - if !opts.Push && opts.RepoOverride == "" { - workflowMarkdownPath := stringutil.LockFileToMarkdown(lockFilePath) - if status, err := checkLockFileStatus(workflowMarkdownPath); err == nil { - if status.Missing { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Lock file is missing")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run 'gh aw run %s --push' to automatically compile and push the lock file", workflowIdOrName))) - } else if status.Outdated { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Lock file is outdated (workflow file is newer)")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run 'gh aw run %s --push' to automatically compile and push the lock file", workflowIdOrName))) - } - } - } - - // Handle --push flag: commit and push workflow files before running + runWorkflowOnGitHubWarnLockStatus(workflowIdOrName, lockFilePath, opts) if opts.Push { - // Only valid for local workflows - if opts.RepoOverride != "" { - return errors.New("--push flag is only supported for local workflows, not remote repositories") - } - - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Collecting workflow files for push...")) - } + return runWorkflowOnGitHubPushFiles(ctx, workflowIdOrName, lockFilePath, opts) + } + return nil +} - // Collect the workflow .md file, .lock.yml file, and transitive imports - workflowMarkdownPath := stringutil.LockFileToMarkdown(lockFilePath) - files, err := collectWorkflowFiles(ctx, workflowMarkdownPath, opts.Verbose, opts.Approve) - if err != nil { - return fmt.Errorf("failed to collect workflow files: %w", err) - } +func runWorkflowOnGitHubRecompileWithEngine(ctx context.Context, lockFilePath string, opts RunOptions) error { + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Recompiling workflow with engine override: "+opts.EngineOverride)) + } + workflowMarkdownPath := stringutil.LockFileToMarkdown(lockFilePath) + config := CompileConfig{MarkdownFiles: []string{workflowMarkdownPath}, Verbose: opts.Verbose, EngineOverride: opts.EngineOverride, Validate: true, Strict: false} + if _, err := CompileWorkflows(ctx, config); err != nil { + return fmt.Errorf("failed to recompile workflow with engine override: %w", err) + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Successfully recompiled workflow with engine: "+opts.EngineOverride)) + } + return nil +} - // Commit and push the files (includes branch verification if --ref is specified) - if err := pushWorkflowFiles(ctx, workflowIdOrName, files, opts.RefOverride, opts.Verbose); err != nil { - return fmt.Errorf("failed to push workflow files: %w", err) +func runWorkflowOnGitHubWarnLockStatus(workflowIdOrName, lockFilePath string, opts RunOptions) { + if opts.Push || opts.RepoOverride != "" { + return + } + workflowMarkdownPath := stringutil.LockFileToMarkdown(lockFilePath) + if status, err := checkLockFileStatus(workflowMarkdownPath); err == nil { + if status.Missing { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Lock file is missing")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run 'gh aw run %s --push' to automatically compile and push the lock file", workflowIdOrName))) + } else if status.Outdated { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Lock file is outdated (workflow file is newer)")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Run 'gh aw run %s --push' to automatically compile and push the lock file", workflowIdOrName))) } + } +} - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Successfully pushed %d file(s) for workflow %s", len(files), workflowIdOrName))) +func runWorkflowOnGitHubPushFiles(ctx context.Context, workflowIdOrName, lockFilePath string, opts RunOptions) error { + if opts.RepoOverride != "" { + return errors.New("--push flag is only supported for local workflows, not remote repositories") + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Collecting workflow files for push...")) + } + workflowMarkdownPath := stringutil.LockFileToMarkdown(lockFilePath) + files, err := collectWorkflowFiles(ctx, workflowMarkdownPath, opts.Verbose, opts.Approve) + if err != nil { + return fmt.Errorf("failed to collect workflow files: %w", err) } + if err := pushWorkflowFiles(ctx, workflowIdOrName, files, opts.RefOverride, opts.Verbose); err != nil { + return fmt.Errorf("failed to push workflow files: %w", err) + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Successfully pushed %d file(s) for workflow %s", len(files), workflowIdOrName))) + return nil +} - // Build the gh workflow run command with optional repo and ref overrides +func runWorkflowOnGitHubBuildArgs(lockFileName string, opts RunOptions) ([]string, string) { args := []string{"workflow", "run", lockFileName} if opts.RepoOverride != "" { args = append(args, "--repo", opts.RepoOverride) } - - // Determine the ref to use (branch/tag) - // If refOverride is specified, use it; otherwise for local workflows, use current branch ref := opts.RefOverride if ref == "" && opts.RepoOverride == "" { - // For local workflows without explicit ref, use the current branch if currentBranch, err := getCurrentBranch(); err == nil { ref = currentBranch executionLog.Printf("Using current branch for workflow run: %s", ref) @@ -318,106 +314,76 @@ func RunWorkflowOnGitHub(ctx context.Context, workflowIdOrName string, opts RunO if ref != "" { args = append(args, "--ref", ref) } - - // Add workflow inputs if provided - if len(opts.Inputs) > 0 { - for _, input := range opts.Inputs { - // Add as raw field flag to gh workflow run - args = append(args, "-f", input) - } + for _, input := range opts.Inputs { + args = append(args, "-f", input) } + return args, ref +} - // Record the start time for auto-merge PR filtering - workflowStartTime := time.Now() - - // Handle dry-run mode: validate everything but skip actual execution - if opts.DryRun { - if opts.Verbose { - var cmdParts []string - cmdParts = append(cmdParts, "gh workflow run", lockFileName) - if opts.RepoOverride != "" { - cmdParts = append(cmdParts, "--repo", opts.RepoOverride) - } - if ref != "" { - cmdParts = append(cmdParts, "--ref", ref) - } - if len(opts.Inputs) > 0 { - for _, input := range opts.Inputs { - cmdParts = append(cmdParts, "-f", input) - } - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Dry run mode - command that would be executed:")) - fmt.Fprintln(os.Stderr, console.FormatCommandMessage(strings.Join(cmdParts, " "))) - } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Validation passed for workflow: %s (dry run - not executed)", lockFileName))) - - // Restore workflow state if it was disabled and we enabled it - if opts.Enable && wasDisabled && workflowID != 0 { - restoreWorkflowState(workflowIdOrName, workflowID, opts.RepoOverride, opts.Verbose) - } +func runWorkflowOnGitHubDryRun(workflowIdOrName, lockFileName, ref string, opts RunOptions, wasDisabled bool, workflowID int64) error { + if opts.Verbose { + cmdParts := runWorkflowOnGitHubCommandParts(lockFileName, ref, opts) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Dry run mode - command that would be executed:")) + fmt.Fprintln(os.Stderr, console.FormatCommandMessage(strings.Join(cmdParts, " "))) + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Validation passed for workflow: %s (dry run - not executed)", lockFileName))) + if opts.Enable && wasDisabled && workflowID != 0 { + restoreWorkflowState(workflowIdOrName, workflowID, opts.RepoOverride, opts.Verbose) + } + return nil +} - return nil +func runWorkflowOnGitHubCommandParts(lockFileName, ref string, opts RunOptions) []string { + cmdParts := []string{"gh workflow run", lockFileName} + if opts.RepoOverride != "" { + cmdParts = append(cmdParts, "--repo", opts.RepoOverride) } + if ref != "" { + cmdParts = append(cmdParts, "--ref", ref) + } + for _, input := range opts.Inputs { + cmdParts = append(cmdParts, "-f", input) + } + return cmdParts +} - // Execute gh workflow run command and capture output +func runWorkflowOnGitHubTrigger(ctx context.Context, workflowIdOrName, lockFileName string, args []string, ref string, opts RunOptions, wasDisabled bool, workflowID int64) (*WorkflowRunInfo, error, error) { cmd := workflow.ExecGHContext(ctx, args...) - if opts.Verbose { - var cmdParts []string - cmdParts = append(cmdParts, "gh workflow run", lockFileName) - if opts.RepoOverride != "" { - cmdParts = append(cmdParts, "--repo", opts.RepoOverride) - } - if ref != "" { - cmdParts = append(cmdParts, "--ref", ref) - } - if len(opts.Inputs) > 0 { - for _, input := range opts.Inputs { - cmdParts = append(cmdParts, "-f", input) - } - } - fmt.Fprintln(os.Stderr, console.FormatCommandMessage(strings.Join(cmdParts, " "))) + fmt.Fprintln(os.Stderr, console.FormatCommandMessage(strings.Join(runWorkflowOnGitHubCommandParts(lockFileName, ref, opts), " "))) } - - // Capture both stdout and stderr stdout, err := cmd.Output() if err != nil { - // If there's an error, try to get stderr for better error reporting - var stderrOutput string - var exitError *exec.ExitError - if errors.As(err, &exitError) { - stderrOutput = string(exitError.Stderr) - fmt.Fprintf(os.Stderr, "%s", exitError.Stderr) - } - - // Restore workflow state if it was disabled and we enabled it (even on error) - if opts.Enable && wasDisabled && workflowID != 0 { - restoreWorkflowState(workflowIdOrName, workflowID, opts.RepoOverride, opts.Verbose) - } - - // Check if this is a permission error in a codespace - errorMsg := err.Error() + " " + stderrOutput - if isRunningInCodespace() && is403PermissionError(errorMsg) { - // Show specialized error message for codespace users - fmt.Fprint(os.Stderr, getCodespacePermissionErrorMessage()) - return errors.New("failed to run workflow on GitHub Actions: permission denied (403)") - } - - return fmt.Errorf("failed to run workflow on GitHub Actions: %w", err) + return nil, nil, runWorkflowOnGitHubHandleTriggerError(err, workflowIdOrName, opts, wasDisabled, workflowID) } - - // Display the output from gh workflow run output := strings.TrimSpace(string(stdout)) if output != "" { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(output)) } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Successfully triggered workflow: "+lockFileName)) executionLog.Printf("Workflow triggered successfully: %s", lockFileName) + return runWorkflowOnGitHubRunInfo(lockFileName, opts, output) +} + +func runWorkflowOnGitHubHandleTriggerError(err error, workflowIdOrName string, opts RunOptions, wasDisabled bool, workflowID int64) error { + var stderrOutput string + var exitError *exec.ExitError + if errors.As(err, &exitError) { + stderrOutput = string(exitError.Stderr) + fmt.Fprintf(os.Stderr, "%s", exitError.Stderr) + } + if opts.Enable && wasDisabled && workflowID != 0 { + restoreWorkflowState(workflowIdOrName, workflowID, opts.RepoOverride, opts.Verbose) + } + errorMsg := err.Error() + " " + stderrOutput + if isRunningInCodespace() && is403PermissionError(errorMsg) { + fmt.Fprint(os.Stderr, getCodespacePermissionErrorMessage()) + return errors.New("failed to run workflow on GitHub Actions: permission denied (403)") + } + return fmt.Errorf("failed to run workflow on GitHub Actions: %w", err) +} - // Try to get run info: first attempt to parse from gh workflow run output (new in v2.87+), - // then fall back to polling with getLatestWorkflowRunWithRetry for older gh CLI versions. - // Parsing failure is expected for older gh CLI versions and the fallback ensures backward compatibility. +func runWorkflowOnGitHubRunInfo(lockFileName string, opts RunOptions, output string) (*WorkflowRunInfo, error, error) { var runInfo *WorkflowRunInfo var runErr error if parsedRunInfo := parseRunInfoFromOutput(output); parsedRunInfo != nil { @@ -426,75 +392,76 @@ func RunWorkflowOnGitHub(ctx context.Context, workflowIdOrName string, opts RunO } else { runInfo, runErr = getLatestWorkflowRunWithRetry(lockFileName, opts.RepoOverride, opts.Verbose) } + return runInfo, runErr, nil +} + +func runWorkflowOnGitHubAfterTrigger(ctx context.Context, workflowIdOrName string, runInfo *WorkflowRunInfo, runErr error, workflowStartTime time.Time, opts RunOptions) error { if runErr == nil && runInfo.URL != "" { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("🔗 View workflow run: "+runInfo.URL)) executionLog.Printf("Workflow run URL: %s (ID: %d)", runInfo.URL, runInfo.DatabaseID) - - // Suggest audit command for analysis fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("💡 To analyze this run, use: %s audit %d", string(constants.CLIExtensionPrefix), runInfo.DatabaseID))) } else if opts.Verbose && runErr != nil { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Note: Could not get workflow run URL: %v", runErr))) } - - // Wait for workflow completion if requested (for --repeat or --auto-merge-prs) if opts.WaitForCompletion || opts.AutoMergePRs { - if runErr != nil { - if opts.AutoMergePRs { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not get workflow run information for auto-merge: %v", runErr))) - } else if opts.WaitForCompletion { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not get workflow run information: %v", runErr))) - } - } else { - // Determine target repository: use repo override if provided, otherwise get current repo - targetRepo := opts.RepoOverride - if targetRepo == "" { - if currentRepo, err := GetCurrentRepoSlug(); err != nil { - if opts.AutoMergePRs { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not determine target repository for auto-merge: %v", err))) - } - targetRepo = "" - } else { - targetRepo = currentRepo - } - } + return runWorkflowOnGitHubWaitAfterTrigger(ctx, runInfo, runErr, workflowStartTime, opts) + } + return nil +} - if targetRepo != "" { - // Wait for workflow completion - if opts.AutoMergePRs { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Auto-merge PRs enabled - waiting for workflow completion...")) - } else { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Waiting for workflow completion...")) - } - - runIDStr := strconv.FormatInt(runInfo.DatabaseID, 10) - if err := WaitForWorkflowCompletion(ctx, targetRepo, runIDStr, workflowCompletionWaitTimeoutMinutes, opts.Verbose); err != nil { - // Propagate interrupts/cancellation so the caller (repeat loop) can stop - if ctx.Err() != nil || errors.Is(err, ErrInterrupted) { - return err - } - if opts.AutoMergePRs { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Workflow did not complete successfully, skipping auto-merge: %v", err))) - } else { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Workflow did not complete successfully: %v", err))) - } - } else { - // Auto-merge PRs if requested and workflow completed successfully - if opts.AutoMergePRs { - if err := AutoMergePullRequestsCreatedAfter(targetRepo, workflowStartTime, opts.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to auto-merge pull requests: %v", err))) - } - } - } - } +func runWorkflowOnGitHubWaitAfterTrigger(ctx context.Context, runInfo *WorkflowRunInfo, runErr error, workflowStartTime time.Time, opts RunOptions) error { + if runErr != nil { + if opts.AutoMergePRs { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not get workflow run information for auto-merge: %v", runErr))) + } else if opts.WaitForCompletion { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not get workflow run information: %v", runErr))) + } + return nil + } + targetRepo := runWorkflowOnGitHubTargetRepo(opts) + if targetRepo == "" { + return nil + } + if opts.AutoMergePRs { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Auto-merge PRs enabled - waiting for workflow completion...")) + } else { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Waiting for workflow completion...")) + } + runIDStr := strconv.FormatInt(runInfo.DatabaseID, 10) + if err := WaitForWorkflowCompletion(ctx, targetRepo, runIDStr, workflowCompletionWaitTimeoutMinutes, opts.Verbose); err != nil { + if ctx.Err() != nil || errors.Is(err, ErrInterrupted) { + return err + } + runWorkflowOnGitHubPrintCompletionWarning(err, opts) + } else if opts.AutoMergePRs { + if err := AutoMergePullRequestsCreatedAfter(targetRepo, workflowStartTime, opts.Verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to auto-merge pull requests: %v", err))) } } + return nil +} - // Restore workflow state if it was disabled and we enabled it - if opts.Enable && wasDisabled && workflowID != 0 { - restoreWorkflowState(workflowIdOrName, workflowID, opts.RepoOverride, opts.Verbose) +func runWorkflowOnGitHubTargetRepo(opts RunOptions) string { + targetRepo := opts.RepoOverride + if targetRepo != "" { + return targetRepo } + currentRepo, err := GetCurrentRepoSlug() + if err != nil { + if opts.AutoMergePRs { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not determine target repository for auto-merge: %v", err))) + } + return "" + } + return currentRepo +} - return nil +func runWorkflowOnGitHubPrintCompletionWarning(err error, opts RunOptions) { + if opts.AutoMergePRs { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Workflow did not complete successfully, skipping auto-merge: %v", err))) + } else { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Workflow did not complete successfully: %v", err))) + } } // RunWorkflowsOnGitHub runs multiple agentic workflows on GitHub Actions, optionally repeating a specified number of times @@ -502,138 +469,139 @@ func RunWorkflowsOnGitHub(ctx context.Context, workflowNames []string, opts RunO if len(workflowNames) == 0 { return errors.New("at least one workflow name or ID is required") } - - // Check context cancellation at the start select { case <-ctx.Done(): fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) return ctx.Err() default: } + if err := runWorkflowsOnGitHubValidate(workflowNames, opts); err != nil { + return err + } + runAllWorkflows := func() error { + return runWorkflowsOnGitHubRunAll(ctx, workflowNames, opts) + } + if opts.JSON { + runAllWorkflows = runWorkflowsOnGitHubJSONWrapper(runAllWorkflows, workflowNames, opts) + } + return ExecuteWithRepeat(RepeatOptions{ + Ctx: ctx, + RepeatCount: opts.RepeatCount, + RepeatMessage: "Repeating workflow run", + ExecuteFunc: runAllWorkflows, + UseStderr: false, + }) +} - // Validate all workflows exist and are runnable before starting +func runWorkflowsOnGitHubValidate(workflowNames []string, opts RunOptions) error { for _, workflowName := range workflowNames { if workflowName == "" { return errors.New("workflow name cannot be empty") } - - // Validate workflow exists if opts.RepoOverride != "" { - // For remote repositories, use remote validation if err := validateRemoteWorkflow(workflowName, opts.RepoOverride, opts.Verbose); err != nil { return fmt.Errorf("failed to validate remote workflow '%s': %w", workflowName, err) } - } else { - // For local workflows, use existing local validation - workflowFile, err := resolveWorkflowFile(workflowName, opts.Verbose) - if err != nil { - // Return error directly without wrapping - it already contains formatted message with suggestions - return err - } - - runnable, err := IsRunnable(workflowFile) - if err != nil { - return fmt.Errorf("failed to check if workflow '%s' is runnable: %w", workflowName, err) - } - - if !runnable { - return fmt.Errorf("workflow '%s' cannot be run on GitHub Actions - it must have 'workflow_dispatch' trigger", workflowName) - } + continue + } + if err := runWorkflowsOnGitHubValidateLocal(workflowName, opts); err != nil { + return err } } + return nil +} - // Function to run all workflows once - runAllWorkflows := func() error { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Running %d workflow(s)...", len(workflowNames)))) - - for i, workflowName := range workflowNames { - // Check for cancellation before each workflow - select { - case <-ctx.Done(): - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) - return ctx.Err() - default: - } - - if len(workflowNames) > 1 { - fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Running workflow %d/%d: %s", i+1, len(workflowNames), workflowName))) - } - - // Create a copy of opts with WaitForCompletion set when using --repeat - workflowOpts := opts - if opts.RepeatCount > 0 { - workflowOpts.WaitForCompletion = true - } - - if err := RunWorkflowOnGitHub(ctx, workflowName, workflowOpts); err != nil { - return fmt.Errorf("failed to run workflow '%s': %w", workflowName, err) - } +func runWorkflowsOnGitHubValidateLocal(workflowName string, opts RunOptions) error { + workflowFile, err := resolveWorkflowFile(workflowName, opts.Verbose) + if err != nil { + return err + } + runnable, err := IsRunnable(workflowFile) + if err != nil { + return fmt.Errorf("failed to check if workflow '%s' is runnable: %w", workflowName, err) + } + if !runnable { + return fmt.Errorf("workflow '%s' cannot be run on GitHub Actions - it must have 'workflow_dispatch' trigger", workflowName) + } + return nil +} - // Add a small delay between workflows to avoid overwhelming GitHub API - if i < len(workflowNames)-1 { - timer := time.NewTimer(betweenWorkflowsDelay) - select { - case <-ctx.Done(): - timer.Stop() - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) - return ctx.Err() - case <-timer.C: - } - } +func runWorkflowsOnGitHubRunAll(ctx context.Context, workflowNames []string, opts RunOptions) error { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Running %d workflow(s)...", len(workflowNames)))) + for i, workflowName := range workflowNames { + if err := runWorkflowsOnGitHubRunOne(ctx, workflowNames, workflowName, i, opts); err != nil { + return err } - - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Successfully triggered %d workflow(s)", len(workflowNames)))) - return nil } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Successfully triggered %d workflow(s)", len(workflowNames)))) + return nil +} - // When JSON output is requested, wrap runAllWorkflows to emit a JSON summary - if opts.JSON { - runAllWorkflowsInner := runAllWorkflows - runAllWorkflows = func() error { - // Build per-workflow results - var results []WorkflowRunResult - for _, workflowName := range workflowNames { - normalizedID := normalizeWorkflowID(workflowName) - lockFileName := normalizedID + ".lock.yml" - status := "triggered" - if opts.DryRun { - status = "dry_run" - } - results = append(results, WorkflowRunResult{ - Workflow: normalizedID, - LockFile: lockFileName, - Status: status, - }) - } +func runWorkflowsOnGitHubRunOne(ctx context.Context, workflowNames []string, workflowName string, i int, opts RunOptions) error { + select { + case <-ctx.Done(): + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) + return ctx.Err() + default: + } + if len(workflowNames) > 1 { + fmt.Fprintln(os.Stderr, console.FormatProgressMessage(fmt.Sprintf("Running workflow %d/%d: %s", i+1, len(workflowNames), workflowName))) + } + workflowOpts := opts + if opts.RepeatCount > 0 { + workflowOpts.WaitForCompletion = true + } + if err := RunWorkflowOnGitHub(ctx, workflowName, workflowOpts); err != nil { + return fmt.Errorf("failed to run workflow '%s': %w", workflowName, err) + } + return runWorkflowsOnGitHubDelay(ctx, workflowNames, i) +} - // Execute the actual runs (text output still goes to stderr) - runErr := runAllWorkflowsInner() - if runErr != nil { - // Mark all as error when we can't distinguish which failed - for i := range results { - results[i].Status = "error" - results[i].Error = runErr.Error() - } - } +func runWorkflowsOnGitHubDelay(ctx context.Context, workflowNames []string, i int) error { + if i >= len(workflowNames)-1 { + return nil + } + timer := time.NewTimer(betweenWorkflowsDelay) + select { + case <-ctx.Done(): + timer.Stop() + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Operation cancelled")) + return ctx.Err() + case <-timer.C: + return nil + } +} - // Output JSON to stdout - jsonBytes, err := json.MarshalIndent(results, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal JSON: %w", err) +func runWorkflowsOnGitHubJSONWrapper(runAllWorkflows func() error, workflowNames []string, opts RunOptions) func() error { + return func() error { + results := runWorkflowsOnGitHubJSONResults(workflowNames, opts) + runErr := runAllWorkflows() + if runErr != nil { + for i := range results { + results[i].Status = "error" + results[i].Error = runErr.Error() } - fmt.Fprintln(os.Stdout, string(jsonBytes)) - return runErr } + jsonBytes, err := json.MarshalIndent(results, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal JSON: %w", err) + } + fmt.Fprintln(os.Stdout, string(jsonBytes)) + return runErr } +} - // Execute workflows with optional repeat functionality - return ExecuteWithRepeat(RepeatOptions{ - Ctx: ctx, - RepeatCount: opts.RepeatCount, - RepeatMessage: "Repeating workflow run", - ExecuteFunc: runAllWorkflows, - UseStderr: false, // Use stdout for run command - }) +func runWorkflowsOnGitHubJSONResults(workflowNames []string, opts RunOptions) []WorkflowRunResult { + var results []WorkflowRunResult + for _, workflowName := range workflowNames { + normalizedID := normalizeWorkflowID(workflowName) + status := "triggered" + if opts.DryRun { + status = "dry_run" + } + results = append(results, WorkflowRunResult{Workflow: normalizedID, LockFile: normalizedID + ".lock.yml", Status: status}) + } + return results } // runInfoURLRegexp matches GitHub Actions run URLs of the form: diff --git a/pkg/cli/run_workflow_tracking.go b/pkg/cli/run_workflow_tracking.go index c8be4bf21a6..872ee698e5d 100644 --- a/pkg/cli/run_workflow_tracking.go +++ b/pkg/cli/run_workflow_tracking.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "os" - "os/exec" "time" "github.com/github/gh-aw/pkg/console" @@ -32,157 +31,157 @@ func getLatestWorkflowRunWithRetry(lockFileName string, repo string, verbose boo const initialDelay = 2 * time.Second const maxDelay = 10 * time.Second - if repo != "" { - console.LogVerbose(verbose, fmt.Sprintf("Getting latest run for workflow: %s in repo: %s (with retry logic)", lockFileName, repo)) - } else { - console.LogVerbose(verbose, fmt.Sprintf("Getting latest run for workflow: %s (with retry logic)", lockFileName)) - } - - // Capture the current time before we start polling - // This helps us identify runs that were created after the workflow was triggered + getLatestWorkflowRunWithRetryLogStart(lockFileName, repo, verbose) startTime := time.Now().UTC() runWorkflowTrackingLog.Printf("Start time for polling: %s", startTime.Format(time.RFC3339)) - - // Create spinner outside the loop so we can update it - var spinner *console.SpinnerWrapper - if !verbose { - spinner = console.NewSpinner("Waiting for workflow run to appear...") - } + spinner := getLatestWorkflowRunWithRetrySpinner(verbose) var lastErr error for attempt := range maxRetries { - if attempt > 0 { - // Calculate delay with exponential backoff, capped at maxDelay - delay := min(time.Duration(attempt)*initialDelay, maxDelay) - - // Calculate elapsed time since start - elapsed := time.Since(startTime).Round(time.Second) - - console.LogVerbose(verbose, fmt.Sprintf("Waiting %v before retry attempt %d/%d...", delay, attempt+1, maxRetries)) - - if !verbose { - // Show spinner starting from second attempt to avoid flickering - if attempt == 1 && spinner != nil { - spinner.Start() - } - // Update spinner with progress information - if spinner != nil { - spinner.UpdateMessage(fmt.Sprintf("Waiting for workflow run... (attempt %d/%d, %v elapsed)", attempt+1, maxRetries, elapsed)) - } - } - time.Sleep(delay) - } - - // Build command with optional repo parameter - var cmd *exec.Cmd - if repo != "" { - cmd = workflow.ExecGH("run", "list", "--repo", repo, "--workflow", lockFileName, "--limit", "1", "--json", "url,databaseId,status,conclusion,createdAt") - } else { - cmd = workflow.ExecGH("run", "list", "--workflow", lockFileName, "--limit", "1", "--json", "url,databaseId,status,conclusion,createdAt") - } - - output, err := cmd.Output() + getLatestWorkflowRunWithRetryDelay(attempt, maxRetries, initialDelay, maxDelay, startTime, spinner, verbose) + runInfo, retry, err := getLatestWorkflowRunWithRetryAttempt(lockFileName, repo, verbose, startTime, attempt, maxRetries, spinner) if err != nil { - lastErr = fmt.Errorf("failed to get workflow runs: %w", err) - runWorkflowTrackingLog.Printf("Attempt %d/%d failed to get runs: %v", attempt+1, maxRetries, err) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Attempt %d/%d failed: %v", attempt+1, maxRetries, err))) + lastErr = err + if retry { + continue } - continue - } - - if len(output) == 0 || string(output) == "[]" { - lastErr = errors.New("no runs found for workflow") - runWorkflowTrackingLog.Printf("Attempt %d/%d: no runs found, output empty or []", attempt+1, maxRetries) - console.LogVerbose(verbose, fmt.Sprintf("Attempt %d/%d: no runs found yet", attempt+1, maxRetries)) - continue - } - - // Parse the JSON output - var runs []struct { - URL string `json:"url"` - DatabaseID int64 `json:"databaseId"` - Status string `json:"status"` - Conclusion string `json:"conclusion"` - CreatedAt string `json:"createdAt"` + return nil, err } - - if err := json.Unmarshal(output, &runs); err != nil { - lastErr = fmt.Errorf("failed to parse workflow run data: %w", err) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Attempt %d/%d failed to parse JSON: %v", attempt+1, maxRetries, err))) - } - continue + if runInfo != nil { + return runInfo, nil } + } + if spinner != nil { + spinner.Stop() + } + if lastErr != nil { + return nil, fmt.Errorf("failed to get workflow run after %d attempts: %w", maxRetries, lastErr) + } + return nil, fmt.Errorf("no workflow run found after %d attempts", maxRetries) +} - if len(runs) == 0 { - lastErr = errors.New("no runs found") - console.LogVerbose(verbose, fmt.Sprintf("Attempt %d/%d: no runs in parsed JSON", attempt+1, maxRetries)) - continue - } +func getLatestWorkflowRunWithRetryLogStart(lockFileName, repo string, verbose bool) { + if repo != "" { + console.LogVerbose(verbose, fmt.Sprintf("Getting latest run for workflow: %s in repo: %s (with retry logic)", lockFileName, repo)) + } else { + console.LogVerbose(verbose, fmt.Sprintf("Getting latest run for workflow: %s (with retry logic)", lockFileName)) + } +} - run := runs[0] +func getLatestWorkflowRunWithRetrySpinner(verbose bool) *console.SpinnerWrapper { + if verbose { + return nil + } + return console.NewSpinner("Waiting for workflow run to appear...") +} - // Parse the creation timestamp - var createdAt time.Time - if run.CreatedAt != "" { - if parsedTime, err := time.Parse(time.RFC3339, run.CreatedAt); err == nil { - createdAt = parsedTime - } else if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not parse creation time '%s': %v", run.CreatedAt, err))) - } +func getLatestWorkflowRunWithRetryDelay(attempt, maxRetries int, initialDelay, maxDelay time.Duration, startTime time.Time, spinner *console.SpinnerWrapper, verbose bool) { + if attempt <= 0 { + return + } + delay := min(time.Duration(attempt)*initialDelay, maxDelay) + elapsed := time.Since(startTime).Round(time.Second) + console.LogVerbose(verbose, fmt.Sprintf("Waiting %v before retry attempt %d/%d...", delay, attempt+1, maxRetries)) + if !verbose && spinner != nil { + if attempt == 1 { + spinner.Start() } + spinner.UpdateMessage(fmt.Sprintf("Waiting for workflow run... (attempt %d/%d, %v elapsed)", attempt+1, maxRetries, elapsed)) + } + time.Sleep(delay) +} - runInfo := &WorkflowRunInfo{ - URL: run.URL, - DatabaseID: run.DatabaseID, - Status: run.Status, - Conclusion: run.Conclusion, - CreatedAt: createdAt, +func getLatestWorkflowRunWithRetryAttempt(lockFileName, repo string, verbose bool, startTime time.Time, attempt, maxRetries int, spinner *console.SpinnerWrapper) (*WorkflowRunInfo, bool, error) { + output, err := getLatestWorkflowRunWithRetryOutput(lockFileName, repo) + if err != nil { + runWorkflowTrackingLog.Printf("Attempt %d/%d failed to get runs: %v", attempt+1, maxRetries, err) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Attempt %d/%d failed: %v", attempt+1, maxRetries, err))) } + return nil, true, fmt.Errorf("failed to get workflow runs: %w", err) + } + if len(output) == 0 || string(output) == "[]" { + runWorkflowTrackingLog.Printf("Attempt %d/%d: no runs found, output empty or []", attempt+1, maxRetries) + console.LogVerbose(verbose, fmt.Sprintf("Attempt %d/%d: no runs found yet", attempt+1, maxRetries)) + return nil, true, errors.New("no runs found for workflow") + } + run, err := getLatestWorkflowRunWithRetryParse(output, verbose, attempt, maxRetries) + if err != nil { + return nil, true, err + } + return getLatestWorkflowRunWithRetrySelect(run, startTime, attempt, maxRetries, spinner, verbose) +} - // If we found a run and it was created after we started (within 30 seconds tolerance), - // it's likely the run we just triggered - if !createdAt.IsZero() && createdAt.After(startTime.Add(-30*time.Second)) { - runWorkflowTrackingLog.Printf("Found matching run: id=%d, created_at=%s, within_tolerance=true", run.DatabaseID, createdAt.Format(time.RFC3339)) - console.LogVerbose(verbose, fmt.Sprintf("Found recent run (ID: %d) created at %v (started polling at %v)", - run.DatabaseID, createdAt.Format(time.RFC3339), startTime.Format(time.RFC3339))) - if spinner != nil { - spinner.StopWithMessage("✓ Found workflow run") - } - return runInfo, nil - } +func getLatestWorkflowRunWithRetryOutput(lockFileName, repo string) ([]byte, error) { + if repo != "" { + return workflow.ExecGH("run", "list", "--repo", repo, "--workflow", lockFileName, "--limit", "1", "--json", "url,databaseId,status,conclusion,createdAt").Output() + } + return workflow.ExecGH("run", "list", "--workflow", lockFileName, "--limit", "1", "--json", "url,databaseId,status,conclusion,createdAt").Output() +} - if createdAt.IsZero() { - console.LogVerbose(verbose, fmt.Sprintf("Attempt %d/%d: Found run (ID: %d) but no creation timestamp available", attempt+1, maxRetries, run.DatabaseID)) - } else { - console.LogVerbose(verbose, fmt.Sprintf("Attempt %d/%d: Found run (ID: %d) but it was created at %v (too old)", - attempt+1, maxRetries, run.DatabaseID, createdAt.Format(time.RFC3339))) - } +type getLatestWorkflowRunWithRetryRun struct { + URL string `json:"url"` + DatabaseID int64 `json:"databaseId"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + CreatedAt string `json:"createdAt"` +} - // For the first few attempts, if we have a run but it's too old, keep trying - if attempt < 3 { - lastErr = errors.New("workflow run appears to be from a previous execution") - continue +func getLatestWorkflowRunWithRetryParse(output []byte, verbose bool, attempt, maxRetries int) (getLatestWorkflowRunWithRetryRun, error) { + var runs []getLatestWorkflowRunWithRetryRun + if err := json.Unmarshal(output, &runs); err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage(fmt.Sprintf("Attempt %d/%d failed to parse JSON: %v", attempt+1, maxRetries, err))) } + return getLatestWorkflowRunWithRetryRun{}, fmt.Errorf("failed to parse workflow run data: %w", err) + } + if len(runs) == 0 { + console.LogVerbose(verbose, fmt.Sprintf("Attempt %d/%d: no runs in parsed JSON", attempt+1, maxRetries)) + return getLatestWorkflowRunWithRetryRun{}, errors.New("no runs found") + } + return runs[0], nil +} - // For later attempts, return what we found even if timing is uncertain - console.LogVerbose(verbose, fmt.Sprintf("Returning workflow run (ID: %d) after %d attempts (timing uncertain)", run.DatabaseID, attempt+1)) +func getLatestWorkflowRunWithRetrySelect(run getLatestWorkflowRunWithRetryRun, startTime time.Time, attempt, maxRetries int, spinner *console.SpinnerWrapper, verbose bool) (*WorkflowRunInfo, bool, error) { + createdAt := getLatestWorkflowRunWithRetryCreatedAt(run.CreatedAt, verbose) + runInfo := &WorkflowRunInfo{URL: run.URL, DatabaseID: run.DatabaseID, Status: run.Status, Conclusion: run.Conclusion, CreatedAt: createdAt} + if !createdAt.IsZero() && createdAt.After(startTime.Add(-30*time.Second)) { + runWorkflowTrackingLog.Printf("Found matching run: id=%d, created_at=%s, within_tolerance=true", run.DatabaseID, createdAt.Format(time.RFC3339)) + console.LogVerbose(verbose, fmt.Sprintf("Found recent run (ID: %d) created at %v (started polling at %v)", run.DatabaseID, createdAt.Format(time.RFC3339), startTime.Format(time.RFC3339))) if spinner != nil { spinner.StopWithMessage("✓ Found workflow run") } - return runInfo, nil + return runInfo, false, nil } - - // Stop spinner on failure + getLatestWorkflowRunWithRetryLogOldRun(run, createdAt, attempt, maxRetries, verbose) + if attempt < 3 { + return nil, true, errors.New("workflow run appears to be from a previous execution") + } + console.LogVerbose(verbose, fmt.Sprintf("Returning workflow run (ID: %d) after %d attempts (timing uncertain)", run.DatabaseID, attempt+1)) if spinner != nil { - spinner.Stop() + spinner.StopWithMessage("✓ Found workflow run") } + return runInfo, false, nil +} - // If we exhausted all retries, return the last error - if lastErr != nil { - return nil, fmt.Errorf("failed to get workflow run after %d attempts: %w", maxRetries, lastErr) +func getLatestWorkflowRunWithRetryCreatedAt(value string, verbose bool) time.Time { + if value == "" { + return time.Time{} + } + parsedTime, err := time.Parse(time.RFC3339, value) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not parse creation time '%s': %v", value, err))) + } + return time.Time{} } + return parsedTime +} - return nil, fmt.Errorf("no workflow run found after %d attempts", maxRetries) +func getLatestWorkflowRunWithRetryLogOldRun(run getLatestWorkflowRunWithRetryRun, createdAt time.Time, attempt, maxRetries int, verbose bool) { + if createdAt.IsZero() { + console.LogVerbose(verbose, fmt.Sprintf("Attempt %d/%d: Found run (ID: %d) but no creation timestamp available", attempt+1, maxRetries, run.DatabaseID)) + return + } + console.LogVerbose(verbose, fmt.Sprintf("Attempt %d/%d: Found run (ID: %d) but it was created at %v (too old)", attempt+1, maxRetries, run.DatabaseID, createdAt.Format(time.RFC3339))) } diff --git a/pkg/cli/run_workflow_validation.go b/pkg/cli/run_workflow_validation.go index b8f79d755ec..3595104273c 100644 --- a/pkg/cli/run_workflow_validation.go +++ b/pkg/cli/run_workflow_validation.go @@ -85,76 +85,62 @@ func IsRunnable(markdownPath string) (bool, error) { // getWorkflowInputs extracts workflow_dispatch inputs from the compiled lock file // This function checks the .lock.yml file because that's what GitHub Actions uses. func getWorkflowInputs(markdownPath string) (map[string]*workflow.InputDefinition, error) { - // Convert markdown path to lock file path + workflowYAML, err := getWorkflowInputsYAML(markdownPath) + if err != nil { + return nil, err + } + workflowDispatchMap := getWorkflowInputsDispatchMap(workflowYAML) + if workflowDispatchMap == nil { + return nil, nil + } + inputsMap, ok := workflowDispatchMap["inputs"].(map[string]any) + if !ok { + return nil, nil + } + + parsed := workflow.ParseInputDefinitions(inputsMap) + delete(parsed, workflow.AwContextInputName) + return parsed, nil +} + +func getWorkflowInputsYAML(markdownPath string) (map[string]any, error) { lockPath := getLockFilePath(markdownPath) cleanLockPath := filepath.Clean(lockPath) - validationLog.Printf("Extracting workflow inputs from lock file: %s", lockPath) - - // Check if the lock file exists if _, err := os.Stat(cleanLockPath); os.IsNotExist(err) { validationLog.Printf("Lock file does not exist: %s", cleanLockPath) return nil, errors.New("workflow has not been compiled yet - run 'gh aw compile' first") } - - // Read the lock file - path is sanitized using filepath.Clean() to prevent path traversal attacks. - // The lockPath is derived from markdownPath which comes from trusted sources (CLI arguments, validated workflow paths). - contentBytes, err := os.ReadFile(cleanLockPath) // #nosec G304 -- path is sanitized with filepath.Clean() and derived from trusted CLI argument + // #nosec G304 -- path is sanitized with filepath.Clean() and derived from trusted CLI argument + contentBytes, err := os.ReadFile(cleanLockPath) if err != nil { return nil, fmt.Errorf("failed to read lock file: %w", err) } - - // Parse the YAML content var workflowYAML map[string]any if err := yaml.Unmarshal(contentBytes, &workflowYAML); err != nil { return nil, fmt.Errorf("failed to parse lock file YAML: %w", err) } + return workflowYAML, nil +} - // Check if 'on' section is present +func getWorkflowInputsDispatchMap(workflowYAML map[string]any) map[string]any { onSection, exists := workflowYAML["on"] if !exists { - return nil, nil + return nil } - - // Convert to map if possible onMap, ok := onSection.(map[string]any) if !ok { - return nil, nil + return nil } - - // Get workflow_dispatch section workflowDispatch, exists := onMap["workflow_dispatch"] if !exists { - return nil, nil + return nil } - - // Convert to map workflowDispatchMap, ok := workflowDispatch.(map[string]any) if !ok { - // workflow_dispatch might be null/empty - return nil, nil - } - - // Get inputs section - inputsSection, exists := workflowDispatchMap["inputs"] - if !exists { - return nil, nil - } - - // Convert to map - inputsMap, ok := inputsSection.(map[string]any) - if !ok { - return nil, nil + return nil } - - // Parse input definitions - parsed := workflow.ParseInputDefinitions(inputsMap) - - // Remove aw_context from the returned inputs - it is an internal input managed by the - // agentic workflow system and should never be surfaced to users for prompting or display. - delete(parsed, workflow.AwContextInputName) - - return parsed, nil + return workflowDispatchMap } // validateWorkflowInputs validates that required inputs are provided and checks for typos. @@ -171,20 +157,32 @@ func getWorkflowInputs(markdownPath string) (map[string]*workflow.InputDefinitio // // This follows the principle that domain-specific validation belongs in domain files. func validateWorkflowInputs(markdownPath string, providedInputs []string) error { - // Extract workflow inputs workflowInputs, err := getWorkflowInputs(markdownPath) if err != nil { - // Don't fail validation if we can't extract inputs validationLog.Printf("Failed to extract workflow inputs: %v", err) return nil } - - // If no inputs are defined, no validation needed if len(workflowInputs) == 0 { return nil } - // Parse provided inputs into a map + providedInputsMap := validateWorkflowInputsProvidedMap(providedInputs) + missingInputs := validateWorkflowInputsMissing(workflowInputs, providedInputsMap) + typos, suggestions := validateWorkflowInputsTypos(workflowInputs, providedInputsMap) + if len(missingInputs) == 0 && len(typos) == 0 { + return nil + } + + errorParts := validateWorkflowInputsErrorParts(markdownPath, workflowInputs, missingInputs, typos, suggestions) + return workflow.NewValidationError( + "on.workflow_dispatch.inputs", + strings.Join(providedInputs, ", "), + strings.Join(errorParts, "\n\n"), + fmt.Sprintf("Define and provide valid workflow_dispatch inputs.\n\nExample workflow frontmatter:\n\non:\n workflow_dispatch:\n inputs:\n issue_url:\n description: \"Issue URL\"\n required: true\n type: string\n\nExample command:\n gh aw run %s -F issue_url=https://github.com/org/repo/issues/123", strings.TrimSuffix(filepath.Base(markdownPath), ".md")), + ) +} + +func validateWorkflowInputsProvidedMap(providedInputs []string) map[string]string { providedInputsMap := make(map[string]string) for _, input := range providedInputs { parts := strings.SplitN(input, "=", 2) @@ -192,8 +190,10 @@ func validateWorkflowInputs(markdownPath string, providedInputs []string) error providedInputsMap[parts[0]] = parts[1] } } + return providedInputsMap +} - // Check for required inputs that are missing (ignore inputs with a default value) +func validateWorkflowInputsMissing(workflowInputs map[string]*workflow.InputDefinition, providedInputsMap map[string]string) []string { var missingInputs []string for inputName, inputDef := range workflowInputs { if inputDef.Required && inputDef.Default == nil { @@ -202,86 +202,69 @@ func validateWorkflowInputs(markdownPath string, providedInputs []string) error } } } + return missingInputs +} - // Check for typos in provided input names +func validateWorkflowInputsTypos(workflowInputs map[string]*workflow.InputDefinition, providedInputsMap map[string]string) ([]string, []string) { var typos []string var suggestions []string validInputNames := slices.Collect(maps.Keys(workflowInputs)) - for providedName := range providedInputsMap { - // Check if this is a valid input name - if _, exists := workflowInputs[providedName]; !exists { - // Find closest matches - matches := parser.FindClosestMatches(providedName, validInputNames, 3) - if len(matches) > 0 { - typos = append(typos, providedName) - suggestions = append(suggestions, fmt.Sprintf("'%s' -> did you mean '%s'?", providedName, strings.Join(matches, "', '"))) - } else { - typos = append(typos, providedName) - suggestions = append(suggestions, fmt.Sprintf("'%s' is not a valid input name", providedName)) - } + if _, exists := workflowInputs[providedName]; exists { + continue + } + matches := parser.FindClosestMatches(providedName, validInputNames, 3) + typos = append(typos, providedName) + if len(matches) > 0 { + suggestions = append(suggestions, fmt.Sprintf("'%s' -> did you mean '%s'?", providedName, strings.Join(matches, "', '"))) + } else { + suggestions = append(suggestions, fmt.Sprintf("'%s' is not a valid input name", providedName)) } } + return typos, suggestions +} - // Build error message if there are validation errors - if len(missingInputs) > 0 || len(typos) > 0 { - var errorParts []string +func validateWorkflowInputsErrorParts(markdownPath string, workflowInputs map[string]*workflow.InputDefinition, missingInputs, typos, suggestions []string) []string { + var errorParts []string + if len(missingInputs) > 0 { + errorParts = append(errorParts, "Missing required input(s): "+strings.Join(missingInputs, ", ")) + } + if len(typos) > 0 { + errorParts = append(errorParts, "Invalid input name(s):\n "+strings.Join(suggestions, "\n ")) + } + if len(workflowInputs) > 0 { + errorParts = append(errorParts, validateWorkflowInputsValidInputsMessage(markdownPath, workflowInputs)) + } + return errorParts +} - if len(missingInputs) > 0 { - errorParts = append(errorParts, "Missing required input(s): "+strings.Join(missingInputs, ", ")) +func validateWorkflowInputsValidInputsMessage(markdownPath string, workflowInputs map[string]*workflow.InputDefinition) string { + var inputDescriptions []string + var syntaxExamples []string + sortedNames := sliceutil.SortedKeys(workflowInputs) + workflowName := strings.TrimSuffix(filepath.Base(markdownPath), ".md") + for _, name := range sortedNames { + def := workflowInputs[name] + required := "" + if def.Required && def.Default == nil { + required = " (required)" + syntaxExamples = append(syntaxExamples, fmt.Sprintf(" gh aw run %s -F %s=", workflowName, name)) } - - if len(typos) > 0 { - errorParts = append(errorParts, "Invalid input name(s):\n "+strings.Join(suggestions, "\n ")) + desc := "" + if def.Description != "" { + desc = ": " + def.Description } - - // Add helpful information about valid inputs - if len(workflowInputs) > 0 { - var inputDescriptions []string - sortedNames := sliceutil.SortedKeys(workflowInputs) - for _, name := range sortedNames { - def := workflowInputs[name] - required := "" - if def.Required && def.Default == nil { - required = " (required)" - } - desc := "" - if def.Description != "" { - desc = ": " + def.Description - } - defaultStr := "" - if def.Default != nil { - defaultStr = fmt.Sprintf(" [default: %s]", def.GetDefaultAsString()) - } - inputDescriptions = append(inputDescriptions, fmt.Sprintf(" %s%s%s%s", name, required, desc, defaultStr)) - } - - // Derive the workflow name for the syntax hint - workflowName := strings.TrimSuffix(filepath.Base(markdownPath), ".md") - var syntaxExamples []string - for _, name := range sortedNames { - def := workflowInputs[name] - if def.Required && def.Default == nil { - syntaxExamples = append(syntaxExamples, fmt.Sprintf(" gh aw run %s -F %s=", workflowName, name)) - } - } - - validInputsMsg := "\nValid inputs:\n" + strings.Join(inputDescriptions, "\n") - if len(syntaxExamples) > 0 { - validInputsMsg += "\n\nTo set required inputs, use:\n" + strings.Join(syntaxExamples, "\n") - } - errorParts = append(errorParts, validInputsMsg) + defaultStr := "" + if def.Default != nil { + defaultStr = fmt.Sprintf(" [default: %s]", def.GetDefaultAsString()) } - - return workflow.NewValidationError( - "on.workflow_dispatch.inputs", - strings.Join(providedInputs, ", "), - strings.Join(errorParts, "\n\n"), - fmt.Sprintf("Define and provide valid workflow_dispatch inputs.\n\nExample workflow frontmatter:\n\non:\n workflow_dispatch:\n inputs:\n issue_url:\n description: \"Issue URL\"\n required: true\n type: string\n\nExample command:\n gh aw run %s -F issue_url=https://github.com/org/repo/issues/123", strings.TrimSuffix(filepath.Base(markdownPath), ".md")), - ) + inputDescriptions = append(inputDescriptions, fmt.Sprintf(" %s%s%s%s", name, required, desc, defaultStr)) } - - return nil + validInputsMsg := "\nValid inputs:\n" + strings.Join(inputDescriptions, "\n") + if len(syntaxExamples) > 0 { + validInputsMsg += "\n\nTo set required inputs, use:\n" + strings.Join(syntaxExamples, "\n") + } + return validInputsMsg } // validateRemoteWorkflow checks if a workflow exists in a remote repository and can be triggered. diff --git a/pkg/cli/runner_guard.go b/pkg/cli/runner_guard.go index 879e16a40c3..6e08ffa565e 100644 --- a/pkg/cli/runner_guard.go +++ b/pkg/cli/runner_guard.go @@ -39,20 +39,32 @@ type runnerGuardOutput struct { // containing workflows using the Docker image. func runRunnerGuardOnDirectory(workflowDir string, verbose bool, strict bool) error { runnerGuardLog.Printf("Running runner-guard taint analysis on directory: %s", workflowDir) + gitRoot, scanPath, err := runRunnerGuardOnDirectoryPaths(workflowDir) + if err != nil { + return err + } + cmd := runRunnerGuardOnDirectoryCommand(gitRoot, scanPath) + runRunnerGuardOnDirectoryPrint(gitRoot, scanPath, verbose) + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err = cmd.Run() + totalFindings, parseErr := parseAndDisplayRunnerGuardOutput(stdout.String(), verbose, gitRoot) + if parseErr != nil { + runRunnerGuardOnDirectoryRawOutput(stdout, stderr, parseErr) + } + return runRunnerGuardOnDirectoryHandleError(err, parseErr, totalFindings, strict) +} - // Find git root to get the absolute path for Docker volume mount +func runRunnerGuardOnDirectoryPaths(workflowDir string) (string, string, error) { gitRoot, err := gitutil.FindGitRoot() if err != nil { - return fmt.Errorf("failed to find git root: %w", err) + return "", "", fmt.Errorf("failed to find git root: %w", err) } - - // Validate gitRoot is an absolute path (security: ensure trusted path from git) if !filepath.IsAbs(gitRoot) { - return fmt.Errorf("git root is not an absolute path: %s", gitRoot) + return "", "", fmt.Errorf("git root is not an absolute path: %s", gitRoot) } - - // Determine the scan path: use workflowDir relative to gitRoot when possible, - // so the scan is scoped to the compiled workflows directory. scanPath := "." if workflowDir != "" { relDir, relErr := filepath.Rel(gitRoot, workflowDir) @@ -60,93 +72,62 @@ func runRunnerGuardOnDirectory(workflowDir string, verbose bool, strict bool) er scanPath = relDir } } + return gitRoot, scanPath, nil +} - // Build the Docker command - // docker run --rm -v "$gitRoot:/workdir" -w /workdir ghcr.io/vigilant-llc/runner-guard:latest scan --format json - // #nosec G204 -- gitRoot comes from git rev-parse (trusted source) and is validated as absolute path. - // exec.Command with separate args (not shell execution) prevents command injection. - cmd := exec.Command( - "docker", - "run", - "--rm", - "-v", gitRoot+":/workdir", - "-w", "/workdir", - RunnerGuardImage, - "scan", - scanPath, - "--format", "json", - ) +func runRunnerGuardOnDirectoryCommand(gitRoot, scanPath string) *exec.Cmd { + return exec.Command("docker", "run", "--rm", "-v", gitRoot+":/workdir", "-w", "/workdir", RunnerGuardImage, "scan", scanPath, "--format", "json") +} - // Always show that runner-guard is running (regular verbosity) +func runRunnerGuardOnDirectoryPrint(gitRoot, scanPath string, verbose bool) { fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Running runner-guard taint analysis scanner")) - - // In verbose mode, also show the command that users can run directly if verbose { - dockerCmd := fmt.Sprintf("docker run --rm -v \"%s:/workdir\" -w /workdir %s scan %s --format json", - gitRoot, RunnerGuardImage, scanPath) + dockerCmd := fmt.Sprintf("docker run --rm -v \"%s:/workdir\" -w /workdir %s scan %s --format json", gitRoot, RunnerGuardImage, scanPath) fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Run runner-guard directly: "+dockerCmd)) } +} - // Capture output - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - // Run the command - err = cmd.Run() - - // Parse and display output - totalFindings, parseErr := parseAndDisplayRunnerGuardOutput(stdout.String(), verbose, gitRoot) - if parseErr != nil { - runnerGuardLog.Printf("Failed to parse runner-guard output: %v", parseErr) - // Fall back to showing raw output - if stdout.Len() > 0 { - fmt.Fprint(os.Stderr, stdout.String()) - } - if stderr.Len() > 0 { - fmt.Fprint(os.Stderr, stderr.String()) - } +func runRunnerGuardOnDirectoryRawOutput(stdout, stderr bytes.Buffer, parseErr error) { + runnerGuardLog.Printf("Failed to parse runner-guard output: %v", parseErr) + if stdout.Len() > 0 { + fmt.Fprint(os.Stderr, stdout.String()) + } + if stderr.Len() > 0 { + fmt.Fprint(os.Stderr, stderr.String()) } +} - // Check if the error is due to findings or actual failure - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - exitCode := exitErr.ExitCode() - runnerGuardLog.Printf("runner-guard exited with code %d (findings=%d)", exitCode, totalFindings) - // Exit code 1 typically indicates findings in the repository - if exitCode == 1 { - if strict { - if parseErr != nil { - // JSON parsing failed but exit code confirms findings exist - return fmt.Errorf("strict mode: runner-guard exited with code 1 (findings present) and output could not be parsed: %w", parseErr) - } - if totalFindings > 0 { - return fmt.Errorf("strict mode: runner-guard found %d security findings - workflows must have no runner-guard findings in strict mode", totalFindings) - } - // Exit code 1 with no parseable findings is still a failure in strict mode - return errors.New("strict mode: runner-guard exited with code 1 indicating findings are present") - } - // In non-strict mode, findings are logged but not treated as errors - return nil - } - // Other exit codes are actual errors - return fmt.Errorf("runner-guard failed with exit code %d", exitCode) - } - // Non-ExitError errors (e.g., command not found) +func runRunnerGuardOnDirectoryHandleError(err error, parseErr error, totalFindings int, strict bool) error { + if err == nil { + return nil + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { return fmt.Errorf("runner-guard failed: %w", err) } - - return nil + exitCode := exitErr.ExitCode() + runnerGuardLog.Printf("runner-guard exited with code %d (findings=%d)", exitCode, totalFindings) + if exitCode != 1 { + return fmt.Errorf("runner-guard failed with exit code %d", exitCode) + } + if !strict { + return nil + } + if parseErr != nil { + return fmt.Errorf("strict mode: runner-guard exited with code 1 (findings present) and output could not be parsed: %w", parseErr) + } + if totalFindings > 0 { + return fmt.Errorf("strict mode: runner-guard found %d security findings - workflows must have no runner-guard findings in strict mode", totalFindings) + } + return errors.New("strict mode: runner-guard exited with code 1 indicating findings are present") } // parseAndDisplayRunnerGuardOutput parses runner-guard JSON output and displays findings. // Returns the total number of findings found. func parseAndDisplayRunnerGuardOutput(stdout string, verbose bool, gitRoot string) (int, error) { if stdout == "" { - return 0, nil // No output means no findings + return 0, nil } - trimmed := strings.TrimSpace(stdout) if !strings.HasPrefix(trimmed, "{") && !strings.HasPrefix(trimmed, "[") { if trimmed != "" { @@ -159,111 +140,115 @@ func parseAndDisplayRunnerGuardOutput(stdout string, verbose bool, gitRoot strin if err := json.Unmarshal([]byte(stdout), &output); err != nil { return 0, fmt.Errorf("failed to parse runner-guard JSON output: %w", err) } - totalFindings := len(output.Findings) if totalFindings == 0 { return 0, nil } + parseAndDisplayRunnerGuardOutputScore(output) + for filePath, findings := range parseAndDisplayRunnerGuardOutputByFile(output.Findings) { + parseAndDisplayRunnerGuardOutputFile(filePath, findings, gitRoot) + } + return totalFindings, nil +} - // Display score/grade if present +func parseAndDisplayRunnerGuardOutputScore(output runnerGuardOutput) { if output.Score > 0 || output.Grade != "" { - fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage( - fmt.Sprintf("Runner-Guard Score: %d/100 (Grade: %s)", output.Score, output.Grade), - )) + fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Runner-Guard Score: %d/100 (Grade: %s)", output.Score, output.Grade))) } +} - // Group findings by file for better readability +func parseAndDisplayRunnerGuardOutputByFile(findings []runnerGuardFinding) map[string][]runnerGuardFinding { findingsByFile := make(map[string][]runnerGuardFinding) - for _, finding := range output.Findings { + for _, finding := range findings { findingsByFile[finding.File] = append(findingsByFile[finding.File], finding) } + return findingsByFile +} - // Display findings for each file - for filePath, findings := range findingsByFile { - // Validate and sanitize file path to prevent path traversal - cleanPath := filepath.Clean(filePath) - - absPath := cleanPath - if !filepath.IsAbs(cleanPath) { - absPath = filepath.Join(gitRoot, cleanPath) - } - - absGitRoot, err := filepath.Abs(gitRoot) - if err != nil { - runnerGuardLog.Printf("Failed to get absolute path for git root: %v", err) - continue - } +func parseAndDisplayRunnerGuardOutputFile(filePath string, findings []runnerGuardFinding, gitRoot string) { + absPath, ok := parseAndDisplayRunnerGuardOutputSafePath(filePath, gitRoot) + if !ok { + return + } + // #nosec G304 -- absPath is validated to be within gitRoot. + fileContent, err := os.ReadFile(absPath) + var fileLines []string + if err == nil { + fileLines = strings.Split(string(fileContent), "\n") + } + for _, finding := range findings { + parseAndDisplayRunnerGuardOutputFinding(finding, fileLines) + } +} - absPath, err = filepath.Abs(absPath) - if err != nil { - runnerGuardLog.Printf("Failed to get absolute path for %s: %v", filePath, err) - continue - } +func parseAndDisplayRunnerGuardOutputSafePath(filePath, gitRoot string) (string, bool) { + cleanPath := filepath.Clean(filePath) + absPath := cleanPath + if !filepath.IsAbs(cleanPath) { + absPath = filepath.Join(gitRoot, cleanPath) + } + absGitRoot, err := filepath.Abs(gitRoot) + if err != nil { + runnerGuardLog.Printf("Failed to get absolute path for git root: %v", err) + return "", false + } + absPath, err = filepath.Abs(absPath) + if err != nil { + runnerGuardLog.Printf("Failed to get absolute path for %s: %v", filePath, err) + return "", false + } + relPath, err := filepath.Rel(absGitRoot, absPath) + if err != nil || relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) { + runnerGuardLog.Printf("Skipping file outside git root: %s", filePath) + return "", false + } + return absPath, true +} - // Check if the resolved path is within gitRoot to prevent path traversal - relPath, err := filepath.Rel(absGitRoot, absPath) - if err != nil || relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) { - runnerGuardLog.Printf("Skipping file outside git root: %s", filePath) - continue - } +func parseAndDisplayRunnerGuardOutputFinding(finding runnerGuardFinding, fileLines []string) { + lineNum := finding.Line + if lineNum == 0 { + lineNum = 1 + } + compilerErr := console.CompilerError{ + Position: console.ErrorPosition{File: finding.File, Line: lineNum, Column: 1}, + Type: parseAndDisplayRunnerGuardOutputSeverity(finding.Severity), + Message: parseAndDisplayRunnerGuardOutputMessage(finding), + Context: parseAndDisplayRunnerGuardOutputContext(fileLines, lineNum), + } + fmt.Fprint(os.Stderr, console.FormatError(compilerErr)) +} - // Read file content for context display - // #nosec G304 -- absPath is validated through: 1) filepath.Clean() normalization, - // 2) absolute path resolution, and 3) filepath.Rel() check ensuring it's within gitRoot. - // Path traversal attacks are prevented by the boundary validation above. - fileContent, err := os.ReadFile(absPath) - var fileLines []string - if err == nil { - fileLines = strings.Split(string(fileContent), "\n") +func parseAndDisplayRunnerGuardOutputContext(fileLines []string, lineNum int) []string { + var context []string + if len(fileLines) == 0 || lineNum <= 0 || lineNum > len(fileLines) { + return context + } + startLine := max(1, lineNum-2) + endLine := min(len(fileLines), lineNum+2) + for i := startLine; i <= endLine; i++ { + if i-1 < len(fileLines) { + context = append(context, fileLines[i-1]) } + } + return context +} - for _, finding := range findings { - lineNum := finding.Line - if lineNum == 0 { - lineNum = 1 - } - - // Create context lines around the finding - var context []string - if len(fileLines) > 0 && lineNum > 0 && lineNum <= len(fileLines) { - startLine := max(1, lineNum-2) - endLine := min(len(fileLines), lineNum+2) - for i := startLine; i <= endLine; i++ { - if i-1 < len(fileLines) { - context = append(context, fileLines[i-1]) - } - } - } - - // Map severity to error type - errorType := "warning" - switch strings.ToLower(finding.Severity) { - case "critical", "high", "error": - errorType = "error" - case "note", "info": - errorType = "info" - } - - // Build message - message := fmt.Sprintf("[%s] %s: %s", finding.Severity, finding.RuleID, finding.Name) - if finding.Description != "" { - message = fmt.Sprintf("%s - %s", message, finding.Description) - } - - compilerErr := console.CompilerError{ - Position: console.ErrorPosition{ - File: finding.File, - Line: lineNum, - Column: 1, - }, - Type: errorType, - Message: message, - Context: context, - } - - fmt.Fprint(os.Stderr, console.FormatError(compilerErr)) - } +func parseAndDisplayRunnerGuardOutputSeverity(severity string) string { + switch strings.ToLower(severity) { + case "critical", "high", "error": + return "error" + case "note", "info": + return "info" + default: + return "warning" } +} - return totalFindings, nil +func parseAndDisplayRunnerGuardOutputMessage(finding runnerGuardFinding) string { + message := fmt.Sprintf("[%s] %s: %s", finding.Severity, finding.RuleID, finding.Name) + if finding.Description != "" { + message = fmt.Sprintf("%s - %s", message, finding.Description) + } + return message } diff --git a/pkg/cli/secret_set_command.go b/pkg/cli/secret_set_command.go index f1ea43c6f4d..25a8edb18d5 100644 --- a/pkg/cli/secret_set_command.go +++ b/pkg/cli/secret_set_command.go @@ -66,51 +66,7 @@ The secret value can be provided in three ways: gh aw secrets set MY_SECRET --value-from-env MY_TOKEN --repo myorg/myrepo`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - secretName := args[0] - secretSetLog.Printf("Setting repository secret: name=%s", secretName) - - // Determine target repository: explicit --repo or current repo by default - flagRepo, _ := cmd.Flags().GetString("repo") - var repoSlug string - if flagRepo != "" { - repoSlug = flagRepo - secretSetLog.Printf("Using explicit repository: %s", repoSlug) - } else { - var err error - repoSlug, err = GetCurrentRepoSlug() - if err != nil { - secretSetLog.Printf("Failed to detect current repository: %v", err) - return fmt.Errorf("failed to detect current repository: %w", err) - } - secretSetLog.Printf("Using current repository: %s", repoSlug) - } - - owner, repo, splitErr := repoutil.SplitRepoSlug(repoSlug) - if splitErr != nil { - return fmt.Errorf("invalid repository slug %q: %w", repoSlug, splitErr) - } - - // Create GitHub REST client using go-gh - client, err := api.NewRESTClient(secretSetClientOptions(flagAPIBase)) - if err != nil { - return fmt.Errorf("cannot create GitHub client: %w", err) - } - - secretValue, err := resolveSecretValueForSet(flagValueEnv, flagValue) - if err != nil { - secretSetLog.Printf("Failed to resolve secret value: %v", err) - return fmt.Errorf("cannot resolve secret value: %w", err) - } - - secretSetLog.Print("Encrypting and uploading secret to GitHub") - if err := setRepoSecret(client, owner, repo, secretName, secretValue); err != nil { - secretSetLog.Printf("Failed to set secret: %v", err) - return fmt.Errorf("failed to set secret: %w", err) - } - - secretSetLog.Printf("Successfully set secret %s for %s/%s", secretName, owner, repo) - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Secret %s updated for %s/%s", secretName, owner, repo))) - return nil + return newSecretsSetSubcommandRun(cmd, args[0], flagValueEnv, flagValue, flagAPIBase) }, } @@ -122,6 +78,50 @@ The secret value can be provided in three ways: return cmd } +func newSecretsSetSubcommandRun(cmd *cobra.Command, secretName, flagValueEnv, flagValue, flagAPIBase string) error { + secretSetLog.Printf("Setting repository secret: name=%s", secretName) + repoSlug, err := newSecretsSetSubcommandRepo(cmd) + if err != nil { + return err + } + owner, repo, splitErr := repoutil.SplitRepoSlug(repoSlug) + if splitErr != nil { + return fmt.Errorf("invalid repository slug %q: %w", repoSlug, splitErr) + } + client, err := api.NewRESTClient(secretSetClientOptions(flagAPIBase)) + if err != nil { + return fmt.Errorf("cannot create GitHub client: %w", err) + } + secretValue, err := resolveSecretValueForSet(flagValueEnv, flagValue) + if err != nil { + secretSetLog.Printf("Failed to resolve secret value: %v", err) + return fmt.Errorf("cannot resolve secret value: %w", err) + } + secretSetLog.Print("Encrypting and uploading secret to GitHub") + if err := setRepoSecret(client, owner, repo, secretName, secretValue); err != nil { + secretSetLog.Printf("Failed to set secret: %v", err) + return fmt.Errorf("failed to set secret: %w", err) + } + secretSetLog.Printf("Successfully set secret %s for %s/%s", secretName, owner, repo) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Secret %s updated for %s/%s", secretName, owner, repo))) + return nil +} + +func newSecretsSetSubcommandRepo(cmd *cobra.Command) (string, error) { + flagRepo, _ := cmd.Flags().GetString("repo") + if flagRepo != "" { + secretSetLog.Printf("Using explicit repository: %s", flagRepo) + return flagRepo, nil + } + repoSlug, err := GetCurrentRepoSlug() + if err != nil { + secretSetLog.Printf("Failed to detect current repository: %v", err) + return "", fmt.Errorf("failed to detect current repository: %w", err) + } + secretSetLog.Printf("Using current repository: %s", repoSlug) + return repoSlug, nil +} + func secretSetClientOptions(apiBase string) api.ClientOptions { opts := api.ClientOptions{ Timeout: constants.DefaultHTTPClientTimeout, diff --git a/pkg/cli/setup_repository.go b/pkg/cli/setup_repository.go index bdda33189c4..856bbab78ed 100644 --- a/pkg/cli/setup_repository.go +++ b/pkg/cli/setup_repository.go @@ -362,84 +362,91 @@ func runSetupRepositoryCheckWithRuntime(opts SetupRepositoryCheckOptions, runtim if err := validateSetupRepositoryCheckOptions(opts); err != nil { return err } - ctx := opts.Ctx if ctx == nil { ctx = context.Background() } - configureDefaultGHHostFromOriginRemoteIfUnset() - if err := runtime.checkAuth(ctx); err != nil { return fmt.Errorf("failed to verify GitHub CLI authentication: %w", err) } + ownerType, err := runSetupRepositoryCheckWithRuntimeOwner(ctx, opts, runtime) + if err != nil { + return err + } + inspection, dir, err := runSetupRepositoryCheckWithRuntimeInspect(ctx, opts, runtime) + if err != nil { + return err + } + result := runSetupRepositoryCheckWithRuntimeResult(opts, ownerType, dir, inspection) + if opts.JSON { + return renderSetupJSON(result) + } + runSetupRepositoryCheckWithRuntimePrint(result, inspection) + return nil +} +func runSetupRepositoryCheckWithRuntimeOwner(ctx context.Context, opts SetupRepositoryCheckOptions, runtime setupRepositoryRuntime) (string, error) { setupRepositoryLog.Printf("Running repository check: repo=%s, requireOwnerType=%s", opts.Repo, opts.RequireOwnerType) - owner := strings.Split(opts.Repo, "/")[0] ownerType, err := runtime.ownerType(ctx, owner) if err != nil { - return err + return "", err } ownerType = normalizeSetupOwnerType(ownerType) setupRepositoryLog.Printf("Resolved owner type for %s: %s", owner, ownerType) if opts.RequireOwnerType != "any" && ownerType != opts.RequireOwnerType { - return fmt.Errorf("owner %s is %s, but --require-owner-type=%s was requested", owner, ownerType, opts.RequireOwnerType) + return "", fmt.Errorf("owner %s is %s, but --require-owner-type=%s was requested", owner, ownerType, opts.RequireOwnerType) } + return ownerType, nil +} +func runSetupRepositoryCheckWithRuntimeInspect(ctx context.Context, opts SetupRepositoryCheckOptions, runtime setupRepositoryRuntime) (setupCheckoutInspection, string, error) { repoExists, err := runtime.repoExists(ctx, opts.Repo) if err != nil { - return err + return setupCheckoutInspection{}, "", err } if !repoExists { - return fmt.Errorf("repository %s does not exist", opts.Repo) + return setupCheckoutInspection{}, "", fmt.Errorf("repository %s does not exist", opts.Repo) } - dir := resolveSetupCheckoutDir(opts.Repo, opts.Dir) inspection, err := inspectSetupCheckout(dir, opts.Repo, runtime.dirOriginRepo) if err != nil { - return err + return setupCheckoutInspection{}, "", err } - if inspection.attached { - if err := withWorkingDir(dir, func() error { - return runtime.checkCleanWorktree(opts.Verbose) - }); err != nil { - return err + if err := withWorkingDir(dir, func() error { return runtime.checkCleanWorktree(opts.Verbose) }); err != nil { + return setupCheckoutInspection{}, "", err } } + return *inspection, dir, nil +} +func runSetupRepositoryCheckWithRuntimeResult(opts SetupRepositoryCheckOptions, ownerType, dir string, inspection setupCheckoutInspection) SetupRepositoryCheckResult { result := SetupRepositoryCheckResult{ - Repository: opts.Repo, - Directory: dir, - Authenticated: true, - RepositoryExists: true, - OwnerType: ownerType, - RequiredOwnerType: opts.RequireOwnerType, - CheckoutAttached: inspection.attached, - CloneNeeded: inspection.cloneNeeded, + Repository: opts.Repo, Directory: dir, Authenticated: true, RepositoryExists: true, + OwnerType: ownerType, RequiredOwnerType: opts.RequireOwnerType, + CheckoutAttached: inspection.attached, CloneNeeded: inspection.cloneNeeded, } if inspection.attached { clean := true result.CleanWorktree = &clean } + return result +} - if opts.JSON { - return renderSetupJSON(result) - } - - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Setup repository check for "+opts.Repo)) +func runSetupRepositoryCheckWithRuntimePrint(result SetupRepositoryCheckResult, inspection setupCheckoutInspection) { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Setup repository check for "+result.Repository)) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("- GitHub CLI authenticated")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("- repository exists")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("- owner type: "+ownerType)) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("- owner type: "+result.OwnerType)) if inspection.attached { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("- attached checkout at "+dir)) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("- attached checkout at "+result.Directory)) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("- working tree is clean")) } else if inspection.cloneNeeded { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("- no checkout at %s; directory is ready for clone", dir))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("- no checkout at %s; directory is ready for clone", result.Directory))) } fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Setup repository checks passed")) - return nil } func renderSetupJSON(output any) error { diff --git a/pkg/cli/shell_completion.go b/pkg/cli/shell_completion.go index 67e128ca12a..db45064fb74 100644 --- a/pkg/cli/shell_completion.go +++ b/pkg/cli/shell_completion.go @@ -124,28 +124,27 @@ func InstallShellCompletion(verbose bool, rootCmd CommandProvider) error { // installBashCompletion installs bash completion func installBashCompletion(verbose bool, cmd *cobra.Command) error { shellCompletionLog.Print("Installing bash completion") - - // Generate completion script using Cobra var buf bytes.Buffer if err := cmd.GenBashCompletion(&buf); err != nil { return fmt.Errorf("failed to generate bash completion: %w", err) } - - completionScript := buf.String() - - // Determine installation path - var completionPath string homeDir, err := os.UserHomeDir() if err != nil { return fmt.Errorf("failed to get home directory: %w", err) } + completionPath := installBashCompletionPath(homeDir) + completionPath, err = installBashCompletionWrite(completionPath, homeDir, buf.String()) + if err != nil { + return err + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Installed bash completion to: "+completionPath)) + return installBashCompletionInstructions(completionPath, homeDir) +} - // Try to determine the best location for bash completions +func installBashCompletionPath(homeDir string) string { if runtime.GOOS == "darwin" { - // macOS with Homebrew brewPrefix := os.Getenv("HOMEBREW_PREFIX") //nolint:osgetenvlibrary if brewPrefix == "" { - // Try common locations for _, prefix := range []string{constants.HomebrewPrefix, constants.UsrLocalPrefix} { if fileutil.DirExists(filepath.Join(prefix, "etc", "bash_completion.d")) { brewPrefix = prefix @@ -154,126 +153,95 @@ func installBashCompletion(verbose bool, cmd *cobra.Command) error { } } if brewPrefix != "" { - completionPath = filepath.Join(brewPrefix, "etc", "bash_completion.d", "gh-aw") - } else { - completionPath = filepath.Join(homeDir, ".bash_completion.d", "gh-aw") - } - } else { - // Linux - if fileutil.DirExists(constants.BashCompletionDir) { - completionPath = constants.BashCompletionGhAwPath - } else { - completionPath = filepath.Join(homeDir, ".bash_completion.d", "gh-aw") + return filepath.Join(brewPrefix, "etc", "bash_completion.d", "gh-aw") } + return filepath.Join(homeDir, ".bash_completion.d", "gh-aw") + } + if fileutil.DirExists(constants.BashCompletionDir) { + return constants.BashCompletionGhAwPath } + return filepath.Join(homeDir, ".bash_completion.d", "gh-aw") +} - // Create directory if needed (for user-level installations) +func installBashCompletionWrite(completionPath, homeDir, completionScript string) (string, error) { completionDir := filepath.Dir(completionPath) if strings.HasPrefix(completionDir, homeDir) { - // Use restrictive permissions (0750) following principle of least privilege if err := os.MkdirAll(completionDir, constants.DirPermSensitive); err != nil { - return fmt.Errorf("failed to create completion directory: %w", err) + return "", fmt.Errorf("failed to create completion directory: %w", err) } } - - // Try to write completion file - // Use restrictive permissions (0600) following principle of least privilege - err = os.WriteFile(completionPath, []byte(completionScript), constants.FilePermSensitive) + err := os.WriteFile(completionPath, []byte(completionScript), constants.FilePermSensitive) if err != nil && strings.HasPrefix(completionPath, "/etc") { - // If system-wide installation fails, fall back to user directory shellCompletionLog.Printf("Failed to install system-wide, falling back to user directory: %v", err) completionPath = filepath.Join(homeDir, ".bash_completion.d", "gh-aw") - // Use restrictive permissions (0750) following principle of least privilege if err := os.MkdirAll(filepath.Dir(completionPath), constants.DirPermSensitive); err != nil { - return fmt.Errorf("failed to create user completion directory: %w", err) + return "", fmt.Errorf("failed to create user completion directory: %w", err) } - // Use restrictive permissions (0600) following principle of least privilege if err := os.WriteFile(completionPath, []byte(completionScript), constants.FilePermSensitive); err != nil { - return fmt.Errorf("failed to write completion file: %w", err) + return "", fmt.Errorf("failed to write completion file: %w", err) } } else if err != nil { - return fmt.Errorf("failed to write completion file: %w", err) + return "", fmt.Errorf("failed to write completion file: %w", err) } + return completionPath, nil +} - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Installed bash completion to: "+completionPath)) - - // Check if .bashrc sources completions +func installBashCompletionInstructions(completionPath, homeDir string) error { bashrcPath := filepath.Join(homeDir, ".bashrc") - if strings.HasPrefix(completionPath, homeDir) { - // For user-level installations, check if .bashrc sources the completion directory - // Clean and validate the path to prevent path traversal - cleanBashrcPath := filepath.Clean(bashrcPath) - if !filepath.IsAbs(cleanBashrcPath) { - shellCompletionLog.Printf("Invalid bashrc path (not absolute): %s", bashrcPath) - return fmt.Errorf("invalid bashrc path: %s", bashrcPath) - } - // #nosec G304 -- bashrcPath is constructed from trusted os.UserHomeDir() and a constant filename - bashrcContent, err := os.ReadFile(cleanBashrcPath) - needsSourceLine := true - if err == nil { - if strings.Contains(string(bashrcContent), ".bash_completion.d") || - strings.Contains(string(bashrcContent), completionPath) { - needsSourceLine = false - } - } - - if needsSourceLine { - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("To enable completions, add the following to your ~/.bashrc:")) - fmt.Fprintln(os.Stderr, "") - fmt.Fprintf(os.Stderr, " for f in ~/.bash_completion.d/*; do [ -f \"$f\" ] && source \"$f\"; done\n") - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Then restart your shell or run: source ~/.bashrc")) - } else { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please restart your shell for completions to take effect")) - } - } else { + if !strings.HasPrefix(completionPath, homeDir) { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please restart your shell for completions to take effect")) + return nil } - + cleanBashrcPath := filepath.Clean(bashrcPath) + if !filepath.IsAbs(cleanBashrcPath) { + shellCompletionLog.Printf("Invalid bashrc path (not absolute): %s", bashrcPath) + return fmt.Errorf("invalid bashrc path: %s", bashrcPath) + } + // #nosec G304 -- bashrcPath is constructed from trusted os.UserHomeDir() and a constant filename + bashrcContent, err := os.ReadFile(cleanBashrcPath) + needsSourceLine := err != nil || (!strings.Contains(string(bashrcContent), ".bash_completion.d") && !strings.Contains(string(bashrcContent), completionPath)) + installBashCompletionPrintInstruction(needsSourceLine) return nil } +func installBashCompletionPrintInstruction(needsSourceLine bool) { + if needsSourceLine { + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("To enable completions, add the following to your ~/.bashrc:")) + fmt.Fprintln(os.Stderr, "") + fmt.Fprintf(os.Stderr, " for f in ~/.bash_completion.d/*; do [ -f \"$f\" ] && source \"$f\"; done\n") + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Then restart your shell or run: source ~/.bashrc")) + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please restart your shell for completions to take effect")) +} + // installZshCompletion installs zsh completion func installZshCompletion(verbose bool, cmd *cobra.Command) error { shellCompletionLog.Print("Installing zsh completion") - - // Generate completion script using Cobra var buf bytes.Buffer if err := cmd.GenZshCompletion(&buf); err != nil { return fmt.Errorf("failed to generate zsh completion: %w", err) } - - completionScript := buf.String() - - // Determine installation path homeDir, err := os.UserHomeDir() if err != nil { return fmt.Errorf("failed to get home directory: %w", err) } - - // Check for fpath directories - var completionPath string - - // Try user's local completion directory first userCompletionDir := filepath.Join(homeDir, ".zsh", "completions") - // Use restrictive permissions (0750) following principle of least privilege if err := os.MkdirAll(userCompletionDir, constants.DirPermSensitive); err != nil { return fmt.Errorf("failed to create completion directory: %w", err) } - completionPath = filepath.Join(userCompletionDir, "_gh-aw") - - // Write completion file - // Use restrictive permissions (0600) following principle of least privilege - if err := os.WriteFile(completionPath, []byte(completionScript), constants.FilePermSensitive); err != nil { + completionPath := filepath.Join(userCompletionDir, "_gh-aw") + if err := os.WriteFile(completionPath, []byte(buf.String()), constants.FilePermSensitive); err != nil { return fmt.Errorf("failed to write completion file: %w", err) } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Installed zsh completion to: "+completionPath)) + return installZshCompletionInstructions(homeDir, userCompletionDir) +} - // Check if .zshrc configures fpath +func installZshCompletionInstructions(homeDir, userCompletionDir string) error { zshrcPath := filepath.Join(homeDir, ".zshrc") - // Clean and validate the path to prevent path traversal cleanZshrcPath := filepath.Clean(zshrcPath) if !filepath.IsAbs(cleanZshrcPath) { shellCompletionLog.Printf("Invalid zshrc path (not absolute): %s", zshrcPath) @@ -281,13 +249,7 @@ func installZshCompletion(verbose bool, cmd *cobra.Command) error { } // #nosec G304 -- zshrcPath is constructed from trusted os.UserHomeDir() and a constant filename zshrcContent, err := os.ReadFile(cleanZshrcPath) - needsFpath := true - if err == nil { - if strings.Contains(string(zshrcContent), userCompletionDir) { - needsFpath = false - } - } - + needsFpath := err != nil || !strings.Contains(string(zshrcContent), userCompletionDir) if needsFpath { fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, console.FormatInfoMessage("To enable completions, add the following to your ~/.zshrc:")) @@ -299,7 +261,6 @@ func installZshCompletion(verbose bool, cmd *cobra.Command) error { } else { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please restart your shell for completions to take effect")) } - return nil } @@ -417,35 +378,10 @@ func uninstallBashCompletion(verbose bool) error { return fmt.Errorf("failed to get home directory: %w", err) } - // Check all possible locations where completion might be installed - var possiblePaths []string - - // User-level installations - possiblePaths = append(possiblePaths, filepath.Join(homeDir, ".bash_completion.d", "gh-aw")) - - // macOS with Homebrew - if runtime.GOOS == "darwin" { - brewPrefix := os.Getenv("HOMEBREW_PREFIX") //nolint:osgetenvlibrary - if brewPrefix == "" { - for _, prefix := range []string{constants.HomebrewPrefix, constants.UsrLocalPrefix} { - if fileutil.DirExists(filepath.Join(prefix, "etc", "bash_completion.d")) { - possiblePaths = append(possiblePaths, filepath.Join(prefix, "etc", "bash_completion.d", "gh-aw")) - } - } - } else { - possiblePaths = append(possiblePaths, filepath.Join(brewPrefix, "etc", "bash_completion.d", "gh-aw")) - } - } - - // System-wide installations (Linux) - if runtime.GOOS == "linux" { - possiblePaths = append(possiblePaths, constants.BashCompletionGhAwPath) - } - removed := false var lastErr error - for _, path := range possiblePaths { + for _, path := range uninstallBashCompletionPaths(homeDir) { if fileutil.FileExists(path) { shellCompletionLog.Printf("Found completion file at: %s", path) if err := os.Remove(path); err != nil { @@ -471,6 +407,26 @@ func uninstallBashCompletion(verbose bool) error { return nil } +func uninstallBashCompletionPaths(homeDir string) []string { + possiblePaths := []string{filepath.Join(homeDir, ".bash_completion.d", "gh-aw")} + if runtime.GOOS == "darwin" { + brewPrefix := os.Getenv("HOMEBREW_PREFIX") //nolint:osgetenvlibrary + if brewPrefix == "" { + for _, prefix := range []string{constants.HomebrewPrefix, constants.UsrLocalPrefix} { + if fileutil.DirExists(filepath.Join(prefix, "etc", "bash_completion.d")) { + possiblePaths = append(possiblePaths, filepath.Join(prefix, "etc", "bash_completion.d", "gh-aw")) + } + } + } else { + possiblePaths = append(possiblePaths, filepath.Join(brewPrefix, "etc", "bash_completion.d", "gh-aw")) + } + } + if runtime.GOOS == "linux" { + possiblePaths = append(possiblePaths, constants.BashCompletionGhAwPath) + } + return possiblePaths +} + // uninstallZshCompletion uninstalls zsh completion func uninstallZshCompletion(verbose bool) error { shellCompletionLog.Print("Uninstalling zsh completion") diff --git a/pkg/cli/signal_aware_poll.go b/pkg/cli/signal_aware_poll.go index 7c4b5ebf714..2d5b408849d 100644 --- a/pkg/cli/signal_aware_poll.go +++ b/pkg/cli/signal_aware_poll.go @@ -57,81 +57,80 @@ type PollOptions struct { // This provides a reusable pattern for any operation that needs to poll with graceful Ctrl-C handling func PollWithSignalHandling(options PollOptions) error { pollLog.Printf("Starting polling: interval=%v, timeout=%v", options.PollInterval, options.Timeout) - if options.Verbose && options.StartMessage != "" { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(options.StartMessage)) } - - // Use provided context or fall back to background context - ctx := options.Ctx - if ctx == nil { - ctx = context.Background() - } - - // Set up signal handling for graceful shutdown - // Signal channel provides a fallback when no context is provided or for direct OS signals + ctx := pollWithSignalHandlingContext(options) sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) defer signal.Stop(sigChan) - - // Set up timeout start := time.Now() ticker := time.NewTicker(options.PollInterval) defer ticker.Stop() - // Perform initial check immediately - result, err := options.PollFunc(ctx) - switch result { - case PollSuccess: - if options.Verbose && options.SuccessMessage != "" { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(options.SuccessMessage)) - } - return nil - case PollFailure: + initialResult, initialErr := options.PollFunc(ctx) + if done, err := pollWithSignalHandlingResult(initialResult, initialErr, options); done || err != nil { return err } - - // Continue polling for { select { case <-ctx.Done(): - pollLog.Printf("Context cancelled (%v), stopping poll", ctx.Err()) - msg := "Operation cancelled, stopping wait..." - if err := ctx.Err(); err != nil { - msg = fmt.Sprintf("Operation cancelled (%v), stopping wait...", err) - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(msg)) - return ErrInterrupted - + return pollWithSignalHandlingContextDone(ctx) case <-sigChan: pollLog.Print("Received interrupt signal") fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Received interrupt signal, stopping wait...")) return ErrInterrupted - case <-ticker.C: - // Check if timeout exceeded - if options.Timeout > 0 && time.Since(start) > options.Timeout { - pollLog.Printf("Timeout exceeded: %v", options.Timeout) - return fmt.Errorf("operation timed out after %v", options.Timeout) - } - - // Poll for status - result, err := options.PollFunc(ctx) - - switch result { - case PollSuccess: - if options.Verbose && options.SuccessMessage != "" { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(options.SuccessMessage)) - } - return nil - case PollFailure: + if err := pollWithSignalHandlingTick(ctx, start, options); err != nil { return err } + } + } +} - // Still waiting, show progress if enabled - if options.Verbose && options.ProgressMessage != "" { - fmt.Fprintln(os.Stderr, console.FormatProgressMessage(options.ProgressMessage)) - } +func pollWithSignalHandlingContext(options PollOptions) context.Context { + ctx := options.Ctx + if ctx == nil { + ctx = context.Background() + } + return ctx +} + +func pollWithSignalHandlingResult(result PollResult, err error, options PollOptions) (bool, error) { + switch result { + case PollSuccess: + if options.Verbose && options.SuccessMessage != "" { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(options.SuccessMessage)) } + return true, nil + case PollFailure: + return true, err + default: + return false, nil + } +} + +func pollWithSignalHandlingContextDone(ctx context.Context) error { + pollLog.Printf("Context cancelled (%v), stopping poll", ctx.Err()) + msg := "Operation cancelled, stopping wait..." + if err := ctx.Err(); err != nil { + msg = fmt.Sprintf("Operation cancelled (%v), stopping wait...", err) + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(msg)) + return ErrInterrupted +} + +func pollWithSignalHandlingTick(ctx context.Context, start time.Time, options PollOptions) error { + if options.Timeout > 0 && time.Since(start) > options.Timeout { + pollLog.Printf("Timeout exceeded: %v", options.Timeout) + return fmt.Errorf("operation timed out after %v", options.Timeout) + } + result, err := options.PollFunc(ctx) + if done, err := pollWithSignalHandlingResult(result, err, options); done || err != nil { + return err + } + if options.Verbose && options.ProgressMessage != "" { + fmt.Fprintln(os.Stderr, console.FormatProgressMessage(options.ProgressMessage)) } + return nil } diff --git a/pkg/cli/spec.go b/pkg/cli/spec.go index db77d22c5c6..936873fc3ab 100644 --- a/pkg/cli/spec.go +++ b/pkg/cli/spec.go @@ -273,153 +273,106 @@ func explicitHostForRepo(repoSlug string) string { func parseWorkflowSpec(spec string) (*WorkflowSpec, error) { specLog.Printf("Parsing workflow spec: %q", spec) - - // Check if this is a GitHub URL if strings.HasPrefix(spec, "http://") || strings.HasPrefix(spec, "https://") { - specLog.Print("Detected URL format") - // Try to parse as a recognized GitHub URL first. - parsedURL, urlErr := url.Parse(spec) - if urlErr == nil && isGitHubHost(parsedURL.Host) { - specLog.Print("Detected GitHub URL format") - // Rewrite dotcom automations UI URLs to the CAPI endpoint before further parsing. - if capiURL, ok := rewriteAutomationsURL(parsedURL); ok { - specLog.Printf("Rewrote automations UI URL to CAPI URL") - return &WorkflowSpec{ - RawURL: capiURL, - WorkflowName: genericURLWorkflowName(capiURL), - }, nil - } - return parseGitHubURL(spec) - } - // Non-GitHub HTTP(S) URL: return a generic URL spec whose content will be - // fetched at resolution time and dispatched on Content-Type. - if urlErr != nil { - return nil, fmt.Errorf("invalid URL %q: %w", spec, urlErr) - } - if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { - return nil, fmt.Errorf("unsupported URL scheme %q: only http and https are supported", parsedURL.Scheme) - } - specLog.Printf("Detected generic import URL: %s", spec) - return &WorkflowSpec{ - RawURL: spec, - WorkflowName: genericURLWorkflowName(spec), - }, nil + return parseWorkflowSpecURL(spec) } - - // Check if this is a local path if isLocalWorkflowPath(spec) { - specLog.Print("Detected local path format") + return parseWorkflowSpecLocal(spec) + } + return parseWorkflowSpecRemote(spec) +} - ws, err := parseLocalWorkflowSpec(spec) - if err != nil { - return nil, err +func parseWorkflowSpecURL(spec string) (*WorkflowSpec, error) { + specLog.Print("Detected URL format") + parsedURL, urlErr := url.Parse(spec) + if urlErr == nil && isGitHubHost(parsedURL.Host) { + specLog.Print("Detected GitHub URL format") + if capiURL, ok := rewriteAutomationsURL(parsedURL); ok { + specLog.Printf("Rewrote automations UI URL to CAPI URL") + return &WorkflowSpec{RawURL: capiURL, WorkflowName: genericURLWorkflowName(capiURL)}, nil } + return parseGitHubURL(spec) + } + if urlErr != nil { + return nil, fmt.Errorf("invalid URL %q: %w", spec, urlErr) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return nil, fmt.Errorf("unsupported URL scheme %q: only http and https are supported", parsedURL.Scheme) + } + specLog.Printf("Detected generic import URL: %s", spec) + return &WorkflowSpec{RawURL: spec, WorkflowName: genericURLWorkflowName(spec)}, nil +} - // Detect local wildcard specs like "./*.md" and mark them so that - // downstream expansion (e.g., expandLocalWildcardWorkflows) can run. - if strings.ContainsAny(spec, "*?[") { - ws.IsWildcard = true - // Ensure a stable WorkflowName for wildcard specs. - if ws.WorkflowName == "" { - ws.WorkflowName = spec - } +func parseWorkflowSpecLocal(spec string) (*WorkflowSpec, error) { + specLog.Print("Detected local path format") + ws, err := parseLocalWorkflowSpec(spec) + if err != nil { + return nil, err + } + if strings.ContainsAny(spec, "*?[") { + ws.IsWildcard = true + if ws.WorkflowName == "" { + ws.WorkflowName = spec } - - return ws, nil } + return ws, nil +} - // Handle version first (anything after @) +func parseWorkflowSpecRemote(spec string) (*WorkflowSpec, error) { parts := strings.SplitN(spec, "@", 2) specWithoutVersion := parts[0] - var version string + version := "" if len(parts) == 2 { version = parts[1] } - - // Split by slashes slashParts := strings.Split(specWithoutVersion, "/") - - // Must have at least 3 parts: owner/repo/workflow-path if len(slashParts) < 3 { return nil, errors.New("workflow specification must be in format 'owner/repo/workflow-name[@version]'") } - - owner := slashParts[0] - repo := slashParts[1] - - // Check if this is a /files/REF/ format (e.g., owner/repo/files/main/path.md) - // This is the format used when copying file paths from GitHub UI - var workflowPath string - if len(slashParts) >= 4 && slashParts[2] == "files" { - // Extract the ref (branch/tag/commit) from slashParts[3] - ref := slashParts[3] - // The file path is everything after /files/REF/ - workflowPath = strings.Join(slashParts[4:], "/") - - // If version was not explicitly provided via @, use the ref from /files/REF/ - if version == "" { - version = ref - } - } else { - // Standard format: owner/repo/path or owner/repo/workflow-name - workflowPath = strings.Join(slashParts[2:], "/") - } - - // Validate owner and repo parts are not empty + owner, repo := slashParts[0], slashParts[1] + workflowPath, version := parseWorkflowSpecRemotePath(slashParts, version) if owner == "" || repo == "" { return nil, errors.New("invalid workflow specification: owner and repo cannot be empty") } - - // Basic validation that owner and repo look like GitHub identifiers if !parser.IsValidGitHubIdentifier(owner) || !parser.IsValidGitHubRepositoryName(repo) { return nil, fmt.Errorf("invalid workflow specification: '%s/%s' does not look like a valid GitHub repository", owner, repo) } - repoSlug := fmt.Sprintf("%s/%s", owner, repo) - - // Determine the API host for this repo. getGitHubHostForRepo returns the canonical - // host, which for well-known public-only repos (githubnext/agentics, github/gh-aw) - // is always public GitHub regardless of GHE configuration. If the repo's canonical - // host differs from the configured host, record the explicit hostname so API fetches - // target the correct server. explicitHost := explicitHostForRepo(repoSlug) - - // Check if this is a wildcard specification (owner/repo/*) if workflowPath == "*" { - return &WorkflowSpec{ - RepoSpec: RepoSpec{ - RepoSlug: repoSlug, - Version: version, - }, - WorkflowPath: "*", - WorkflowName: "*", - IsWildcard: true, - Host: explicitHost, - }, nil - } - - // Handle different cases based on the number of path parts - if len(slashParts) == 3 && !strings.HasSuffix(workflowPath, ".md") { - // Three-part spec: owner/repo/workflow-name - // Add "workflows/" prefix - workflowPath = "workflows/" + workflowPath + ".md" - } else { - // Four or more parts: owner/repo/workflows/workflow-name or owner/repo/path/to/workflow-name - // Require .md extension to be explicit - if !strings.HasSuffix(workflowPath, ".md") { - return nil, fmt.Errorf("workflow specification with path must end with '.md' extension: %s", workflowPath) + return parseWorkflowSpecRemoteWildcard(repoSlug, version, explicitHost), nil + } + workflowPath, err := parseWorkflowSpecRemoteWorkflowPath(slashParts, workflowPath) + if err != nil { + return nil, err + } + return &WorkflowSpec{RepoSpec: RepoSpec{RepoSlug: repoSlug, Version: version}, WorkflowPath: workflowPath, WorkflowName: normalizeWorkflowID(workflowPath), Host: explicitHost}, nil +} + +func parseWorkflowSpecRemotePath(slashParts []string, version string) (string, string) { + if len(slashParts) >= 4 && slashParts[2] == "files" { + ref := slashParts[3] + workflowPath := strings.Join(slashParts[4:], "/") + if version == "" { + version = ref } + return workflowPath, version } + return strings.Join(slashParts[2:], "/"), version +} - return &WorkflowSpec{ - RepoSpec: RepoSpec{ - RepoSlug: repoSlug, - Version: version, - }, - WorkflowPath: workflowPath, - WorkflowName: normalizeWorkflowID(workflowPath), - Host: explicitHost, - }, nil +func parseWorkflowSpecRemoteWildcard(repoSlug, version, explicitHost string) *WorkflowSpec { + return &WorkflowSpec{RepoSpec: RepoSpec{RepoSlug: repoSlug, Version: version}, WorkflowPath: "*", WorkflowName: "*", IsWildcard: true, Host: explicitHost} +} + +func parseWorkflowSpecRemoteWorkflowPath(slashParts []string, workflowPath string) (string, error) { + if len(slashParts) == 3 && !strings.HasSuffix(workflowPath, ".md") { + return "workflows/" + workflowPath + ".md", nil + } + if !strings.HasSuffix(workflowPath, ".md") { + return "", fmt.Errorf("workflow specification with path must end with '.md' extension: %s", workflowPath) + } + return workflowPath, nil } // parseLocalWorkflowSpec parses a local workflow specification starting with "./" diff --git a/pkg/cli/status_command.go b/pkg/cli/status_command.go index 00a30aa63f0..6301a7afbcd 100644 --- a/pkg/cli/status_command.go +++ b/pkg/cli/status_command.go @@ -42,161 +42,152 @@ type WorkflowStatus struct { // For CLI usage, use StatusWorkflows which handles output formatting. func GetWorkflowStatuses(pattern string, ref string, labelFilter string, repoOverride string) ([]WorkflowStatus, error) { statusLog.Printf("Getting workflow statuses: pattern=%s, ref=%s, labelFilter=%s, repo=%s", pattern, ref, labelFilter, repoOverride) - - // Get GitHub workflows data - statusLog.Print("Fetching GitHub workflow status") - githubWorkflows, err := fetchGitHubWorkflows(repoOverride, false) - if err != nil { - statusLog.Printf("Failed to fetch GitHub workflows: %v", err) - githubWorkflows = make(map[string]*GitHubWorkflow) - } else { - statusLog.Printf("Successfully fetched %d GitHub workflows", len(githubWorkflows)) - } - - // Fetch latest workflow runs for ref if specified - var latestRunsByWorkflow map[string]*WorkflowRun - if ref != "" { - latestRunsByWorkflow, err = fetchLatestRunsByRef(ref, repoOverride, false) - if err != nil { - statusLog.Printf("Failed to fetch workflow runs for ref %s: %v", ref, err) - latestRunsByWorkflow = make(map[string]*WorkflowRun) - } else { - statusLog.Printf("Successfully fetched %d workflow runs for ref %s", len(latestRunsByWorkflow), ref) - } - } - - // When --repo is specified, build statuses from GitHub API data only. - // Local markdown files are not available for a remote repository, so - // local-only fields (EngineID, Compiled, TimeRemaining, Labels, On) are - // omitted from the results. + githubWorkflows := getWorkflowStatusesGitHub(repoOverride) + latestRunsByWorkflow := getWorkflowStatusesLatestRuns(ref, repoOverride) if repoOverride != "" { - // Label metadata is not exposed by the GitHub Actions workflow API, so - // filtering by label is not supported when --repo is specified. if labelFilter != "" { return nil, errors.New("--label filter is not supported with --repo: label information is not available from the GitHub Actions API") } return buildRemoteWorkflowStatuses(pattern, githubWorkflows, latestRunsByWorkflow), nil } - - // Local path: discover markdown workflow files from the local filesystem. mdFiles, err := getMarkdownWorkflowFiles("") if err != nil { statusLog.Printf("Failed to get markdown workflow files: %v", err) return nil, fmt.Errorf("failed to get markdown workflow files: %w", err) } - statusLog.Printf("Found %d markdown workflow files", len(mdFiles)) if len(mdFiles) == 0 { return []WorkflowStatus{}, nil } + return getWorkflowStatusesLocal(mdFiles, pattern, labelFilter, githubWorkflows, latestRunsByWorkflow), nil +} - // Build status list +func getWorkflowStatusesGitHub(repoOverride string) map[string]*GitHubWorkflow { + statusLog.Print("Fetching GitHub workflow status") + githubWorkflows, err := fetchGitHubWorkflows(repoOverride, false) + if err != nil { + statusLog.Printf("Failed to fetch GitHub workflows: %v", err) + return make(map[string]*GitHubWorkflow) + } + statusLog.Printf("Successfully fetched %d GitHub workflows", len(githubWorkflows)) + return githubWorkflows +} + +func getWorkflowStatusesLatestRuns(ref, repoOverride string) map[string]*WorkflowRun { + if ref == "" { + return nil + } + latestRunsByWorkflow, err := fetchLatestRunsByRef(ref, repoOverride, false) + if err != nil { + statusLog.Printf("Failed to fetch workflow runs for ref %s: %v", ref, err) + return make(map[string]*WorkflowRun) + } + statusLog.Printf("Successfully fetched %d workflow runs for ref %s", len(latestRunsByWorkflow), ref) + return latestRunsByWorkflow +} + +func getWorkflowStatusesLocal(mdFiles []string, pattern, labelFilter string, githubWorkflows map[string]*GitHubWorkflow, latestRunsByWorkflow map[string]*WorkflowRun) []WorkflowStatus { var statuses []WorkflowStatus for _, file := range mdFiles { - base := filepath.Base(file) - name := strings.TrimSuffix(base, ".md") - - // Skip if pattern specified and doesn't match - if pattern != "" && !strings.Contains(strings.ToLower(name), strings.ToLower(pattern)) { - continue + status, ok := getWorkflowStatusesLocalFile(file, pattern, labelFilter, githubWorkflows, latestRunsByWorkflow) + if ok { + statuses = append(statuses, status) } + } + return statuses +} - // Extract engine ID from workflow file - agent := extractEngineIDFromFile(file) - - // Check if compiled (.lock.yml file is in .github/workflows) - lockFile := stringutil.MarkdownToLockFile(file) - compiled := "N/A" - timeRemaining := "N/A" - - if fileutil.FileExists(lockFile) { - // Check if up to date using hash comparison - compiled = isCompiledUpToDate(file, lockFile) - - // Extract stop-time from lock file - if stopTime := workflow.ExtractStopTimeFromLockFile(lockFile); stopTime != "" { - timeRemaining = calculateTimeRemaining(stopTime) - } - } +func getWorkflowStatusesLocalFile(file, pattern, labelFilter string, githubWorkflows map[string]*GitHubWorkflow, latestRunsByWorkflow map[string]*WorkflowRun) (WorkflowStatus, bool) { + name := strings.TrimSuffix(filepath.Base(file), ".md") + if pattern != "" && !strings.Contains(strings.ToLower(name), strings.ToLower(pattern)) { + return WorkflowStatus{}, false + } + agent := extractEngineIDFromFile(file) + compiled, timeRemaining := getWorkflowStatusesCompiled(file) + onField, labels, dependencies := getWorkflowStatusesFrontmatter(file) + if labelFilter != "" && !getWorkflowStatusesHasLabel(labels, labelFilter) { + return WorkflowStatus{}, false + } + runID, runStatus, runConclusion := getWorkflowStatusesRun(name, latestRunsByWorkflow) + return WorkflowStatus{ + WorkflowListItem: WorkflowListItem{Workflow: name, EngineID: agent, Compiled: compiled, Labels: labels, On: onField}, + Status: getWorkflowStatusesGitHubState(name, githubWorkflows), + TimeRemaining: timeRemaining, + Dependencies: dependencies, + RunID: runID, + RunStatus: runStatus, + RunConclusion: runConclusion, + }, true +} - // Get GitHub workflow status - status := "Unknown" - if workflow, exists := githubWorkflows[name]; exists { - if workflow.State == "disabled_manually" { - status = "disabled" - } else { - status = workflow.State - } +func getWorkflowStatusesCompiled(file string) (string, string) { + lockFile := stringutil.MarkdownToLockFile(file) + compiled := "N/A" + timeRemaining := "N/A" + if fileutil.FileExists(lockFile) { + compiled = isCompiledUpToDate(file, lockFile) + if stopTime := workflow.ExtractStopTimeFromLockFile(lockFile); stopTime != "" { + timeRemaining = calculateTimeRemaining(stopTime) } + } + return compiled, timeRemaining +} - // Extract "on" field and labels from frontmatter - var onField any - var labels []string - var dependencies []string - if content, err := os.ReadFile(file); err == nil { - contentStr := string(content) - if result, err := parser.ExtractFrontmatterFromContent(contentStr); err == nil { - if result.Frontmatter != nil { - onField = result.Frontmatter["on"] - dependencies = extractWorkflowDependencies(contentStr, result.Frontmatter) - // Extract labels field if present - if labelsField, ok := result.Frontmatter["labels"]; ok { - if labelsArray, ok := labelsField.([]any); ok { - for _, label := range labelsArray { - if labelStr, ok := label.(string); ok { - labels = append(labels, labelStr) - } - } - } - } - } - } +func getWorkflowStatusesGitHubState(name string, githubWorkflows map[string]*GitHubWorkflow) string { + status := "Unknown" + if workflow, exists := githubWorkflows[name]; exists { + if workflow.State == "disabled_manually" { + return "disabled" } + status = workflow.State + } + return status +} - // Skip if label filter specified and workflow doesn't have the label - if labelFilter != "" { - hasLabel := false - for _, label := range labels { - if strings.EqualFold(label, labelFilter) { - hasLabel = true - break +func getWorkflowStatusesFrontmatter(file string) (any, []string, []string) { + var onField any + var labels []string + var dependencies []string + content, err := os.ReadFile(file) + if err != nil { + return onField, labels, dependencies + } + contentStr := string(content) + result, err := parser.ExtractFrontmatterFromContent(contentStr) + if err != nil || result.Frontmatter == nil { + return onField, labels, dependencies + } + onField = result.Frontmatter["on"] + dependencies = extractWorkflowDependencies(contentStr, result.Frontmatter) + if labelsField, ok := result.Frontmatter["labels"]; ok { + if labelsArray, ok := labelsField.([]any); ok { + for _, label := range labelsArray { + if labelStr, ok := label.(string); ok { + labels = append(labels, labelStr) } } - if !hasLabel { - continue - } } + } + return onField, labels, dependencies +} - // Get run status for ref if available - var runID int64 - var runStatus, runConclusion string - if latestRunsByWorkflow != nil { - if run, exists := latestRunsByWorkflow[name]; exists { - runID = run.DatabaseID - runStatus = run.Status - runConclusion = run.Conclusion - } +func getWorkflowStatusesHasLabel(labels []string, labelFilter string) bool { + for _, label := range labels { + if strings.EqualFold(label, labelFilter) { + return true } - - // Build status object - statuses = append(statuses, WorkflowStatus{ - WorkflowListItem: WorkflowListItem{ - Workflow: name, - EngineID: agent, - Compiled: compiled, - Labels: labels, - On: onField, - }, - Status: status, - TimeRemaining: timeRemaining, - Dependencies: dependencies, - RunID: runID, - RunStatus: runStatus, - RunConclusion: runConclusion, - }) } + return false +} - return statuses, nil +func getWorkflowStatusesRun(name string, latestRunsByWorkflow map[string]*WorkflowRun) (int64, string, string) { + if latestRunsByWorkflow == nil { + return 0, "", "" + } + if run, exists := latestRunsByWorkflow[name]; exists { + return run.DatabaseID, run.Status, run.Conclusion + } + return 0, "", "" } // buildRemoteWorkflowStatuses constructs workflow statuses from GitHub API data when @@ -484,96 +475,83 @@ func isCompiledUpToDateWithCache(workflowPath, lockFilePath string, cache *parse // fetchLatestRunsByRef fetches the latest workflow run for each workflow from a specific ref (branch or tag) func fetchLatestRunsByRef(ref string, repoOverride string, verbose bool) (map[string]*WorkflowRun, error) { statusLog.Printf("Fetching latest workflow runs for ref: %s, repo: %s", ref, repoOverride) - - // Start spinner for network operation (only if not in verbose mode) spinner := console.NewSpinner("Fetching workflow runs for ref...") if !verbose { spinner.Start() } - - // Fetch workflow runs for the ref (uses --branch flag which also works for tags) - args := []string{"run", "list", "--branch", ref, "--json", "databaseId,number,url,status,conclusion,workflowName,createdAt,headBranch", "--limit", "100"} - if repoOverride != "" { - args = append(args, "--repo", repoOverride) + output, err := fetchLatestRunsByRefOutput(ref, repoOverride) + if err != nil { + if !verbose { + spinner.Stop() + } + return nil, fetchLatestRunsByRefError(err, ref, repoOverride) } - cmd := workflow.ExecGH(args...) - output, err := cmd.Output() - + runs, err := fetchLatestRunsByRefParse(output) if err != nil { - // Stop spinner on error if !verbose { spinner.Stop() } + return nil, err + } + if !verbose { + spinner.StopWithMessage(fmt.Sprintf("✓ Fetched %d workflow runs", len(runs))) + } + latestRuns := fetchLatestRunsByRefMap(runs) + statusLog.Printf("Fetched latest runs for %d workflows on ref %s", len(latestRuns), ref) + return latestRuns, nil +} - // Extract detailed error information including exit code and stderr - var exitCode int - var stderr string - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - exitCode = exitErr.ExitCode() - stderr = string(exitErr.Stderr) - statusLog.Printf("gh run list command failed with exit code %d. Command: gh %v", exitCode, args) - statusLog.Printf("stderr output: %s", stderr) - - // Check for invalid field errors first (before generic error) - // GitHub CLI returns these when JSON fields don't exist or are misspelled - combinedMsg := err.Error() + " " + stderr - if strings.Contains(combinedMsg, "invalid field") || - strings.Contains(combinedMsg, "unknown field") || - strings.Contains(combinedMsg, "field not found") || - strings.Contains(combinedMsg, "no such field") { - return nil, fmt.Errorf("invalid field in JSON query (exit code %d): %s", exitCode, stderr) - } +func fetchLatestRunsByRefOutput(ref string, repoOverride string) ([]byte, error) { + args := []string{"run", "list", "--branch", ref, "--json", "databaseId,number,url,status,conclusion,workflowName,createdAt,headBranch", "--limit", "100"} + if repoOverride != "" { + args = append(args, "--repo", repoOverride) + } + return workflow.ExecGH(args...).Output() +} - return nil, fmt.Errorf("failed to execute gh run list command (exit code %d): %w. stderr: %s", exitCode, err, stderr) +func fetchLatestRunsByRefError(err error, ref string, repoOverride string) error { + args := []string{"run", "list", "--branch", ref, "--json", "databaseId,number,url,status,conclusion,workflowName,createdAt,headBranch", "--limit", "100"} + if repoOverride != "" { + args = append(args, "--repo", repoOverride) + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitCode := exitErr.ExitCode() + stderr := string(exitErr.Stderr) + statusLog.Printf("gh run list command failed with exit code %d. Command: gh %v", exitCode, args) + statusLog.Printf("stderr output: %s", stderr) + combinedMsg := err.Error() + " " + stderr + if strings.Contains(combinedMsg, "invalid field") || strings.Contains(combinedMsg, "unknown field") || strings.Contains(combinedMsg, "field not found") || strings.Contains(combinedMsg, "no such field") { + return fmt.Errorf("invalid field in JSON query (exit code %d): %s", exitCode, stderr) } - - // If not an ExitError, log what we can - statusLog.Printf("gh run list command failed with error (not ExitError): %v. Command: gh %v", err, args) - return nil, fmt.Errorf("failed to execute gh run list command: %w", err) + return fmt.Errorf("failed to execute gh run list command (exit code %d): %w. stderr: %s", exitCode, err, stderr) } + statusLog.Printf("gh run list command failed with error (not ExitError): %v. Command: gh %v", err, args) + return fmt.Errorf("failed to execute gh run list command: %w", err) +} - // Check if output is empty +func fetchLatestRunsByRefParse(output []byte) ([]WorkflowRun, error) { if len(output) == 0 { - if !verbose { - spinner.Stop() - } return nil, errors.New("gh run list returned empty output") } - - // Validate JSON before unmarshaling if !json.Valid(output) { - if !verbose { - spinner.Stop() - } return nil, errors.New("gh run list returned invalid JSON") } - var runs []WorkflowRun if err := json.Unmarshal(output, &runs); err != nil { - if !verbose { - spinner.Stop() - } return nil, fmt.Errorf("failed to parse workflow runs: %w", err) } + return runs, nil +} - // Stop spinner with success message - if !verbose { - spinner.StopWithMessage(fmt.Sprintf("✓ Fetched %d workflow runs", len(runs))) - } - - // Build map of latest run for each workflow (first occurrence is the latest) +func fetchLatestRunsByRefMap(runs []WorkflowRun) map[string]*WorkflowRun { latestRuns := make(map[string]*WorkflowRun) for i := range runs { run := &runs[i] - // Extract workflow name from workflowName field workflowName := extractWorkflowNameFromPath(run.WorkflowName) - // Only keep the first (latest) run for each workflow if _, exists := latestRuns[workflowName]; !exists { latestRuns[workflowName] = run } } - - statusLog.Printf("Fetched latest runs for %d workflows on ref %s", len(latestRuns), ref) - return latestRuns, nil + return latestRuns } diff --git a/pkg/cli/token_usage.go b/pkg/cli/token_usage.go index 55679e4b26e..59ab39a33bf 100644 --- a/pkg/cli/token_usage.go +++ b/pkg/cli/token_usage.go @@ -135,21 +135,30 @@ var subagentDispatchPattern = regexp.MustCompile(`([A-Za-z0-9][A-Za-z0-9._-]*)\( // parseTokenUsageFile parses a token-usage.jsonl file and returns the aggregated summary. func parseTokenUsageFile(filePath string) (*TokenUsageSummary, error) { tokenUsageLog.Printf("Parsing token usage file: %s", filePath) + entries, lineNum, err := parseTokenUsageFileEntries(filePath) + if err != nil { + return nil, err + } + if len(entries) == 0 { + tokenUsageLog.Print("No token usage entries found") + return nil, nil + } + summary := &TokenUsageSummary{ByModel: make(map[string]*ModelTokenUsage)} + parseTokenUsageFileAggregate(summary, entries) + tokenUsageLog.Printf("Parsed %d entries: %d input, %d output, %d cache_read, %d cache_write, %d requests", lineNum, summary.TotalInputTokens, summary.TotalOutputTokens, summary.TotalCacheReadTokens, summary.TotalCacheWriteTokens, summary.TotalRequests) + populateAIC(summary) + summary.AmbientContext = extractAmbientContextMetrics(entries) + return summary, nil +} +func parseTokenUsageFileEntries(filePath string) ([]TokenUsageEntry, int, error) { file, err := os.Open(filePath) if err != nil { - return nil, fmt.Errorf("failed to open token usage file: %w", err) + return nil, 0, fmt.Errorf("failed to open token usage file: %w", err) } defer file.Close() - - summary := &TokenUsageSummary{ - ByModel: make(map[string]*ModelTokenUsage), - } - scanner := bufio.NewScanner(file) - // Increase buffer size for potentially large lines scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) - entries := make([]TokenUsageEntry, 0) lineNum := 0 for scanner.Scan() { @@ -158,7 +167,6 @@ func parseTokenUsageFile(filePath string) (*TokenUsageSummary, error) { if line == "" { continue } - var entry TokenUsageEntry if err := json.Unmarshal([]byte(line), &entry); err != nil { tokenUsageLog.Printf("Skipping invalid JSON at line %d: %v", lineNum, err) @@ -166,18 +174,14 @@ func parseTokenUsageFile(filePath string) (*TokenUsageSummary, error) { } entries = append(entries, entry) } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading token usage file: %w", err) - } - - if len(entries) == 0 { - tokenUsageLog.Print("No token usage entries found") - return nil, nil + return nil, lineNum, fmt.Errorf("error reading token usage file: %w", err) } + return entries, lineNum, nil +} +func parseTokenUsageFileAggregate(summary *TokenUsageSummary, entries []TokenUsageEntry) { for _, entry := range entries { - // Aggregate totals summary.TotalInputTokens += entry.InputTokens summary.TotalOutputTokens += entry.OutputTokens summary.TotalCacheReadTokens += entry.CacheReadTokens @@ -185,36 +189,27 @@ func parseTokenUsageFile(filePath string) (*TokenUsageSummary, error) { summary.TotalRequests++ summary.TotalDurationMs += entry.DurationMs summary.TotalResponseBytes += entry.ResponseBytes - - // Aggregate by model - model := entry.Model - if model == "" { - model = "unknown" - } - if _, exists := summary.ByModel[model]; !exists { - summary.ByModel[model] = &ModelTokenUsage{ - Provider: entry.Provider, - } - } - m := summary.ByModel[model] - m.InputTokens += entry.InputTokens - m.OutputTokens += entry.OutputTokens - m.CacheReadTokens += entry.CacheReadTokens - m.CacheWriteTokens += entry.CacheWriteTokens - m.ReasoningTokens += entry.ReasoningTokens - m.Requests++ - m.DurationMs += entry.DurationMs - m.ResponseBytes += entry.ResponseBytes + parseTokenUsageFileAggregateModel(summary, entry) } +} - tokenUsageLog.Printf("Parsed %d entries: %d input, %d output, %d cache_read, %d cache_write, %d requests", - lineNum, summary.TotalInputTokens, summary.TotalOutputTokens, - summary.TotalCacheReadTokens, summary.TotalCacheWriteTokens, summary.TotalRequests) - - populateAIC(summary) - summary.AmbientContext = extractAmbientContextMetrics(entries) - - return summary, nil +func parseTokenUsageFileAggregateModel(summary *TokenUsageSummary, entry TokenUsageEntry) { + model := entry.Model + if model == "" { + model = "unknown" + } + if _, exists := summary.ByModel[model]; !exists { + summary.ByModel[model] = &ModelTokenUsage{Provider: entry.Provider} + } + m := summary.ByModel[model] + m.InputTokens += entry.InputTokens + m.OutputTokens += entry.OutputTokens + m.CacheReadTokens += entry.CacheReadTokens + m.CacheWriteTokens += entry.CacheWriteTokens + m.ReasoningTokens += entry.ReasoningTokens + m.Requests++ + m.DurationMs += entry.DurationMs + m.ResponseBytes += entry.ResponseBytes } func extractAmbientContextMetrics(entries []TokenUsageEntry) *AmbientContextMetrics { @@ -288,34 +283,39 @@ func parseTokenUsageTimestamp(value string) (time.Time, bool) { // findTokenUsageFile searches for token-usage.jsonl in the run directory func findTokenUsageFile(runDir string) string { - usageArtifactCandidate := filepath.Join(runDir, "usage", "agent", "token_usage.jsonl") - if fileutil.FileExists(usageArtifactCandidate) { - tokenUsageLog.Printf("Found token usage file in usage artifact: %s", usageArtifactCandidate) - return usageArtifactCandidate + if path := findTokenUsageFileKnown(runDir); path != "" { + return path } - - // Primary path: sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl - primary := filepath.Join(runDir, "sandbox", "firewall", "logs", tokenUsageJSONLPath) - if fileutil.FileExists(primary) { - tokenUsageLog.Printf("Found token usage file at primary path: %s", primary) - return primary + if path := findTokenUsageFileArtifact(runDir); path != "" { + return path } + if path := findTokenUsageFileWalk(runDir); path != "" { + return path + } + tokenUsageLog.Print("No token usage file found") + return "" +} - // AWF v0.27.7+ audit-dir path: sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl - // In newer AWF versions the proxy logs are written under --audit-dir rather than - // --proxy-logs-dir, so check this path explicitly before falling back to the walk. - awfAuditPath := filepath.Join(runDir, "sandbox", "firewall", "audit", tokenUsageJSONLPath) - if fileutil.FileExists(awfAuditPath) { - tokenUsageLog.Printf("Found token usage file at AWF audit path: %s", awfAuditPath) - return awfAuditPath +func findTokenUsageFileKnown(runDir string) string { + candidates := []struct{ path, label string }{ + {filepath.Join(runDir, "usage", "agent", "token_usage.jsonl"), "usage artifact"}, + {filepath.Join(runDir, "sandbox", "firewall", "logs", tokenUsageJSONLPath), "primary path"}, + {filepath.Join(runDir, "sandbox", "firewall", "audit", tokenUsageJSONLPath), "AWF audit path"}, + } + for _, candidate := range candidates { + if fileutil.FileExists(candidate.path) { + tokenUsageLog.Printf("Found token usage file at %s: %s", candidate.label, candidate.path) + return candidate.path + } } + return "" +} - // Check legacy firewall-audit-logs artifact directory (backward compat for older runs) +func findTokenUsageFileArtifact(runDir string) string { entries, err := os.ReadDir(runDir) if err != nil { return "" } - for _, entry := range entries { if !entry.IsDir() { continue @@ -329,8 +329,11 @@ func findTokenUsageFile(runDir string) string { } } } + return "" +} - // Walk sandbox directory for any token-usage.jsonl +func findTokenUsageFileWalk(runDir string) string { + var found string if walkErr := filepath.Walk(runDir, func(path string, info os.FileInfo, err error) error { if err != nil { tokenUsageLog.Printf("walk error at %s: %v", path, err) @@ -340,20 +343,17 @@ func findTokenUsageFile(runDir string) string { return nil } if info.Name() == "token-usage.jsonl" || info.Name() == "token_usage.jsonl" { - primary = path + found = path return filepath.SkipAll } return nil }); walkErr != nil && !errors.Is(walkErr, filepath.SkipAll) { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("filesystem error walking %s: %v", runDir, walkErr))) } - if primary != filepath.Join(runDir, "sandbox", "firewall", "logs", tokenUsageJSONLPath) { - tokenUsageLog.Printf("Found token usage file via walk: %s", primary) - return primary + if found != "" { + tokenUsageLog.Printf("Found token usage file via walk: %s", found) } - - tokenUsageLog.Print("No token usage file found") - return "" + return found } // findAgentUsageFile searches for agent_usage.json in the run directory. @@ -413,18 +413,35 @@ type agentUsageEntry struct { } func parseAgentUsageFile(filePath string) (*TokenUsageSummary, error) { + entry, err := parseAgentUsageFileEntry(filePath) + if err != nil { + return nil, err + } + model, provider := parseAgentUsageFileModel(entry) + summary := parseAgentUsageFileSummary(entry, model, provider) + if entry.AICredits > 0 { + parseAgentUsageFileAIC(summary, entry, model, provider) + } else if parseAgentUsageFileHasRaw(entry) { + populateAIC(summary) + } + tokenUsageLog.Printf("Parsed agent usage file: input=%d, output=%d, cache_read=%d, cache_write=%d", summary.TotalInputTokens, summary.TotalOutputTokens, summary.TotalCacheReadTokens, summary.TotalCacheWriteTokens) + return summary, nil +} + +func parseAgentUsageFileEntry(filePath string) (agentUsageEntry, error) { cleanPath := filepath.Clean(filePath) data, err := os.ReadFile(cleanPath) if err != nil { - return nil, fmt.Errorf("failed to read agent usage file: %w", err) + return agentUsageEntry{}, fmt.Errorf("failed to read agent usage file: %w", err) } - var entry agentUsageEntry if err := json.Unmarshal(data, &entry); err != nil { - return nil, fmt.Errorf("failed to parse agent usage file: %w", err) + return agentUsageEntry{}, fmt.Errorf("failed to parse agent usage file: %w", err) } + return entry, nil +} - // Prefer primary_model when set; fall back to model; default to "unknown". +func parseAgentUsageFileModel(entry agentUsageEntry) (string, string) { model := strings.TrimSpace(entry.PrimaryModel) if model == "" { model = strings.TrimSpace(entry.Model) @@ -432,69 +449,39 @@ func parseAgentUsageFile(filePath string) (*TokenUsageSummary, error) { if model == "" { model = "unknown" } - // Prefer provider from entry; primary_model entries may omit it. - provider := strings.TrimSpace(entry.Provider) - - summary := &TokenUsageSummary{ - TotalInputTokens: entry.InputTokens, - TotalOutputTokens: entry.OutputTokens, - TotalCacheReadTokens: entry.CacheReadTokens, - TotalCacheWriteTokens: entry.CacheWriteTokens, - ByModel: make(map[string]*ModelTokenUsage), - } + return model, strings.TrimSpace(entry.Provider) +} - hasRawTokenData := summary.TotalInputTokens > 0 || - summary.TotalOutputTokens > 0 || - summary.TotalCacheReadTokens > 0 || - summary.TotalCacheWriteTokens > 0 || - entry.ReasoningTokens > 0 - hasTokenData := hasRawTokenData - if hasTokenData { +func parseAgentUsageFileSummary(entry agentUsageEntry, model, provider string) *TokenUsageSummary { + summary := &TokenUsageSummary{TotalInputTokens: entry.InputTokens, TotalOutputTokens: entry.OutputTokens, TotalCacheReadTokens: entry.CacheReadTokens, TotalCacheWriteTokens: entry.CacheWriteTokens, ByModel: make(map[string]*ModelTokenUsage)} + if parseAgentUsageFileHasRaw(entry) { summary.TotalRequests = 1 - summary.ByModel[model] = &ModelTokenUsage{ - Provider: provider, - TokenCoreMetrics: TokenCoreMetrics{ - InputTokens: entry.InputTokens, - OutputTokens: entry.OutputTokens, - CacheReadTokens: entry.CacheReadTokens, - CacheWriteTokens: entry.CacheWriteTokens, - ReasoningTokens: entry.ReasoningTokens, - }, - Requests: 1, - } + summary.ByModel[model] = &ModelTokenUsage{Provider: provider, TokenCoreMetrics: TokenCoreMetrics{InputTokens: entry.InputTokens, OutputTokens: entry.OutputTokens, CacheReadTokens: entry.CacheReadTokens, CacheWriteTokens: entry.CacheWriteTokens, ReasoningTokens: entry.ReasoningTokens}, Requests: 1} } - ambientInputTokens := entry.InputTokens if entry.AmbientContextTokens != nil { ambientInputTokens = *entry.AmbientContextTokens } - summary.AmbientContext = &AmbientContextMetrics{ - InputTokens: ambientInputTokens, - CachedTokens: entry.CacheReadTokens, - } + summary.AmbientContext = &AmbientContextMetrics{InputTokens: ambientInputTokens, CachedTokens: entry.CacheReadTokens} + return summary +} - if entry.AICredits > 0 { - // Use the pre-computed AI Credits value written by parse_token_usage.cjs. - // This is more accurate than recomputing from raw token counts because it - // was computed at the time the run completed with full per-request pricing. - summary.TotalAIC = entry.AICredits - if summary.ByModel[model] == nil { - summary.ByModel[model] = &ModelTokenUsage{} - } - summary.ByModel[model].Provider = provider - summary.ByModel[model].InputTokens = entry.InputTokens - summary.ByModel[model].OutputTokens = entry.OutputTokens - summary.ByModel[model].CacheReadTokens = entry.CacheReadTokens - summary.ByModel[model].CacheWriteTokens = entry.CacheWriteTokens - summary.ByModel[model].ReasoningTokens = entry.ReasoningTokens - summary.ByModel[model].AIC = entry.AICredits - } else if hasRawTokenData { - populateAIC(summary) - } +func parseAgentUsageFileHasRaw(entry agentUsageEntry) bool { + return entry.InputTokens > 0 || entry.OutputTokens > 0 || entry.CacheReadTokens > 0 || entry.CacheWriteTokens > 0 || entry.ReasoningTokens > 0 +} - tokenUsageLog.Printf("Parsed agent usage file: input=%d, output=%d, cache_read=%d, cache_write=%d", - summary.TotalInputTokens, summary.TotalOutputTokens, summary.TotalCacheReadTokens, summary.TotalCacheWriteTokens) - return summary, nil +func parseAgentUsageFileAIC(summary *TokenUsageSummary, entry agentUsageEntry, model, provider string) { + summary.TotalAIC = entry.AICredits + if summary.ByModel[model] == nil { + summary.ByModel[model] = &ModelTokenUsage{} + } + summary.ByModel[model].Provider = provider + summary.ByModel[model].InputTokens = entry.InputTokens + summary.ByModel[model].OutputTokens = entry.OutputTokens + summary.ByModel[model].CacheReadTokens = entry.CacheReadTokens + summary.ByModel[model].CacheWriteTokens = entry.CacheWriteTokens + summary.ByModel[model].ReasoningTokens = entry.ReasoningTokens + summary.ByModel[model].AIC = entry.AICredits } // analyzeTokenUsage finds and parses the token-usage.jsonl file from a run directory. @@ -703,7 +690,6 @@ func processOneUsageJSONLFile(filePath string) (total float64, found bool, err e // It intentionally skips effective-token computation for callers that only need cost. func analyzeTokenUsageAICOnly(runDir string, verbose bool) (*TokenUsageSummary, error) { tokenUsageLog.Printf("Analyzing token usage (AIC only) in: %s", runDir) - usageJSONLFiles := findUsageJSONLFiles(runDir) if len(usageJSONLFiles) > 0 { console.LogVerbose(verbose, " Found usage JSONL files: "+strings.Join(usageJSONLFiles, ", ")) @@ -715,48 +701,64 @@ func analyzeTokenUsageAICOnly(runDir string, verbose bool) (*TokenUsageSummary, return &TokenUsageSummary{TotalAIC: totalAIC}, nil } } + if summary, found, err := analyzeTokenUsageAICOnlyTokenFile(runDir, verbose); err != nil || found { + return summary, err + } + return analyzeTokenUsageAICOnlyAgentUsage(runDir, verbose) +} +func analyzeTokenUsageAICOnlyTokenFile(runDir string, verbose bool) (*TokenUsageSummary, bool, error) { filePath := findTokenUsageFile(runDir) - if filePath != "" { - fileInfo, _ := os.Stat(filePath) - if fileInfo != nil { - console.LogVerbose(verbose, fmt.Sprintf(" Found token usage file: %s (%d bytes)", filepath.Base(filePath), fileInfo.Size())) - } + if filePath == "" { + return nil, false, nil + } + fileInfo, _ := os.Stat(filePath) + if fileInfo != nil { + console.LogVerbose(verbose, fmt.Sprintf(" Found token usage file: %s (%d bytes)", filepath.Base(filePath), fileInfo.Size())) + } + file, err := os.Open(filePath) + if err != nil { + return nil, true, fmt.Errorf("failed to open token usage file: %w", err) + } + defer file.Close() + totalAIC, found, err := analyzeTokenUsageAICOnlyScan(file) + if err != nil { + return nil, true, err + } + if found { + return &TokenUsageSummary{TotalAIC: totalAIC}, true, nil + } + return nil, false, nil +} - file, err := os.Open(filePath) - if err != nil { - return nil, fmt.Errorf("failed to open token usage file: %w", err) - } - defer file.Close() - - totalAIC := 0.0 - found := false - scanner := bufio.NewScanner(file) - scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - var entry TokenUsageEntry - if err := json.Unmarshal([]byte(line), &entry); err != nil { - continue - } - model := entry.Model - if model == "" { - model = "unknown" - } - totalAIC += computeModelInferenceAIC(entry.Provider, model, entry.InputTokens, entry.OutputTokens, entry.CacheReadTokens, entry.CacheWriteTokens, entry.ReasoningTokens) - found = true +func analyzeTokenUsageAICOnlyScan(file *os.File) (float64, bool, error) { + totalAIC := 0.0 + found := false + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading token usage file: %w", err) + var entry TokenUsageEntry + if err := json.Unmarshal([]byte(line), &entry); err != nil { + continue } - if found { - return &TokenUsageSummary{TotalAIC: totalAIC}, nil + model := entry.Model + if model == "" { + model = "unknown" } + totalAIC += computeModelInferenceAIC(entry.Provider, model, entry.InputTokens, entry.OutputTokens, entry.CacheReadTokens, entry.CacheWriteTokens, entry.ReasoningTokens) + found = true + } + if err := scanner.Err(); err != nil { + return 0, false, fmt.Errorf("error reading token usage file: %w", err) } + return totalAIC, found, nil +} +func analyzeTokenUsageAICOnlyAgentUsage(runDir string, verbose bool) (*TokenUsageSummary, error) { agentUsagePath := findAgentUsageFile(runDir) if agentUsagePath == "" { return nil, nil @@ -765,14 +767,11 @@ func analyzeTokenUsageAICOnly(runDir string, verbose bool) (*TokenUsageSummary, if agentFileInfo != nil { console.LogVerbose(verbose, fmt.Sprintf(" Found agent usage file: %s (%d bytes)", filepath.Base(agentUsagePath), agentFileInfo.Size())) } - summary, err := parseAgentUsageFile(agentUsagePath) if err != nil || summary == nil { return summary, err } - return &TokenUsageSummary{ - TotalAIC: summary.TotalAIC, - }, nil + return &TokenUsageSummary{TotalAIC: summary.TotalAIC}, nil } func countAPIProxySteeringEvents(runDir string) int { @@ -897,24 +896,26 @@ func augmentSubagentModelAttribution(runDir string, summary *TokenUsageSummary) if summary == nil { return } - requests := extractSubagentModelRequests(runDir) if len(requests) == 0 { return } addTokenUsageWarning(summary, subagentStdioWarning) + actuals, observedModels := augmentSubagentModelAttributionActuals(summary) + summary.SubagentModelActuals = actuals + requestRows, mismatchCount := augmentSubagentModelAttributionRequests(requests, observedModels) + summary.SubagentModelRequests = requestRows + summary.MismatchCount = mismatchCount +} +func augmentSubagentModelAttributionActuals(summary *TokenUsageSummary) ([]SubagentModelActual, map[string]string) { actuals := make([]SubagentModelActual, 0, len(summary.ByModel)) observedModels := make(map[string]string, len(summary.ByModel)) for model, usage := range summary.ByModel { if usage == nil || model == "" { continue } - actuals = append(actuals, SubagentModelActual{ - Model: model, - Provider: usage.Provider, - Requests: usage.Requests, - }) + actuals = append(actuals, SubagentModelActual{Model: model, Provider: usage.Provider, Requests: usage.Requests}) observedModels[model] = usage.Provider } slices.SortStableFunc(actuals, func(a, b SubagentModelActual) int { @@ -933,15 +934,16 @@ func augmentSubagentModelAttribution(runDir string, summary *TokenUsageSummary) return 0 } }) - summary.SubagentModelActuals = actuals + return actuals, observedModels +} - var fallbackEffectiveModel string +func augmentSubagentModelAttributionRequests(requests []SubagentModelRequest, observedModels map[string]string) ([]SubagentModelRequest, int) { + fallbackEffectiveModel := "" if len(observedModels) == 1 { for model := range observedModels { fallbackEffectiveModel = model } } - requestRows := make([]SubagentModelRequest, 0, len(requests)) mismatchCount := 0 for _, row := range requests { @@ -958,8 +960,7 @@ func augmentSubagentModelAttribution(runDir string, summary *TokenUsageSummary) } requestRows = append(requestRows, row) } - summary.SubagentModelRequests = requestRows - summary.MismatchCount = mismatchCount + return requestRows, mismatchCount } func addTokenUsageWarning(summary *TokenUsageSummary, warning string) { @@ -977,53 +978,61 @@ func extractSubagentModelRequests(runDir string) []SubagentModelRequest { if agentStdioPath == "" { return nil } - file, err := os.Open(agentStdioPath) if err != nil { return nil } defer file.Close() - - type key struct { - agent string - model string + counts, ok := extractSubagentModelRequestsCounts(file) + if !ok { + return nil } - counts := make(map[key]int) + return extractSubagentModelRequestsRows(counts) +} +type extractSubagentModelRequestsKey struct { + agent string + model string +} + +func extractSubagentModelRequestsCounts(file *os.File) (map[extractSubagentModelRequestsKey]int, bool) { + counts := make(map[extractSubagentModelRequestsKey]int) reader := bufio.NewReader(file) for { line, readErr := reader.ReadString('\n') line = strings.TrimSpace(line) if line != "" { - matches := subagentDispatchPattern.FindAllStringSubmatch(line, -1) - for _, m := range matches { - if len(m) < 3 { - continue - } - agentName := strings.TrimSpace(m[1]) - requestedModel := strings.TrimSpace(m[2]) - if agentName == "" || requestedModel == "" { - continue - } - counts[key{agent: agentName, model: requestedModel}]++ - } + extractSubagentModelRequestsLine(counts, line) } - if readErr == io.EOF { break } if readErr != nil { - return nil + return nil, false + } + } + return counts, true +} + +func extractSubagentModelRequestsLine(counts map[extractSubagentModelRequestsKey]int, line string) { + matches := subagentDispatchPattern.FindAllStringSubmatch(line, -1) + for _, m := range matches { + if len(m) < 3 { + continue + } + agentName := strings.TrimSpace(m[1]) + requestedModel := strings.TrimSpace(m[2]) + if agentName == "" || requestedModel == "" { + continue } + counts[extractSubagentModelRequestsKey{agent: agentName, model: requestedModel}]++ } +} +func extractSubagentModelRequestsRows(counts map[extractSubagentModelRequestsKey]int) []SubagentModelRequest { rows := make([]SubagentModelRequest, 0, len(counts)) for k, n := range counts { - rows = append(rows, SubagentModelRequest{ - AgentName: k.agent, - RequestedModel: k.model, - InvocationCount: n, - }) + rows = append(rows, SubagentModelRequest{AgentName: k.agent, RequestedModel: k.model, InvocationCount: n}) } slices.SortStableFunc(rows, func(a, b SubagentModelRequest) int { if a.AgentName != b.AgentName { diff --git a/pkg/cli/tokens_bootstrap.go b/pkg/cli/tokens_bootstrap.go index 40a04513390..7059445e6da 100644 --- a/pkg/cli/tokens_bootstrap.go +++ b/pkg/cli/tokens_bootstrap.go @@ -43,85 +43,65 @@ Only required secrets are prompted for. Optional secrets are not shown.`, func runTokensBootstrap(engine, repo string, nonInteractive bool) error { tokensBootstrapLog.Printf("Running tokens bootstrap: engine=%s, repo=%s, nonInteractive=%v", engine, repo, nonInteractive) - - // Configure the gh CLI host from the git remote before any gh calls so that - // secret discovery and upload both target the correct host. configureDefaultGHHostFromOriginRemoteIfUnset() - - var repoSlug string - var err error - - // Determine target repository - if repo != "" { - repoSlug = repo - } else { - repoSlug, err = GetCurrentRepoSlug() - if err != nil { - return fmt.Errorf("failed to detect current repository: %w", err) - } + repoSlug, err := runTokensBootstrapRepo(repo) + if err != nil { + return err } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Analyzing workflows in %s...", repoSlug))) - - // Discover workflows in the repository requirements, err := getSecretRequirements(engine) if err != nil { return fmt.Errorf("failed to analyze workflows: %w", err) } - tokensBootstrapLog.Printf("Collected %d required secrets from workflows", len(requirements)) - - // Check existing secrets in repository - existingSecrets, err := getExistingSecretsInRepo(repoSlug) - if err != nil { - // If we can't check existing secrets (e.g., no gh auth), continue with empty map - tokensBootstrapLog.Printf("Could not check existing secrets: %v", err) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Unable to check existing repository secrets. Will assume all secrets need to be configured.")) - existingSecrets = make(map[string]struct { - }) - } - - // Filter to only required secrets that are missing + existingSecrets := runTokensBootstrapExistingSecrets(repoSlug) missing := getMissingRequiredSecrets(requirements, existingSecrets) - - // Always display summary table of all required secrets with their status displaySecretsSummaryTable(requirements, existingSecrets) - if len(missing) == 0 { tokensBootstrapLog.Print("All required secrets present") fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("All required secrets are configured.")) return nil } - tokensBootstrapLog.Printf("Found %d missing required secrets", len(missing)) - - // In non-interactive mode, just display what's missing if nonInteractive { displayMissingSecrets(missing, repoSlug, existingSecrets) return nil } + return runTokensBootstrapPrompt(repoSlug, existingSecrets, missing) +} - // Interactive mode: prompt for missing secrets - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d missing required secret(s). You will be prompted to provide them.", len(missing)))) - fmt.Fprintln(os.Stderr, "") +func runTokensBootstrapRepo(repo string) (string, error) { + if repo != "" { + return repo, nil + } + repoSlug, err := GetCurrentRepoSlug() + if err != nil { + return "", fmt.Errorf("failed to detect current repository: %w", err) + } + return repoSlug, nil +} - config := EngineSecretConfig{ - RepoSlug: repoSlug, - ExistingSecrets: existingSecrets, - IncludeSystemSecrets: true, - IncludeOptional: false, +func runTokensBootstrapExistingSecrets(repoSlug string) map[string]struct{} { + existingSecrets, err := getExistingSecretsInRepo(repoSlug) + if err != nil { + tokensBootstrapLog.Printf("Could not check existing secrets: %v", err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Unable to check existing repository secrets. Will assume all secrets need to be configured.")) + return make(map[string]struct{}) } + return existingSecrets +} - // Prompt for each missing secret +func runTokensBootstrapPrompt(repoSlug string, existingSecrets map[string]struct{}, missing []SecretRequirement) error { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d missing required secret(s). You will be prompted to provide them.", len(missing)))) + fmt.Fprintln(os.Stderr, "") + config := EngineSecretConfig{RepoSlug: repoSlug, ExistingSecrets: existingSecrets, IncludeSystemSecrets: true, IncludeOptional: false} for _, req := range missing { if err := promptForSecret(req, config); err != nil { return fmt.Errorf("failed to collect secret %s: %w", req.Name, err) } } - fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("All required secrets have been configured.")) - return nil } diff --git a/pkg/cli/tool_graph.go b/pkg/cli/tool_graph.go index 07b95fe4d45..38640b5c264 100644 --- a/pkg/cli/tool_graph.go +++ b/pkg/cli/tool_graph.go @@ -66,107 +66,109 @@ func (g *ToolGraph) GenerateMermaidGraph() string { toolGraphLog.Print("No tool calls found for Mermaid graph generation") return console.FormatInfoMessage("No tool calls found") } - toolGraphLog.Printf("Generating Mermaid graph: tools=%d, transitions=%d", len(g.Tools), len(g.Transitions)) var sb strings.Builder sb.WriteString("```mermaid\n") sb.WriteString("stateDiagram-v2\n") + toolToStateMap := g.generateMermaidGraphStates(&sb) + g.generateMermaidGraphStart(&sb, toolToStateMap) + g.generateMermaidGraphTransitions(&sb, toolToStateMap) + sb.WriteString("```\n") + return sb.String() +} - // Add tool states with normalized names for Mermaid +func (g *ToolGraph) generateMermaidGraphStates(sb *strings.Builder) map[string]string { toolToStateMap := make(map[string]string) tools := sliceutil.SortedKeys(g.Tools) - for i, tool := range tools { stateId := fmt.Sprintf("tool%d", i) toolToStateMap[tool] = stateId - // Escape special characters in tool names for display displayName := strings.ReplaceAll(tool, "\"", "\\\"") - fmt.Fprintf(&sb, " %s : %s\n", stateId, displayName) + fmt.Fprintf(sb, " %s : %s\n", stateId, displayName) } + return toolToStateMap +} - // Add start state +func (g *ToolGraph) generateMermaidGraphStart(sb *strings.Builder, toolToStateMap map[string]string) { sb.WriteString(" [*] --> start_tool : begin\n") - - // Find the most common starting tool(s) startCounts := make(map[string]int) for _, sequence := range g.sequences { if len(sequence) > 0 { startCounts[sequence[0]]++ } } + if len(startCounts) == 0 { + return + } + for _, tool := range generateMermaidGraphMostCommon(startCounts) { + if stateId, exists := toolToStateMap[tool]; exists { + fmt.Fprintf(sb, " start_tool --> %s\n", stateId) + } + } +} - // Connect start to the most common starting tools - if len(startCounts) > 0 { - var startTools []string - maxCount := 0 - for tool, count := range startCounts { - if count > maxCount { - maxCount = count - startTools = []string{tool} - } else if count == maxCount { - startTools = append(startTools, tool) - } +func generateMermaidGraphMostCommon(counts map[string]int) []string { + var result []string + maxCount := 0 + for tool, count := range counts { + if count > maxCount { + maxCount = count + result = []string{tool} + } else if count == maxCount { + result = append(result, tool) } + } + return result +} - for _, tool := range startTools { - if stateId, exists := toolToStateMap[tool]; exists { - fmt.Fprintf(&sb, " start_tool --> %s\n", stateId) +func (g *ToolGraph) generateMermaidGraphTransitions(sb *strings.Builder, toolToStateMap map[string]string) { + transitions := g.generateMermaidGraphTransitionList() + for _, transition := range transitions { + fromState, fromExists := toolToStateMap[transition.From] + toState, toExists := toolToStateMap[transition.To] + if fromExists && toExists { + label := "" + if transition.Count > 1 { + label = fmt.Sprintf(" : %dx", transition.Count) } + fmt.Fprintf(sb, " %s --> %s%s\n", fromState, toState, label) } } +} - // Add transitions with counts as labels +func (g *ToolGraph) generateMermaidGraphTransitionList() []ToolTransition { var transitions []ToolTransition for key, count := range g.Transitions { parts := strings.Split(key, "->") if len(parts) == 2 { - transitions = append(transitions, ToolTransition{ - From: parts[0], - To: parts[1], - Count: count, - }) + transitions = append(transitions, ToolTransition{From: parts[0], To: parts[1], Count: count}) } } + slices.SortFunc(transitions, generateMermaidGraphCompareTransition) + return transitions +} - // Sort transitions by count (descending) for better visualization - slices.SortFunc(transitions, func(a, b ToolTransition) int { - if a.Count != b.Count { - if a.Count > b.Count { - return -1 - } - return 1 - } - if a.From != b.From { - if a.From < b.From { - return -1 - } - return 1 - } - switch { - case a.To < b.To: +func generateMermaidGraphCompareTransition(a, b ToolTransition) int { + if a.Count != b.Count { + if a.Count > b.Count { return -1 - case a.To > b.To: - return 1 - default: - return 0 } - }) - - for _, transition := range transitions { - fromState, fromExists := toolToStateMap[transition.From] - toState, toExists := toolToStateMap[transition.To] - - if fromExists && toExists { - label := "" - if transition.Count > 1 { - label = fmt.Sprintf(" : %dx", transition.Count) - } - fmt.Fprintf(&sb, " %s --> %s%s\n", fromState, toState, label) + return 1 + } + if a.From != b.From { + if a.From < b.From { + return -1 } + return 1 + } + switch { + case a.To < b.To: + return -1 + case a.To > b.To: + return 1 + default: + return 0 } - - sb.WriteString("```\n") - return sb.String() } // generateToolGraph analyzes processed runs and generates a tool sequence graph diff --git a/pkg/cli/trial_command.go b/pkg/cli/trial_command.go index 0653cec514a..651f2f478ae 100644 --- a/pkg/cli/trial_command.go +++ b/pkg/cli/trial_command.go @@ -10,9 +10,21 @@ import ( // NewTrialCommand creates the trial command func NewTrialCommand(validateEngine func(string) error) *cobra.Command { cmd := &cobra.Command{ - Use: "trial ...", - Short: "Run one or more agentic workflows in trial mode against a simulated repository", - Long: `Run one or more agentic workflows in trial mode against a simulated repository. + Use: "trial ...", + Short: "Run one or more agentic workflows in trial mode against a simulated repository", + Long: newTrialCommandLong(), + Example: newTrialCommandExample(), + Args: newTrialCommandArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return newTrialCommandRun(cmd, args, validateEngine) + }, + } + newTrialCommandFlags(cmd) + return cmd +} + +func newTrialCommandLong() string { + return `Run one or more agentic workflows in trial mode against a simulated repository. This command creates a temporary private repository in your GitHub account, installs the specified workflows from their source repositories, and runs them in "trial mode" to capture safe outputs without @@ -26,8 +38,11 @@ Repository modes: All workflows must support workflow_dispatch trigger to be used in trial mode. The host repository will be created as private and kept by default unless --delete-host-repo-after is specified. -Trial results are saved both locally (in the trials/ directory) and in the host repository for future reference.`, - Example: ` ` + string(constants.CLIExtensionPrefix) + ` trial githubnext/agentics/weekly-research # Run a single workflow in a temporary trial repository +Trial results are saved both locally (in the trials/ directory) and in the host repository for future reference.` +} + +func newTrialCommandExample() string { + return ` ` + string(constants.CLIExtensionPrefix) + ` trial githubnext/agentics/weekly-research # Run a single workflow in a temporary trial repository ` + string(constants.CLIExtensionPrefix) + ` trial githubnext/agentics/daily-plan githubnext/agentics/weekly-research # Compare multiple workflows ` + string(constants.CLIExtensionPrefix) + ` trial githubnext/agentics/daily-plan myorg/myrepo/custom-workflow # Run workflows from different repositories ` + string(constants.CLIExtensionPrefix) + ` trial githubnext/agentics/my-workflow --host-repo myorg/myrepo # Use an existing host repository @@ -41,75 +56,56 @@ Trial results are saved both locally (in the trials/ directory) and in the host ` + string(constants.CLIExtensionPrefix) + ` trial githubnext/agentics/my-workflow --host-repo . # Use the current repository as the host ` + string(constants.CLIExtensionPrefix) + ` trial ./local-workflow.md --clone-repo upstream/repo --repeat 2 # Run a local workflow against cloned contents ` + string(constants.CLIExtensionPrefix) + ` trial githubnext/agentics/my-workflow --trigger-context https://github.com/owner/repo/issues/123 # Provide issue context for issue-triggered workflows -`, - Args: func(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return fmt.Errorf("missing workflow specification\n\nUsage:\n %s ...\n\nExample:\n %[1]s githubnext/agentics/daily-plan Trial a workflow from a repository\n %[1]s ./local-workflow.md Trial a local workflow\n\nRun '%[1]s --help' for more information", cmd.CommandPath()) - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - workflowSpecs := args - trialLog.Printf("Trial command invoked: workflow_count=%d", len(workflowSpecs)) - logicalRepoSpec, _ := cmd.Flags().GetString("logical-repo") - cloneRepoSpec, _ := cmd.Flags().GetString("clone-repo") - hostRepoSpec, _ := cmd.Flags().GetString("host-repo") - deleteHostRepo, _ := cmd.Flags().GetBool("delete-host-repo-after") - forceDeleteHostRepo, _ := cmd.Flags().GetBool("force-delete-host-repo-before") - yes, _ := cmd.Flags().GetBool("yes") - dryRun, _ := cmd.Flags().GetBool("dry-run") - jsonOutput, _ := cmd.Flags().GetBool("json") - timeout, _ := cmd.Flags().GetInt("timeout") - triggerContext, _ := cmd.Flags().GetString("trigger-context") - repeatCount, _ := cmd.Flags().GetInt("repeat") - autoMergePRs, _ := cmd.Flags().GetBool("auto-merge-prs") - engineOverride, _ := cmd.Flags().GetString("engine") - appendText, _ := cmd.Flags().GetString("append") - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - disableSecurityScanner, _ := cmd.Flags().GetBool("no-security-scanner") - disableSecurityScannerLegacy, _ := cmd.Flags().GetBool("disable-security-scanner") - disableSecurityScanner = disableSecurityScanner || disableSecurityScannerLegacy +` +} - if err := validateEngine(engineOverride); err != nil { - trialLog.Printf("Engine validation failed: engine=%s, err=%v", engineOverride, err) - return err - } - if trialLog.Enabled() { - trialLog.Printf("Trial options: dry_run=%v, repeat=%d, timeout_min=%d, auto_merge_prs=%v, logical_repo=%q, clone_repo=%q, host_repo=%q", - dryRun, repeatCount, timeout, autoMergePRs, logicalRepoSpec, cloneRepoSpec, hostRepoSpec) - } - opts := TrialOptions{ - Repos: TrialRepoContext{ - LogicalRepo: logicalRepoSpec, - CloneRepo: cloneRepoSpec, - HostRepo: hostRepoSpec, - }, - DeleteHostRepo: deleteHostRepo, - ForceDelete: forceDeleteHostRepo, - Quiet: yes, - DryRun: dryRun, - JSONOutput: jsonOutput, - TimeoutMinutes: timeout, - TriggerContext: triggerContext, - RepeatCount: repeatCount, - AutoMergePRs: autoMergePRs, - EngineOverride: engineOverride, - AppendText: appendText, - Verbose: verbose, - DisableSecurityScanner: disableSecurityScanner, - } +func newTrialCommandArgs(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return fmt.Errorf("missing workflow specification\n\nUsage:\n %s ...\n\nExample:\n %[1]s githubnext/agentics/daily-plan Trial a workflow from a repository\n %[1]s ./local-workflow.md Trial a local workflow\n\nRun '%[1]s --help' for more information", cmd.CommandPath()) + } + return nil +} - if err := RunWorkflowTrials(cmd.Context(), workflowSpecs, opts); err != nil { - return err - } - return nil - }, +func newTrialCommandRun(cmd *cobra.Command, workflowSpecs []string, validateEngine func(string) error) error { + trialLog.Printf("Trial command invoked: workflow_count=%d", len(workflowSpecs)) + opts, err := newTrialCommandOptions(cmd) + if err != nil { + return err } + if err := validateEngine(opts.EngineOverride); err != nil { + trialLog.Printf("Engine validation failed: engine=%s, err=%v", opts.EngineOverride, err) + return err + } + if trialLog.Enabled() { + trialLog.Printf("Trial options: dry_run=%v, repeat=%d, timeout_min=%d, auto_merge_prs=%v, logical_repo=%q, clone_repo=%q, host_repo=%q", opts.DryRun, opts.RepeatCount, opts.TimeoutMinutes, opts.AutoMergePRs, opts.Repos.LogicalRepo, opts.Repos.CloneRepo, opts.Repos.HostRepo) + } + return RunWorkflowTrials(cmd.Context(), workflowSpecs, opts) +} - // Add flags +func newTrialCommandOptions(cmd *cobra.Command) (TrialOptions, error) { + logicalRepoSpec, _ := cmd.Flags().GetString("logical-repo") + cloneRepoSpec, _ := cmd.Flags().GetString("clone-repo") + hostRepoSpec, _ := cmd.Flags().GetString("host-repo") + deleteHostRepo, _ := cmd.Flags().GetBool("delete-host-repo-after") + forceDeleteHostRepo, _ := cmd.Flags().GetBool("force-delete-host-repo-before") + yes, _ := cmd.Flags().GetBool("yes") + dryRun, _ := cmd.Flags().GetBool("dry-run") + jsonOutput, _ := cmd.Flags().GetBool("json") + timeout, _ := cmd.Flags().GetInt("timeout") + triggerContext, _ := cmd.Flags().GetString("trigger-context") + repeatCount, _ := cmd.Flags().GetInt("repeat") + autoMergePRs, _ := cmd.Flags().GetBool("auto-merge-prs") + engineOverride, _ := cmd.Flags().GetString("engine") + appendText, _ := cmd.Flags().GetString("append") + verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") + disableSecurityScanner, _ := cmd.Flags().GetBool("no-security-scanner") + disableSecurityScannerLegacy, _ := cmd.Flags().GetBool("disable-security-scanner") + return TrialOptions{Repos: TrialRepoContext{LogicalRepo: logicalRepoSpec, CloneRepo: cloneRepoSpec, HostRepo: hostRepoSpec}, DeleteHostRepo: deleteHostRepo, ForceDelete: forceDeleteHostRepo, Quiet: yes, DryRun: dryRun, JSONOutput: jsonOutput, TimeoutMinutes: timeout, TriggerContext: triggerContext, RepeatCount: repeatCount, AutoMergePRs: autoMergePRs, EngineOverride: engineOverride, AppendText: appendText, Verbose: verbose, DisableSecurityScanner: disableSecurityScanner || disableSecurityScannerLegacy}, nil +} + +func newTrialCommandFlags(cmd *cobra.Command) { cmd.Flags().StringP("logical-repo", "l", "", "Repository to simulate workflow execution against, as if the workflow was installed there (defaults to current repository)") cmd.Flags().String("clone-repo", "", "Clone the contents of the specified repository into the host repository before execution (useful for testing against actual repository state)") - cmd.Flags().String("host-repo", "", "Custom host repository slug (defaults to '/gh-aw-trial'). Use '.' for current repository") cmd.Flags().Bool("delete-host-repo-after", false, "Delete the host repository after completion (kept by default)") cmd.Flags().Bool("force-delete-host-repo-before", false, "Force delete the host repository before creation if it already exists") @@ -126,6 +122,4 @@ Trial results are saved both locally (in the trials/ directory) and in the host cmd.Flags().Bool("disable-security-scanner", false, "Skip security scanning of workflow markdown content") _ = cmd.Flags().MarkDeprecated("disable-security-scanner", "use --no-security-scanner instead") cmd.MarkFlagsMutuallyExclusive("logical-repo", "clone-repo") - - return cmd } diff --git a/pkg/cli/trial_confirmation.go b/pkg/cli/trial_confirmation.go index 6c7dc661346..77c0ebd9fad 100644 --- a/pkg/cli/trial_confirmation.go +++ b/pkg/cli/trial_confirmation.go @@ -37,18 +37,30 @@ type trialConfirmationOptions struct { // showTrialConfirmation displays a confirmation prompt to the user using parsed workflow specs func showTrialConfirmation(opts trialConfirmationOptions) error { trialConfirmationLog.Printf("Showing trial confirmation: workflows=%d, hostRepo=%s, cloneRepo=%s, repeat=%d, directMode=%v", len(opts.parsedSpecs), opts.hostRepoSlug, opts.cloneRepoSlug, opts.repeatCount, opts.directTrialMode) + showTrialConfirmationPlan(opts) + showTrialConfirmationExecutionSteps(opts) + return showTrialConfirmationPrompt() +} + +func showTrialConfirmationPlan(opts trialConfirmationOptions) { githubHost := getGitHubHost() hostRepoSlugURL := fmt.Sprintf("%s/%s", githubHost, opts.hostRepoSlug) - var sections []string - - // Title box with double border - titleText := "Trial Execution Plan" - sections = append(sections, console.RenderTitleBox(titleText, 80)...) - + sections = append(sections, console.RenderTitleBox("Trial Execution Plan", 80)...) + sections = append(sections, "") + sections = append(sections, console.RenderInfoSection(showTrialConfirmationWorkflowInfo(opts))...) sections = append(sections, "") + sections = append(sections, console.RenderInfoSection(showTrialConfirmationModeInfo(opts))...) + sections = append(sections, "") + sections = append(sections, console.RenderInfoSection(fmt.Sprintf("Host Repo: %s\n %s", opts.hostRepoSlug, hostRepoSlugURL))...) + sections = append(sections, "") + sections = append(sections, console.RenderInfoSection(showTrialConfirmationConfigInfo(opts))...) + sections = append(sections, "") + console.RenderComposedSections(sections) + console.RenderComposedSections(console.RenderTitleBox("Execution Steps", 80)) +} - // Workflow information section +func showTrialConfirmationWorkflowInfo(opts trialConfirmationOptions) string { var workflowInfo strings.Builder if len(opts.parsedSpecs) == 1 { fmt.Fprintf(&workflowInfo, "Workflow: %s (from %s)", opts.parsedSpecs[0].WorkflowName, opts.parsedSpecs[0].RepoSlug) @@ -58,85 +70,77 @@ func showTrialConfirmation(opts trialConfirmationOptions) error { fmt.Fprintf(&workflowInfo, "\n • %s (from %s)", spec.WorkflowName, spec.RepoSlug) } } + return workflowInfo.String() +} - sections = append(sections, console.RenderInfoSection(workflowInfo.String())...) - - sections = append(sections, "") - - // Display target repository info based on mode +func showTrialConfirmationModeInfo(opts trialConfirmationOptions) string { var modeInfo strings.Builder if opts.cloneRepoSlug != "" { - // Clone-repo mode fmt.Fprintf(&modeInfo, "Source: %s (will be cloned)\n", opts.cloneRepoSlug) modeInfo.WriteString("Mode: Clone repository contents into host repository") } else if opts.directTrialMode { - // Direct trial mode fmt.Fprintf(&modeInfo, "Target: %s (direct)\n", opts.hostRepoSlug) modeInfo.WriteString("Mode: Run workflows directly in repository (no simulation)") } else { - // Logical-repo mode fmt.Fprintf(&modeInfo, "Target: %s (simulated)\n", opts.logicalRepoSlug) modeInfo.WriteString("Mode: Simulate execution against target repository") } + return modeInfo.String() +} - sections = append(sections, console.RenderInfoSection(modeInfo.String())...) - - sections = append(sections, "") - - // Host repository info - var hostInfo strings.Builder - fmt.Fprintf(&hostInfo, "Host Repo: %s\n", opts.hostRepoSlug) - fmt.Fprintf(&hostInfo, " %s", hostRepoSlugURL) - - sections = append(sections, console.RenderInfoSection(hostInfo.String())...) - - sections = append(sections, "") - - // Configuration settings +func showTrialConfirmationConfigInfo(opts trialConfirmationOptions) string { var configInfo strings.Builder if opts.deleteHostRepo { configInfo.WriteString("Cleanup: Host repository will be deleted after completion") } else { configInfo.WriteString("Cleanup: Host repository will be preserved") } - - // Display secret usage information (only when engine override is specified) if opts.engineOverride != "" { - configInfo.WriteString("\n") - fmt.Fprintf(&configInfo, "Secrets: Will prompt for %s API key if needed (stored as repository secret)", opts.engineOverride) + fmt.Fprintf(&configInfo, "\nSecrets: Will prompt for %s API key if needed (stored as repository secret)", opts.engineOverride) } - - // Display repeat count if set if opts.repeatCount > 0 { fmt.Fprintf(&configInfo, "\nRepeat: Will run %d times (total executions: %d)", opts.repeatCount, opts.repeatCount+1) } - - // Display auto-merge setting if enabled if opts.autoMergePRs { configInfo.WriteString("\nAuto-merge: Pull requests will be automatically merged") } + return configInfo.String() +} - sections = append(sections, console.RenderInfoSection(configInfo.String())...) - - sections = append(sections, "") - - // Compose and output all sections - console.RenderComposedSections(sections) - - // Add "Execution Steps" section separator - executionStepsSections := console.RenderTitleBox("Execution Steps", 80) - console.RenderComposedSections(executionStepsSections) +func showTrialConfirmationExecutionSteps(opts trialConfirmationOptions) { + hostRepoExists := showTrialConfirmationHostRepoExists(opts) + stepNum := 1 + stepNum = showTrialConfirmationRepoStep(opts, hostRepoExists, stepNum) + if opts.cloneRepoSlug != "" { + stepNum = showTrialConfirmationCloneSteps(opts, hostRepoExists, stepNum) + } + stepNum = showTrialConfirmationInstallStep(opts, stepNum) + if opts.engineOverride != "" { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Ensure %s API key secret is configured\n"), stepNum, opts.engineOverride) + stepNum++ + } + stepNum = showTrialConfirmationExecuteStep(opts, stepNum) + if opts.deleteHostRepo { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Delete the host repository\n"), stepNum) + } else { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Preserve the host repository for inspection\n"), stepNum) + } + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessageStderr("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")) + fmt.Fprintln(os.Stderr, "") +} - // Check if host repository already exists to update messaging +func showTrialConfirmationHostRepoExists(opts trialConfirmationOptions) bool { hostRepoExists := false checkCmd := workflow.ExecGH("repo", "view", opts.hostRepoSlug) if err := checkCmd.Run(); err == nil { hostRepoExists = true } trialConfirmationLog.Printf("Host repo check: exists=%v, forceDelete=%v", hostRepoExists, opts.forceDeleteHostRepo) + return hostRepoExists +} - // Step 1: Repository creation/reuse - stepNum := 1 +func showTrialConfirmationRepoStep(opts trialConfirmationOptions, hostRepoExists bool, stepNum int) int { if hostRepoExists && opts.forceDeleteHostRepo { fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Delete and recreate host repository\n"), stepNum) } else if hostRepoExists { @@ -144,104 +148,88 @@ func showTrialConfirmation(opts trialConfirmationOptions) error { } else { fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Create a private host repository\n"), stepNum) } - stepNum++ - - // Step 2: Clone contents (only in clone-repo mode) - if opts.cloneRepoSlug != "" { - if hostRepoExists && !opts.forceDeleteHostRepo { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Force push contents from %s (overwriting existing content)\n"), stepNum, opts.cloneRepoSlug) - } else { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Clone contents from %s\n"), stepNum, opts.cloneRepoSlug) - } - stepNum++ + return stepNum + 1 +} - // Show that workflows will be disabled - if len(opts.parsedSpecs) == 1 { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Disable all workflows in cloned repository except %s\n"), stepNum, opts.parsedSpecs[0].WorkflowName) - } else { - workflowNames := sliceutil.Map(opts.parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Disable all workflows in cloned repository except: %s\n"), stepNum, strings.Join(workflowNames, ", ")) - } - stepNum++ +func showTrialConfirmationCloneSteps(opts trialConfirmationOptions, hostRepoExists bool, stepNum int) int { + if hostRepoExists && !opts.forceDeleteHostRepo { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Force push contents from %s (overwriting existing content)\n"), stepNum, opts.cloneRepoSlug) + } else { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Clone contents from %s\n"), stepNum, opts.cloneRepoSlug) } + stepNum++ + if len(opts.parsedSpecs) == 1 { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Disable all workflows in cloned repository except %s\n"), stepNum, opts.parsedSpecs[0].WorkflowName) + } else { + workflowNames := sliceutil.Map(opts.parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Disable all workflows in cloned repository except: %s\n"), stepNum, strings.Join(workflowNames, ", ")) + } + return stepNum + 1 +} - // Step 3/2: Install and compile workflows +func showTrialConfirmationInstallStep(opts trialConfirmationOptions, stepNum int) int { if len(opts.parsedSpecs) == 1 { fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Install and compile %s\n"), stepNum, opts.parsedSpecs[0].WorkflowName) } else { workflowNames := sliceutil.Map(opts.parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Install and compile: %s\n"), stepNum, strings.Join(workflowNames, ", ")) } - stepNum++ + return stepNum + 1 +} - // Step: Configure secrets (only when engine override is specified) - if opts.engineOverride != "" { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Ensure %s API key secret is configured\n"), stepNum, opts.engineOverride) - stepNum++ +func showTrialConfirmationExecuteStep(opts trialConfirmationOptions, stepNum int) int { + if len(opts.parsedSpecs) == 1 { + return showTrialConfirmationExecuteSingle(opts, stepNum) } + return showTrialConfirmationExecuteMultiple(opts, stepNum) +} - // Step 5/4: Execute workflows and auto-merge (repeated if --repeat is used) - if len(opts.parsedSpecs) == 1 { - workflowName := opts.parsedSpecs[0].WorkflowName - if opts.repeatCount > 0 && opts.autoMergePRs { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. For each of %d executions:\n"), stepNum, opts.repeatCount+1) - fmt.Fprintln(os.Stderr, formatTrialNestedListItem("Execute "+workflowName)) - fmt.Fprintln(os.Stderr, formatTrialNestedListItem("Auto-merge any pull requests created during execution")) - } else if opts.repeatCount > 0 { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute %s %d times\n"), stepNum, workflowName, opts.repeatCount+1) - } else if opts.autoMergePRs { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute %s\n"), stepNum, workflowName) - stepNum++ - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Auto-merge any pull requests created during execution\n"), stepNum) - } else { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute %s\n"), stepNum, workflowName) - } +func showTrialConfirmationExecuteSingle(opts trialConfirmationOptions, stepNum int) int { + workflowName := opts.parsedSpecs[0].WorkflowName + if opts.repeatCount > 0 && opts.autoMergePRs { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. For each of %d executions:\n"), stepNum, opts.repeatCount+1) + fmt.Fprintln(os.Stderr, formatTrialNestedListItem("Execute "+workflowName)) + fmt.Fprintln(os.Stderr, formatTrialNestedListItem("Auto-merge any pull requests created during execution")) + } else if opts.repeatCount > 0 { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute %s %d times\n"), stepNum, workflowName, opts.repeatCount+1) + } else if opts.autoMergePRs { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute %s\n"), stepNum, workflowName) + stepNum++ + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Auto-merge any pull requests created during execution\n"), stepNum) } else { - workflowNames := sliceutil.Map(opts.parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) - workflowList := strings.Join(workflowNames, ", ") - - if opts.repeatCount > 0 && opts.autoMergePRs { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. For each of %d executions:\n"), stepNum, opts.repeatCount+1) - fmt.Fprintln(os.Stderr, formatTrialNestedListItem("Execute: "+workflowList)) - fmt.Fprintln(os.Stderr, formatTrialNestedListItem("Auto-merge any pull requests created during execution")) - } else if opts.repeatCount > 0 { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute %d times: %s\n"), stepNum, opts.repeatCount+1, workflowList) - } else if opts.autoMergePRs { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute: %s\n"), stepNum, workflowList) - stepNum++ - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Auto-merge any pull requests created during execution\n"), stepNum) - } else { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute: %s\n"), stepNum, workflowList) - } + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute %s\n"), stepNum, workflowName) } - stepNum++ + return stepNum + 1 +} - // Final step: Delete/preserve repository - if opts.deleteHostRepo { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Delete the host repository\n"), stepNum) +func showTrialConfirmationExecuteMultiple(opts trialConfirmationOptions, stepNum int) int { + workflowNames := sliceutil.Map(opts.parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) + workflowList := strings.Join(workflowNames, ", ") + if opts.repeatCount > 0 && opts.autoMergePRs { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. For each of %d executions:\n"), stepNum, opts.repeatCount+1) + fmt.Fprintln(os.Stderr, formatTrialNestedListItem("Execute: "+workflowList)) + fmt.Fprintln(os.Stderr, formatTrialNestedListItem("Auto-merge any pull requests created during execution")) + } else if opts.repeatCount > 0 { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute %d times: %s\n"), stepNum, opts.repeatCount+1, workflowList) + } else if opts.autoMergePRs { + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute: %s\n"), stepNum, workflowList) + stepNum++ + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Auto-merge any pull requests created during execution\n"), stepNum) } else { - fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Preserve the host repository for inspection\n"), stepNum) + fmt.Fprintf(os.Stderr, console.FormatInfoMessageStderr(" %d. Execute: %s\n"), stepNum, workflowList) } + return stepNum + 1 +} - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessageStderr("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")) - fmt.Fprintln(os.Stderr, "") - - // Ask for confirmation using console helper - confirmed, err := console.ConfirmAction( - "Do you want to continue?", - "Yes, proceed", - "No, cancel", - ) +func showTrialConfirmationPrompt() error { + confirmed, err := console.ConfirmAction("Do you want to continue?", "Yes, proceed", "No, cancel") if err != nil { return fmt.Errorf("confirmation failed: %w", err) } - if !confirmed { trialConfirmationLog.Print("Trial cancelled by user") return errors.New("trial cancelled by user") } - trialConfirmationLog.Print("Trial confirmed by user, proceeding") return nil } diff --git a/pkg/cli/trial_helpers.go b/pkg/cli/trial_helpers.go index 3e3cfa3730b..bdf54744f1d 100644 --- a/pkg/cli/trial_helpers.go +++ b/pkg/cli/trial_helpers.go @@ -41,18 +41,13 @@ func executeTrialRun(ctx context.Context, parsedSpecs []*WorkflowSpec, hostRepoS // Determine target repo slug for filenames once // In direct trial mode, use hostRepoSlug; otherwise use logicalRepoSlug - targetRepoForFilename := logicalRepoSlug - if directTrialMode { - targetRepoForFilename = hostRepoSlug - } + targetRepoForFilename := executeTrialRunTargetRepoForFilename(hostRepoSlug, logicalRepoSlug, directTrialMode) // Step 3: Clone host repository to local temp directory - trialLog.Printf("Cloning trial host repository: %s", hostRepoSlug) - tempDir, err := cloneTrialHostRepository(hostRepoSlug, opts.Verbose) + tempDir, err := executeTrialRunCloneHostRepo(hostRepoSlug, opts.Verbose) if err != nil { - return fmt.Errorf("failed to clone host repository: %w", err) + return err } - trialLog.Printf("Cloned repository to: %s", tempDir) defer func() { if err := os.RemoveAll(tempDir); err != nil { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to cleanup local temp directory: %v", err))) @@ -68,125 +63,200 @@ func executeTrialRun(ctx context.Context, parsedSpecs []*WorkflowSpec, hostRepoS var workflowResults []WorkflowTrialResult for _, parsedSpec := range parsedSpecs { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== Running trial for workflow: %s ===", parsedSpec.WorkflowName))) - - // Install workflow with trial mode compilation - if err := installWorkflowInTrialMode(ctx, tempDir, parsedSpec, logicalRepoSlug, cloneRepoSlug, hostRepoSlug, directTrialMode, &opts); err != nil { - return fmt.Errorf("failed to install workflow '%s' in trial mode: %w", parsedSpec.WorkflowName, err) + result, err := executeTrialRunWorkflow(executeTrialRunWorkflowParams{ + Ctx: ctx, + TempDir: tempDir, + ParsedSpec: parsedSpec, + HostRepoSlug: hostRepoSlug, + LogicalRepoSlug: logicalRepoSlug, + CloneRepoSlug: cloneRepoSlug, + DirectTrialMode: directTrialMode, + TargetRepoForFilename: targetRepoForFilename, + DateTimeID: dateTimeID, + Opts: opts, + }) + if err != nil { + return err } + workflowResults = append(workflowResults, result) + } - // Display workflow description if present - workflowPath := filepath.Join(tempDir, constants.GetWorkflowDir(), parsedSpec.WorkflowName+".md") - if description := ExtractWorkflowDescriptionFromFile(workflowPath); description != "" { - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(description)) - fmt.Fprintln(os.Stderr, "") - } + // Step 6: Save combined results for multi-workflow trials + executeTrialRunSaveCombinedResults(parsedSpecs, workflowResults, targetRepoForFilename, dateTimeID, opts.Verbose) - // Run the workflow and wait for completion (with trigger context if provided) - runID, err := triggerWorkflowRun(hostRepoSlug, parsedSpec.WorkflowName, opts.TriggerContext, opts.Verbose) - if err != nil { - return fmt.Errorf("failed to trigger workflow run for '%s': %w", parsedSpec.WorkflowName, err) - } + // Step 6.5: Copy trial results to host repository and commit them + workflowNames := sliceutil.Map(parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) + if err := copyTrialResultsToHostRepo(tempDir, dateTimeID, workflowNames, targetRepoForFilename, opts.Verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to copy trial results to repository: %v", err))) + } - // Generate workflow run URL - githubHost := getGitHubHost() - workflowRunURL := fmt.Sprintf("%s/%s/actions/runs/%s", githubHost, hostRepoSlug, runID) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow run started with ID: %s (%s)", runID, workflowRunURL))) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("All trials completed successfully")) + return nil +} - // Wait for workflow completion - if err := WaitForWorkflowCompletion(ctx, hostRepoSlug, runID, opts.TimeoutMinutes, opts.Verbose); err != nil { - // If the context was canceled or its deadline was exceeded, return that directly - if ctxErr := ctx.Err(); ctxErr != nil { - return ctxErr - } - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return err - } - return fmt.Errorf("workflow '%s' execution failed or timed out: %w", parsedSpec.WorkflowName, err) - } +func executeTrialRunTargetRepoForFilename(hostRepoSlug, logicalRepoSlug string, directTrialMode bool) string { + if directTrialMode { + return hostRepoSlug + } + return logicalRepoSlug +} - // Auto-merge PRs if requested - if opts.AutoMergePRs { - if err := AutoMergePullRequestsLegacy(hostRepoSlug, opts.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to auto-merge pull requests: %v", err))) - } - } +func executeTrialRunCloneHostRepo(hostRepoSlug string, verbose bool) (string, error) { + trialLog.Printf("Cloning trial host repository: %s", hostRepoSlug) + tempDir, err := cloneTrialHostRepository(hostRepoSlug, verbose) + if err != nil { + return "", fmt.Errorf("failed to clone host repository: %w", err) + } + trialLog.Printf("Cloned repository to: %s", tempDir) + return tempDir, nil +} - // Download and process all artifacts - artifacts, err := downloadAllArtifacts(hostRepoSlug, runID, opts.Verbose) - if err != nil { - return fmt.Errorf("failed to download artifacts for '%s': %w", parsedSpec.WorkflowName, err) - } +type executeTrialRunWorkflowParams struct { + Ctx context.Context + TempDir string + ParsedSpec *WorkflowSpec + HostRepoSlug string + LogicalRepoSlug string + CloneRepoSlug string + DirectTrialMode bool + TargetRepoForFilename string + DateTimeID string + Opts TrialOptions +} - // Save individual workflow results - result := WorkflowTrialResult{ - WorkflowName: parsedSpec.WorkflowName, - RunID: runID, - SafeOutputs: artifacts.SafeOutputs, - //AgentStdioLogs: artifacts.AgentStdioLogs, - AgenticRunInfo: artifacts.AgenticRunInfo, - AdditionalArtifacts: artifacts.AdditionalArtifacts, - Timestamp: time.Now(), - } - workflowResults = append(workflowResults, result) +func executeTrialRunWorkflow(p executeTrialRunWorkflowParams) (WorkflowTrialResult, error) { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== Running trial for workflow: %s ===", p.ParsedSpec.WorkflowName))) - // Save individual trial file - sanitizedTargetRepo := stringutil.SanitizeForFilename(targetRepoForFilename) - individualFilename := fmt.Sprintf("trials/%s-%s.%s.json", parsedSpec.WorkflowName, sanitizedTargetRepo, dateTimeID) - if err := saveTrialResult(individualFilename, result, opts.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to save individual trial result: %v", err))) - } + // Install workflow with trial mode compilation + if err := installWorkflowInTrialMode(p.Ctx, p.TempDir, p.ParsedSpec, p.LogicalRepoSlug, p.CloneRepoSlug, p.HostRepoSlug, p.DirectTrialMode, &p.Opts); err != nil { + return WorkflowTrialResult{}, fmt.Errorf("failed to install workflow '%s' in trial mode: %w", p.ParsedSpec.WorkflowName, err) + } + executeTrialRunDisplayDescription(p.TempDir, p.ParsedSpec.WorkflowName) - // Display safe outputs to stdout - if len(artifacts.SafeOutputs) > 0 { - outputBytes, _ := json.MarshalIndent(artifacts.SafeOutputs, "", " ") - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("=== Safe Outputs from %s ===", parsedSpec.WorkflowName))) - fmt.Fprintln(os.Stdout, string(outputBytes)) - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("=== End of Safe Outputs ===")) - } else { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== No Safe Outputs Generated by %s ===", parsedSpec.WorkflowName))) - } + runID, err := executeTrialRunTriggerAndWait(p.Ctx, p.HostRepoSlug, p.ParsedSpec.WorkflowName, p.Opts) + if err != nil { + return WorkflowTrialResult{}, err + } - // Display additional artifact information if available - // if len(artifacts.AgentStdioLogs) > 0 { - // fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== Agent Stdio Logs Available from %s (%d files) ===", parsedSpec.WorkflowName, len(artifacts.AgentStdioLogs)))) - // } - if len(artifacts.AgenticRunInfo) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== Agentic Run Information Available from %s ===", parsedSpec.WorkflowName))) - } - if len(artifacts.AdditionalArtifacts) > 0 { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== Additional Artifacts Available from %s (%d files) ===", parsedSpec.WorkflowName, len(artifacts.AdditionalArtifacts)))) + // Auto-merge PRs if requested + if p.Opts.AutoMergePRs { + if err := AutoMergePullRequestsLegacy(p.HostRepoSlug, p.Opts.Verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to auto-merge pull requests: %v", err))) } + } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Trial completed for workflow: "+parsedSpec.WorkflowName)) + artifacts, err := downloadAllArtifacts(p.HostRepoSlug, runID, p.Opts.Verbose) + if err != nil { + return WorkflowTrialResult{}, fmt.Errorf("failed to download artifacts for '%s': %w", p.ParsedSpec.WorkflowName, err) } - // Step 6: Save combined results for multi-workflow trials - if len(parsedSpecs) > 1 { - workflowNames := sliceutil.Map(parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) - workflowNamesStr := strings.Join(workflowNames, "-") - sanitizedTargetRepo := stringutil.SanitizeForFilename(targetRepoForFilename) - combinedFilename := fmt.Sprintf("trials/%s-%s.%s.json", workflowNamesStr, sanitizedTargetRepo, dateTimeID) - combinedResult := CombinedTrialResult{ - WorkflowNames: workflowNames, - Results: workflowResults, - Timestamp: time.Now(), + result := executeTrialRunResult(p.ParsedSpec.WorkflowName, runID, artifacts) + executeTrialRunSaveIndividualResult(result, p.TargetRepoForFilename, p.DateTimeID, p.Opts.Verbose) + executeTrialRunDisplayArtifacts(p.ParsedSpec.WorkflowName, artifacts) + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Trial completed for workflow: "+p.ParsedSpec.WorkflowName)) + return result, nil +} + +func executeTrialRunDisplayDescription(tempDir, workflowName string) { + // Display workflow description if present + workflowPath := filepath.Join(tempDir, constants.GetWorkflowDir(), workflowName+".md") + if description := ExtractWorkflowDescriptionFromFile(workflowPath); description != "" { + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(description)) + fmt.Fprintln(os.Stderr, "") + } +} + +func executeTrialRunTriggerAndWait(ctx context.Context, hostRepoSlug, workflowName string, opts TrialOptions) (string, error) { + // Run the workflow and wait for completion (with trigger context if provided) + runID, err := triggerWorkflowRun(hostRepoSlug, workflowName, opts.TriggerContext, opts.Verbose) + if err != nil { + return "", fmt.Errorf("failed to trigger workflow run for '%s': %w", workflowName, err) + } + + // Generate workflow run URL + githubHost := getGitHubHost() + workflowRunURL := fmt.Sprintf("%s/%s/actions/runs/%s", githubHost, hostRepoSlug, runID) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow run started with ID: %s (%s)", runID, workflowRunURL))) + + // Wait for workflow completion + if err := WaitForWorkflowCompletion(ctx, hostRepoSlug, runID, opts.TimeoutMinutes, opts.Verbose); err != nil { + // If the context was canceled or its deadline was exceeded, return that directly + if ctxErr := ctx.Err(); ctxErr != nil { + return "", ctxErr } - if err := saveTrialResult(combinedFilename, combinedResult, opts.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to save combined trial result: %v", err))) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "", err } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Combined results saved to: "+combinedFilename)) + return "", fmt.Errorf("workflow '%s' execution failed or timed out: %w", workflowName, err) } - // Step 6.5: Copy trial results to host repository and commit them - workflowNames := sliceutil.Map(parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) - if err := copyTrialResultsToHostRepo(tempDir, dateTimeID, workflowNames, targetRepoForFilename, opts.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to copy trial results to repository: %v", err))) + return runID, nil +} + +func executeTrialRunResult(workflowName, runID string, artifacts *TrialArtifacts) WorkflowTrialResult { + return WorkflowTrialResult{ + WorkflowName: workflowName, + RunID: runID, + SafeOutputs: artifacts.SafeOutputs, + //AgentStdioLogs: artifacts.AgentStdioLogs, + AgenticRunInfo: artifacts.AgenticRunInfo, + AdditionalArtifacts: artifacts.AdditionalArtifacts, + Timestamp: time.Now(), } +} - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("All trials completed successfully")) - return nil +func executeTrialRunSaveIndividualResult(result WorkflowTrialResult, targetRepoForFilename, dateTimeID string, verbose bool) { + // Save individual trial file + sanitizedTargetRepo := stringutil.SanitizeForFilename(targetRepoForFilename) + individualFilename := fmt.Sprintf("trials/%s-%s.%s.json", result.WorkflowName, sanitizedTargetRepo, dateTimeID) + if err := saveTrialResult(individualFilename, result, verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to save individual trial result: %v", err))) + } +} + +func executeTrialRunDisplayArtifacts(workflowName string, artifacts *TrialArtifacts) { + // Display safe outputs to stdout + if len(artifacts.SafeOutputs) > 0 { + outputBytes, _ := json.MarshalIndent(artifacts.SafeOutputs, "", " ") + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("=== Safe Outputs from %s ===", workflowName))) + fmt.Fprintln(os.Stdout, string(outputBytes)) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("=== End of Safe Outputs ===")) + } else { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== No Safe Outputs Generated by %s ===", workflowName))) + } + + // Display additional artifact information if available + // if len(artifacts.AgentStdioLogs) > 0 { + // fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== Agent Stdio Logs Available from %s (%d files) ===", workflowName, len(artifacts.AgentStdioLogs)))) + // } + if len(artifacts.AgenticRunInfo) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== Agentic Run Information Available from %s ===", workflowName))) + } + if len(artifacts.AdditionalArtifacts) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("=== Additional Artifacts Available from %s (%d files) ===", workflowName, len(artifacts.AdditionalArtifacts)))) + } +} + +func executeTrialRunSaveCombinedResults(parsedSpecs []*WorkflowSpec, workflowResults []WorkflowTrialResult, targetRepoForFilename, dateTimeID string, verbose bool) { + if len(parsedSpecs) <= 1 { + return + } + + workflowNames := sliceutil.Map(parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) + workflowNamesStr := strings.Join(workflowNames, "-") + sanitizedTargetRepo := stringutil.SanitizeForFilename(targetRepoForFilename) + combinedFilename := fmt.Sprintf("trials/%s-%s.%s.json", workflowNamesStr, sanitizedTargetRepo, dateTimeID) + combinedResult := CombinedTrialResult{ + WorkflowNames: workflowNames, + Results: workflowResults, + Timestamp: time.Now(), + } + if err := saveTrialResult(combinedFilename, combinedResult, verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to save combined trial result: %v", err))) + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Combined results saved to: "+combinedFilename)) } func triggerWorkflowRun(repoSlug, workflowName string, triggerContext string, verbose bool) (string, error) { @@ -300,8 +370,24 @@ func copyTrialResultsToHostRepo(tempDir, dateTimeID string, workflowNames []stri return fmt.Errorf("failed to create trials directory in repository: %w", err) } - // Copy individual workflow result files + copyTrialResultsToHostRepoFiles(trialsDir, dateTimeID, workflowNames, targetRepoSlug, verbose) + if err := copyTrialResultsToHostRepoCommitAndPush(tempDir, dateTimeID, workflowNames, verbose); err != nil { + return err + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Trial results copied to repository and pushed")) + + return nil +} + +func copyTrialResultsToHostRepoFiles(trialsDir, dateTimeID string, workflowNames []string, targetRepoSlug string, verbose bool) { sanitizedTargetRepo := stringutil.SanitizeForFilename(targetRepoSlug) + copyTrialResultsToHostRepoIndividualFiles(trialsDir, dateTimeID, workflowNames, sanitizedTargetRepo, verbose) + copyTrialResultsToHostRepoCombinedFile(trialsDir, dateTimeID, workflowNames, sanitizedTargetRepo, verbose) +} + +func copyTrialResultsToHostRepoIndividualFiles(trialsDir, dateTimeID string, workflowNames []string, sanitizedTargetRepo string, verbose bool) { + // Copy individual workflow result files for _, workflowName := range workflowNames { sourceFile := fmt.Sprintf("trials/%s-%s.%s.json", workflowName, sanitizedTargetRepo, dateTimeID) destFile := filepath.Join(trialsDir, fmt.Sprintf("%s-%s.%s.json", workflowName, sanitizedTargetRepo, dateTimeID)) @@ -317,32 +403,33 @@ func copyTrialResultsToHostRepo(tempDir, dateTimeID string, workflowNames []stri fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Copied %s to repository", sourceFile))) } } +} +func copyTrialResultsToHostRepoCombinedFile(trialsDir, dateTimeID string, workflowNames []string, sanitizedTargetRepo string, verbose bool) { // Copy combined results file if it exists (for multi-workflow trials) - if len(workflowNames) > 1 { - workflowNamesStr := strings.Join(workflowNames, "-") - combinedSourceFile := fmt.Sprintf("trials/%s-%s.%s.json", workflowNamesStr, sanitizedTargetRepo, dateTimeID) - combinedDestFile := filepath.Join(trialsDir, fmt.Sprintf("%s-%s.%s.json", workflowNamesStr, sanitizedTargetRepo, dateTimeID)) + if len(workflowNames) <= 1 { + return + } - if err := fileutil.CopyFile(combinedSourceFile, combinedDestFile); err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to copy combined results: %v", err))) - } - } else if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Copied %s to repository", combinedSourceFile))) + workflowNamesStr := strings.Join(workflowNames, "-") + combinedSourceFile := fmt.Sprintf("trials/%s-%s.%s.json", workflowNamesStr, sanitizedTargetRepo, dateTimeID) + combinedDestFile := filepath.Join(trialsDir, fmt.Sprintf("%s-%s.%s.json", workflowNamesStr, sanitizedTargetRepo, dateTimeID)) + + if err := fileutil.CopyFile(combinedSourceFile, combinedDestFile); err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to copy combined results: %v", err))) } + } else if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Copied %s to repository", combinedSourceFile))) } +} - // Change to temp directory to commit the changes - originalDir, err := os.Getwd() +func copyTrialResultsToHostRepoCommitAndPush(tempDir, dateTimeID string, workflowNames []string, verbose bool) error { + restoreDir, err := copyTrialResultsToHostRepoChdir(tempDir) if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - defer os.Chdir(originalDir) - - if err := os.Chdir(tempDir); err != nil { - return fmt.Errorf("failed to change to temp directory: %w", err) + return err } + defer restoreDir() // Add trial results to git cmd := exec.Command("git", "add", "trials/") @@ -350,11 +437,41 @@ func copyTrialResultsToHostRepo(tempDir, dateTimeID string, workflowNames []stri return fmt.Errorf("failed to add trial results: %w (output: %s)", err, string(output)) } + changed, err := copyTrialResultsToHostRepoHasChanges(verbose) + if err != nil || !changed { + return err + } + + commitMsg := fmt.Sprintf("Add trial results for %s (%s)", strings.Join(workflowNames, ", "), dateTimeID) + cmd = exec.Command("git", "commit", "-m", commitMsg) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to commit trial results: %w (output: %s)", err, string(output)) + } + + return copyTrialResultsToHostRepoPullAndPush(tempDir, verbose) +} + +func copyTrialResultsToHostRepoChdir(tempDir string) (func(), error) { + // Change to temp directory to commit the changes + originalDir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to get current directory: %w", err) + } + + if err := os.Chdir(tempDir); err != nil { + return nil, fmt.Errorf("failed to change to temp directory: %w", err) + } + return func() { + _ = os.Chdir(originalDir) + }, nil +} + +func copyTrialResultsToHostRepoHasChanges(verbose bool) (bool, error) { // Check if there are any changes to commit statusCmd := exec.Command("git", "status", "--porcelain", "trials/") statusOutput, err := statusCmd.Output() if err != nil { - return fmt.Errorf("failed to check git status: %w", err) + return false, fmt.Errorf("failed to check git status: %w", err) } // If no changes, skip commit and push @@ -363,16 +480,12 @@ func copyTrialResultsToHostRepo(tempDir, dateTimeID string, workflowNames []stri if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No new trial results to commit")) } - return nil - } - - // Commit trial results - commitMsg := fmt.Sprintf("Add trial results for %s (%s)", strings.Join(workflowNames, ", "), dateTimeID) - cmd = exec.Command("git", "commit", "-m", commitMsg) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to commit trial results: %w (output: %s)", err, string(output)) + return false, nil } + return true, nil +} +func copyTrialResultsToHostRepoPullAndPush(tempDir string, verbose bool) error { branch, err := getCurrentBranchIn(tempDir) if err != nil { trialLog.Printf("Failed to detect branch in %s: %v, falling back to main", tempDir, err) @@ -382,7 +495,7 @@ func copyTrialResultsToHostRepo(tempDir, dateTimeID string, workflowNames []stri if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Pulling latest changes from "+branch+" branch")) } - cmd = exec.Command("git", "pull", "--rebase", "origin", branch) + cmd := exec.Command("git", "pull", "--rebase", "origin", branch) cmd.Dir = tempDir if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to pull latest changes: %w (output: %s)", err, string(output)) @@ -394,8 +507,5 @@ func copyTrialResultsToHostRepo(tempDir, dateTimeID string, workflowNames []stri if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to push trial results: %w (output: %s)", err, string(output)) } - - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Trial results copied to repository and pushed")) - return nil } diff --git a/pkg/cli/trial_repository.go b/pkg/cli/trial_repository.go index 2aafd7be8b7..988a22fd8d7 100644 --- a/pkg/cli/trial_repository.go +++ b/pkg/cli/trial_repository.go @@ -44,9 +44,8 @@ func trialRepositoryActionsSettingsURL(repoSlug string) string { func ensureTrialRepository(repoSlug string, cloneRepoSlug string, forceDeleteHostRepo bool, dryRun bool, verbose bool) error { trialRepoLog.Printf("Ensuring trial repository: %s (cloneRepo=%s, forceDelete=%v, dryRun=%v)", repoSlug, cloneRepoSlug, forceDeleteHostRepo, dryRun) - parts := strings.Split(repoSlug, "/") - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return fmt.Errorf("invalid repository slug format: %s. Expected format: owner/repo. Example: github/gh-aw", repoSlug) + if err := ensureTrialRepositoryValidateSlug(repoSlug); err != nil { + return err } // Check if repository already exists @@ -54,54 +53,82 @@ func ensureTrialRepository(repoSlug string, cloneRepoSlug string, forceDeleteHos output, err := cmd.CombinedOutput() repoExists := err == nil - if dryRun && verbose { - if repoExists { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("[DRY RUN] Repository %s exists", repoSlug))) - } else { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("[DRY RUN] Repository %s does not exist (output: %s)", repoSlug, string(output)))) + ensureTrialRepositoryLogDryRunExists(repoSlug, output, repoExists, dryRun, verbose) + + if repoExists { + shouldCreate, err := ensureTrialRepositoryHandleExisting(repoSlug, cloneRepoSlug, forceDeleteHostRepo, dryRun, verbose) + if err != nil || !shouldCreate { + return err } } + // Repository doesn't exist, create it + return ensureTrialRepositoryCreate(repoSlug, dryRun, verbose) +} + +func ensureTrialRepositoryValidateSlug(repoSlug string) error { + parts := strings.Split(repoSlug, "/") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return fmt.Errorf("invalid repository slug format: %s. Expected format: owner/repo. Example: github/gh-aw", repoSlug) + } + return nil +} + +func ensureTrialRepositoryLogDryRunExists(repoSlug string, output []byte, repoExists, dryRun, verbose bool) { + if !dryRun || !verbose { + return + } if repoExists { - trialRepoLog.Printf("Repository %s already exists", repoSlug) - // Repository exists - determine what to do - if forceDeleteHostRepo { - // Force delete mode: delete the existing repository first - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Force deleting existing host repository: "+repoSlug)) - } - - if dryRun { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Would delete repository: "+repoSlug)) - } else { - if deleteOutput, deleteErr := workflow.RunGHCombined("Deleting repository...", "repo", "delete", repoSlug, "--yes"); deleteErr != nil { - return fmt.Errorf("failed to force delete existing host repository %s: %w (output: %s)", repoSlug, deleteErr, string(deleteOutput)) - } - - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Force deleted existing host repository: "+repoSlug)) - } - - // Continue to create the repository below + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("[DRY RUN] Repository %s exists", repoSlug))) + } else { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("[DRY RUN] Repository %s does not exist (output: %s)", repoSlug, string(output)))) + } +} + +func ensureTrialRepositoryHandleExisting(repoSlug string, cloneRepoSlug string, forceDeleteHostRepo bool, dryRun bool, verbose bool) (bool, error) { + trialRepoLog.Printf("Repository %s already exists", repoSlug) + // Repository exists - determine what to do + if forceDeleteHostRepo { + // Force delete mode: delete the existing repository first + return true, ensureTrialRepositoryForceDelete(repoSlug, dryRun, verbose) + } + + // Both clone-repo and logical-repo modes: reusing is allowed + // In clone-repo mode, the cloneRepoContentsIntoHost function will force push the new contents + if verbose { + if cloneRepoSlug != "" { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Reusing existing host repository: %s (contents will be force-pushed)", repoSlug))) } else { - // Both clone-repo and logical-repo modes: reusing is allowed - // In clone-repo mode, the cloneRepoContentsIntoHost function will force push the new contents - if verbose { - if cloneRepoSlug != "" { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Reusing existing host repository: %s (contents will be force-pushed)", repoSlug))) - } else { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Reusing existing host repository: "+repoSlug)) - } - } - prefix := "" - if dryRun { - prefix = "[DRY RUN] " - } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("%sUsing existing host repository: %s", prefix, trialRepositoryURL(repoSlug)))) - return nil + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Reusing existing host repository: "+repoSlug)) } } + prefix := "" + if dryRun { + prefix = "[DRY RUN] " + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("%sUsing existing host repository: %s", prefix, trialRepositoryURL(repoSlug)))) + return false, nil +} - // Repository doesn't exist, create it +func ensureTrialRepositoryForceDelete(repoSlug string, dryRun bool, verbose bool) error { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Force deleting existing host repository: "+repoSlug)) + } + + if dryRun { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Would delete repository: "+repoSlug)) + return nil + } + + if deleteOutput, deleteErr := workflow.RunGHCombined("Deleting repository...", "repo", "delete", repoSlug, "--yes"); deleteErr != nil { + return fmt.Errorf("failed to force delete existing host repository %s: %w (output: %s)", repoSlug, deleteErr, string(deleteOutput)) + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Force deleted existing host repository: "+repoSlug)) + return nil +} + +func ensureTrialRepositoryCreate(repoSlug string, dryRun bool, verbose bool) error { if verbose || dryRun { prefix := "" if dryRun { @@ -111,25 +138,15 @@ func ensureTrialRepository(repoSlug string, cloneRepoSlug string, forceDeleteHos } if dryRun { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Would create repository with description: 'GitHub Agentic Workflows host repository'")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Would enable GitHub Actions permissions at: "+trialRepositoryActionsSettingsURL(repoSlug))) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Would enable discussions")) - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("[DRY RUN] Would create host repository: "+trialRepositoryURL(repoSlug))) + ensureTrialRepositoryPrintDryRunCreate(repoSlug) return nil } // Use gh CLI to create private repo with initial README using full OWNER/REPO format - output, err = workflow.RunGHCombined("Creating repository...", "repo", "create", repoSlug, "--private", "--add-readme", "--description", "GitHub Agentic Workflows host repository") + output, err := workflow.RunGHCombined("Creating repository...", "repo", "create", repoSlug, "--private", "--add-readme", "--description", "GitHub Agentic Workflows host repository") if err != nil { - // Check if the error is because the repository already exists - outputStr := string(output) - if strings.Contains(outputStr, "name already exists") { - // Repository exists but gh repo view failed earlier - this is okay, reuse it - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Repository already exists (detected via create error): "+repoSlug)) - } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Using existing host repository: "+trialRepositoryURL(repoSlug))) + if ensureTrialRepositoryHandleCreateExists(repoSlug, output, verbose) { return nil } return fmt.Errorf("failed to create host repository: %w (output: %s)", err, string(output)) @@ -138,6 +155,36 @@ func ensureTrialRepository(repoSlug string, cloneRepoSlug string, forceDeleteHos // Show host repository creation message with URL fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Created host repository: "+trialRepositoryURL(repoSlug))) + ensureTrialRepositoryPromptForActions(repoSlug) + ensureTrialRepositoryEnableDiscussions(repoSlug, verbose) + + // Give GitHub a moment to fully initialize the repository + time.Sleep(trialRepoInitDelay) + + return nil +} + +func ensureTrialRepositoryPrintDryRunCreate(repoSlug string) { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Would create repository with description: 'GitHub Agentic Workflows host repository'")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Would enable GitHub Actions permissions at: "+trialRepositoryActionsSettingsURL(repoSlug))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Would enable discussions")) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("[DRY RUN] Would create host repository: "+trialRepositoryURL(repoSlug))) +} + +func ensureTrialRepositoryHandleCreateExists(repoSlug string, output []byte, verbose bool) bool { + if !strings.Contains(string(output), "name already exists") { + return false + } + + // Repository exists but gh repo view failed earlier - this is okay, reuse it + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Repository already exists (detected via create error): "+repoSlug)) + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Using existing host repository: "+trialRepositoryURL(repoSlug))) + return true +} + +func ensureTrialRepositoryPromptForActions(repoSlug string) { // Prompt user to enable GitHub Actions permissions fmt.Fprintln(os.Stderr, console.FormatInfoMessage("")) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("IMPORTANT: You must enable GitHub Actions permissions for the repository.")) @@ -151,7 +198,9 @@ func ensureTrialRepository(repoSlug string, cloneRepoSlug string, forceDeleteHos var userInput string _, _ = fmt.Scanln(&userInput) // Ignore error (user pressed Enter without typing anything) fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Continuing with trial setup")) +} +func ensureTrialRepositoryEnableDiscussions(repoSlug string, verbose bool) { // Enable discussions in the repository as most workflows use them if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Enabling discussions in repository: "+repoSlug)) @@ -163,11 +212,6 @@ func ensureTrialRepository(repoSlug string, cloneRepoSlug string, forceDeleteHos } else if verbose { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Enabled discussions in host repository")) } - - // Give GitHub a moment to fully initialize the repository - time.Sleep(trialRepoInitDelay) - - return nil } func cleanupTrialRepository(repoSlug string, verbose bool) error { @@ -222,50 +266,87 @@ func installWorkflowInTrialMode(ctx context.Context, tempDir string, parsedSpec trialRepoLog.Printf("Installing workflow in trial mode: workflow=%s, hostRepo=%s, directMode=%v", parsedSpec.WorkflowName, hostRepoSlug, directTrialMode) // Change to temp directory + originalDir, restoreDir, err := installWorkflowInTrialModeChdir(tempDir) + if err != nil { + return err + } + defer restoreDir() + + // Fetch workflow content - handle local workflows specially since they need + // to be resolved from the original directory, not the tempDir + fetched, err := installWorkflowInTrialModeFetch(ctx, tempDir, originalDir, parsedSpec, opts) + if err != nil { + return fmt.Errorf("failed to fetch workflow: %w", err) + } + + content := fetched.Content + parsedSpec = installWorkflowInTrialModeNormalizeSpec(parsedSpec, fetched) + content = installWorkflowInTrialModeAddSource(parsedSpec, fetched, content, opts) + + result, err := writeWorkflowToTrialDir(tempDir, parsedSpec.WorkflowName, content, opts) + if err != nil { + return err + } + if err := installWorkflowInTrialModePostWrite(installWorkflowInTrialModePostWriteParams{ + Ctx: ctx, + TempDir: tempDir, + ParsedSpec: parsedSpec, + Fetched: fetched, + Content: content, + Result: result, + LogicalRepoSlug: logicalRepoSlug, + DirectTrialMode: directTrialMode, + Opts: opts, + }); err != nil { + return err + } + if err := installWorkflowInTrialModeCompile(ctx, parsedSpec.WorkflowName, logicalRepoSlug, cloneRepoSlug, directTrialMode, opts); err != nil { + return err + } + + // Commit and push the changes + if err := commitAndPushWorkflow(tempDir, parsedSpec.WorkflowName, opts.Verbose); err != nil { + return fmt.Errorf("failed to commit and push workflow: %w", err) + } + + return nil +} + +func installWorkflowInTrialModeChdir(tempDir string) (string, func(), error) { originalDir, err := os.Getwd() if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) + return "", nil, fmt.Errorf("failed to get current directory: %w", err) } - defer os.Chdir(originalDir) if err := os.Chdir(tempDir); err != nil { - return fmt.Errorf("failed to change to temp directory: %w", err) + return "", nil, fmt.Errorf("failed to change to temp directory: %w", err) } + return originalDir, func() { _ = os.Chdir(originalDir) }, nil +} - // Fetch workflow content - handle local workflows specially since they need - // to be resolved from the original directory, not the tempDir +func installWorkflowInTrialModeFetch(ctx context.Context, tempDir, originalDir string, parsedSpec *WorkflowSpec, opts *TrialOptions) (*FetchedWorkflow, error) { specToFetch := parsedSpec - var fetched *FetchedWorkflow - if isLocalWorkflowPath(parsedSpec.WorkflowPath) { - // For local workflows, temporarily change to original dir for fetch - // Use a closure to ensure directory is restored even on error - fetched, err = func() (*FetchedWorkflow, error) { - if chErr := os.Chdir(originalDir); chErr != nil { - return nil, fmt.Errorf("failed to change to original directory for local fetch: %w", chErr) - } - // Always restore to tempDir when this closure exits - defer os.Chdir(tempDir) - - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Installing local workflow '%s' from '%s' in trial mode", parsedSpec.WorkflowName, parsedSpec.WorkflowPath))) - } - return FetchWorkflowFromSourceWithContext(ctx, specToFetch, opts.Verbose) - }() - } else { - // Remote workflows can be fetched from any directory + if chErr := os.Chdir(originalDir); chErr != nil { + return nil, fmt.Errorf("failed to change to original directory for local fetch: %w", chErr) + } + // Always restore to tempDir before returning + defer os.Chdir(tempDir) + if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Installing workflow '%s' from '%s' in trial mode", parsedSpec.WorkflowName, parsedSpec.RepoSlug))) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Installing local workflow '%s' from '%s' in trial mode", parsedSpec.WorkflowName, parsedSpec.WorkflowPath))) } - fetched, err = FetchWorkflowFromSourceWithContext(ctx, specToFetch, opts.Verbose) + return FetchWorkflowFromSourceWithContext(ctx, specToFetch, opts.Verbose) } - if err != nil { - return fmt.Errorf("failed to fetch workflow: %w", err) + // Remote workflows can be fetched from any directory + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Installing workflow '%s' from '%s' in trial mode", parsedSpec.WorkflowName, parsedSpec.RepoSlug))) } + return FetchWorkflowFromSourceWithContext(ctx, specToFetch, opts.Verbose) +} - content := fetched.Content - +func installWorkflowInTrialModeNormalizeSpec(parsedSpec *WorkflowSpec, fetched *FetchedWorkflow) *WorkflowSpec { // When the fetch used a fallback path (e.g. .github/workflows/my-workflow.md // instead of the short-form my-workflow.md), SourcePath holds the actual // repo-root-relative path. Normalize parsedSpec so all downstream dependency @@ -274,57 +355,74 @@ func installWorkflowInTrialMode(ctx context.Context, tempDir string, parsedSpec if !fetched.IsLocal && fetched.SourcePath != "" && fetched.SourcePath != parsedSpec.WorkflowPath { specCopy := *parsedSpec specCopy.WorkflowPath = fetched.SourcePath - parsedSpec = &specCopy + return &specCopy } + return parsedSpec +} +func installWorkflowInTrialModeAddSource(parsedSpec *WorkflowSpec, fetched *FetchedWorkflow, content []byte, opts *TrialOptions) []byte { // Add source field to frontmatter for remote workflows - if !fetched.IsLocal && fetched.CommitSHA != "" { - sourceString := buildSourceStringWithCommitSHA(parsedSpec, fetched.CommitSHA) - if sourceString != "" { - updatedContent, err := addSourceToWorkflow(string(content), sourceString) - if err != nil { - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to add source field: %v", err))) - } - } else { - content = []byte(updatedContent) - } - } + if fetched.IsLocal || fetched.CommitSHA == "" { + return content } - // Use common helper for security scan, directory creation, and writing - result, err := writeWorkflowToTrialDir(tempDir, parsedSpec.WorkflowName, content, opts) + sourceString := buildSourceStringWithCommitSHA(parsedSpec, fetched.CommitSHA) + if sourceString == "" { + return content + } + + updatedContent, err := addSourceToWorkflow(string(content), sourceString) if err != nil { - return err + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to add source field: %v", err))) + } + return content } + return []byte(updatedContent) +} - if opts.Verbose { - if fetched.IsLocal { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Copied local workflow to "+result.DestPath)) +type installWorkflowInTrialModePostWriteParams struct { + Ctx context.Context + TempDir string + ParsedSpec *WorkflowSpec + Fetched *FetchedWorkflow + Content []byte + Result *trialWorkflowWriteResult + LogicalRepoSlug string + DirectTrialMode bool + Opts *TrialOptions +} + +func installWorkflowInTrialModePostWrite(p installWorkflowInTrialModePostWriteParams) error { + if p.Opts.Verbose { + if p.Fetched.IsLocal { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Copied local workflow to "+p.Result.DestPath)) } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched remote workflow to "+result.DestPath)) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Fetched remote workflow to "+p.Result.DestPath)) } } // Fetch and save all remote dependencies (includes, imports, dispatch workflows, resources) - if !fetched.IsLocal { - if err := fetchAllRemoteDependencies(ctx, string(content), parsedSpec, result.WorkflowsDir, opts.Verbose, true, nil); err != nil { + if !p.Fetched.IsLocal { + if err := fetchAllRemoteDependencies(p.Ctx, string(p.Content), p.ParsedSpec, p.Result.WorkflowsDir, p.Opts.Verbose, true, nil); err != nil { return fmt.Errorf("failed to fetch remote dependencies: %w", err) } } // Modify the workflow for trial mode (skip in direct trial mode) - if !directTrialMode { - if err := modifyWorkflowForTrialMode(tempDir, parsedSpec.WorkflowName, logicalRepoSlug, opts.Verbose); err != nil { + if !p.DirectTrialMode { + if err := modifyWorkflowForTrialMode(p.TempDir, p.ParsedSpec.WorkflowName, p.LogicalRepoSlug, p.Opts.Verbose); err != nil { return fmt.Errorf("failed to modify workflow for trial mode: %w", err) } - } else if opts.Verbose { + } else if p.Opts.Verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Direct trial mode: Skipping trial mode modifications")) } + return nil +} - // Compile the workflow with trial modifications +func installWorkflowInTrialModeCompile(ctx context.Context, workflowName, logicalRepoSlug, cloneRepoSlug string, directTrialMode bool, opts *TrialOptions) error { config := CompileConfig{ - MarkdownFiles: []string{constants.WorkflowsDirSlash + parsedSpec.WorkflowName + ".md"}, + MarkdownFiles: []string{constants.WorkflowsDirSlash + workflowName + ".md"}, Verbose: opts.Verbose, EngineOverride: opts.EngineOverride, Validate: true, @@ -345,12 +443,6 @@ func installWorkflowInTrialMode(ctx context.Context, tempDir string, parsedSpec } // Note: workflowData is used for validation; secrets are ensured before installWorkflowInTrialMode is called _ = workflowDataList[0] - - // Commit and push the changes - if err := commitAndPushWorkflow(tempDir, parsedSpec.WorkflowName, opts.Verbose); err != nil { - return fmt.Errorf("failed to commit and push workflow: %w", err) - } - return nil } @@ -445,31 +537,7 @@ func modifyWorkflowForTrialMode(tempDir, workflowName, logicalRepoSlug string, v modifiedContent := string(content) if logicalRepoSlug != "" { - // Replace github.repository references to point to simulated host repo - modifiedContent = strings.ReplaceAll(modifiedContent, "${{ github.repository }}", logicalRepoSlug) - - // Also replace any hardcoded checkout actions to use the simulated host repo - // Split content into lines to preserve indentation - lines := strings.Split(modifiedContent, "\n") - - var newLines []string - for _, line := range lines { - if matches := checkoutActionPattern.FindStringSubmatch(line); len(matches) >= 3 { - indentation := matches[1] - usesLine := matches[2] - remainder := matches[3] - - // Add the original uses line - newLines = append(newLines, fmt.Sprintf("%s%s%s", indentation, usesLine, remainder)) - // Add the with clause at the same indentation level as uses - newLines = append(newLines, indentation+"with:") - newLines = append(newLines, fmt.Sprintf("%s repository: %s", indentation, logicalRepoSlug)) - } else { - newLines = append(newLines, line) - } - } - - modifiedContent = strings.Join(newLines, "\n") + modifiedContent = modifyWorkflowForTrialModeRepositoryRefs(modifiedContent, logicalRepoSlug) } else if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping repository simulation modifications (using clone-repo mode)")) } @@ -486,6 +554,34 @@ func modifyWorkflowForTrialMode(tempDir, workflowName, logicalRepoSlug string, v return nil } +func modifyWorkflowForTrialModeRepositoryRefs(content, logicalRepoSlug string) string { + // Replace github.repository references to point to simulated host repo + modifiedContent := strings.ReplaceAll(content, "${{ github.repository }}", logicalRepoSlug) + + // Also replace any hardcoded checkout actions to use the simulated host repo + // Split content into lines to preserve indentation + lines := strings.Split(modifiedContent, "\n") + + var newLines []string + for _, line := range lines { + if matches := checkoutActionPattern.FindStringSubmatch(line); len(matches) >= 3 { + indentation := matches[1] + usesLine := matches[2] + remainder := matches[3] + + // Add the original uses line + newLines = append(newLines, fmt.Sprintf("%s%s%s", indentation, usesLine, remainder)) + // Add the with clause at the same indentation level as uses + newLines = append(newLines, indentation+"with:") + newLines = append(newLines, fmt.Sprintf("%s repository: %s", indentation, logicalRepoSlug)) + } else { + newLines = append(newLines, line) + } + } + + return strings.Join(newLines, "\n") +} + // commitAndPushWorkflow commits and pushes the workflow changes func commitAndPushWorkflow(tempDir, workflowName string, verbose bool) error { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Committing workflow and lock files to host repository")) diff --git a/pkg/cli/trial_runner.go b/pkg/cli/trial_runner.go index 286f2336c92..edb2dba9822 100644 --- a/pkg/cli/trial_runner.go +++ b/pkg/cli/trial_runner.go @@ -22,227 +22,268 @@ func RunWorkflowTrials(ctx context.Context, workflowSpecs []string, opts TrialOp // Show welcome banner for interactive mode console.ShowWelcomeBanner("This tool will run a trial of your workflow in a test repository.") - // Parse all workflow specifications + parsedSpecs, err := runWorkflowTrialsParseSpecs(workflowSpecs) + if err != nil { + return err + } + if opts.DryRun { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Showing what would be done without making changes")) + } + runWorkflowTrialsPrintStart(parsedSpecs) + + mode, err := runWorkflowTrialsDetermineMode(opts) + if err != nil { + return err + } + hostRepoSlug, err := runWorkflowTrialsHostRepo(ctx, opts) + if err != nil { + return err + } + + if err := runWorkflowTrialsConfirm(parsedSpecs, mode, hostRepoSlug, opts); err != nil { + return err + } + + // Step 2: Create or reuse host repository + trialLog.Printf("Ensuring trial repository exists: %s", hostRepoSlug) + if err := ensureTrialRepository(hostRepoSlug, mode.cloneRepoSlug, opts.ForceDelete, opts.DryRun, opts.Verbose); err != nil { + return fmt.Errorf("failed to ensure host repository: %w", err) + } + + // In dry-run mode, stop here after showing what would be done + if opts.DryRun { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Stopping here. No actual changes were made.")) + return nil + } + + cleanup, err := runWorkflowTrialsPrepareExecution(ctx, mode, hostRepoSlug, opts) + if err != nil { + return err + } + defer cleanup() + + // Step 2.8: Disable all workflows except the ones being trialled (only in clone-repo mode, done once before all trials) + if err := runWorkflowTrialsDisableClonedWorkflows(parsedSpecs, mode.cloneRepoSlug, hostRepoSlug, opts); err != nil { + return err + } + + return runWorkflowTrialsExecute(ctx, parsedSpecs, mode, hostRepoSlug, opts) + +} + +func runWorkflowTrialsPrepareExecution(ctx context.Context, mode runWorkflowTrialsMode, hostRepoSlug string, opts TrialOptions) (func(), error) { + // Step 2.5: Ensure engine secrets are configured when an explicit engine override is provided + // When no override is specified, the workflow will use its frontmatter engine and handle secrets during compilation + if err := runWorkflowTrialsEnsureEngineSecrets(ctx, hostRepoSlug, opts); err != nil { + return func() {}, err + } + + cleanup := runWorkflowTrialsSetupCleanup(hostRepoSlug, opts) + if cleanup == nil { + cleanup = func() {} + } + + // Step 2.7: Clone source repository contents if in clone-repo mode + if mode.cloneRepoSlug != "" { + if err := cloneRepoContentsIntoHost(mode.cloneRepoSlug, mode.cloneRepoVersion, hostRepoSlug, opts.Verbose); err != nil { + cleanup() + return func() {}, fmt.Errorf("failed to clone repository contents: %w", err) + } + } + return cleanup, nil +} + +type runWorkflowTrialsMode struct { + logicalRepoSlug string + cloneRepoSlug string + cloneRepoVersion string + directTrialMode bool +} + +func runWorkflowTrialsParseSpecs(workflowSpecs []string) ([]*WorkflowSpec, error) { var parsedSpecs []*WorkflowSpec for _, spec := range workflowSpecs { parsedSpec, err := parseWorkflowSpec(spec) if err != nil { - return fmt.Errorf("invalid workflow specification '%s': %w", spec, err) + return nil, fmt.Errorf("invalid workflow specification '%s': %w", spec, err) } parsedSpecs = append(parsedSpecs, parsedSpec) } + return parsedSpecs, nil +} - if opts.DryRun { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Showing what would be done without making changes")) - } - +func runWorkflowTrialsPrintStart(parsedSpecs []*WorkflowSpec) { if len(parsedSpecs) == 1 { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Starting trial of workflow '%s' from '%s'", parsedSpecs[0].WorkflowName, parsedSpecs[0].RepoSlug))) - } else { - workflowNames := sliceutil.Map(parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) - joinedNames := strings.Join(workflowNames, ", ") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Starting trial of %d workflows (%s)", len(parsedSpecs), joinedNames))) + return } - // Step 0: Determine workflow mode (mutual exclusion is enforced by Cobra) - var logicalRepoSlug string - var cloneRepoSlug string - var cloneRepoVersion string - var directTrialMode bool + workflowNames := sliceutil.Map(parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) + joinedNames := strings.Join(workflowNames, ", ") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Starting trial of %d workflows (%s)", len(parsedSpecs), joinedNames))) +} +func runWorkflowTrialsDetermineMode(opts TrialOptions) (runWorkflowTrialsMode, error) { if opts.Repos.CloneRepo != "" { - // Use clone-repo mode: clone the specified repo contents into host repo cloneRepo, err := parseRepoSpec(opts.Repos.CloneRepo) if err != nil { - return fmt.Errorf("invalid --clone-repo specification '%s': %w", opts.Repos.CloneRepo, err) + return runWorkflowTrialsMode{}, fmt.Errorf("invalid --clone-repo specification '%s': %w", opts.Repos.CloneRepo, err) } - - cloneRepoSlug = cloneRepo.RepoSlug - cloneRepoVersion = cloneRepo.Version - logicalRepoSlug = "" // Empty string means skip logical repo simulation - directTrialMode = false - trialLog.Printf("Using clone-repo mode: %s (version=%s)", cloneRepoSlug, cloneRepoVersion) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Clone mode: Will clone contents from %s into host repository", cloneRepoSlug))) - } else if opts.Repos.LogicalRepo != "" { - // Use logical-repo mode: simulate the workflow running against the specified repo + trialLog.Printf("Using clone-repo mode: %s (version=%s)", cloneRepo.RepoSlug, cloneRepo.Version) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Clone mode: Will clone contents from %s into host repository", cloneRepo.RepoSlug))) + return runWorkflowTrialsMode{cloneRepoSlug: cloneRepo.RepoSlug, cloneRepoVersion: cloneRepo.Version}, nil + } + if opts.Repos.LogicalRepo != "" { logicalRepo, err := parseRepoSpec(opts.Repos.LogicalRepo) if err != nil { - return fmt.Errorf("invalid --logical-repo specification '%s': %w", opts.Repos.LogicalRepo, err) + return runWorkflowTrialsMode{}, fmt.Errorf("invalid --logical-repo specification '%s': %w", opts.Repos.LogicalRepo, err) } + trialLog.Printf("Using logical-repo mode: %s", logicalRepo.RepoSlug) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Target repository (specified): "+logicalRepo.RepoSlug)) + return runWorkflowTrialsMode{logicalRepoSlug: logicalRepo.RepoSlug}, nil + } + return runWorkflowTrialsDefaultMode(opts) +} - logicalRepoSlug = logicalRepo.RepoSlug - directTrialMode = false - trialLog.Printf("Using logical-repo mode: %s", logicalRepoSlug) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Target repository (specified): "+logicalRepoSlug)) - } else { - // No --clone-repo or --logical-repo specified - // If --repo is specified without simulation flags, it's direct trial mode - // Otherwise, fall back to current repository for logical-repo mode - if opts.Repos.HostRepo != "" { - // Direct trial mode: run workflows directly in the specified repo without simulation - logicalRepoSlug = "" - cloneRepoSlug = "" - directTrialMode = true - trialLog.Print("Using direct trial mode (no simulation)") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Direct trial mode: Workflows will be installed and run directly in the specified repository")) - } else { - // Fall back to current repository for logical-repo mode - var err error - logicalRepoSlug, err = GetCurrentRepoSlug() - if err != nil { - return fmt.Errorf("failed to determine simulated host repository: %w", err) - } - directTrialMode = false - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Target repository (current): "+logicalRepoSlug)) - } +func runWorkflowTrialsDefaultMode(opts TrialOptions) (runWorkflowTrialsMode, error) { + if opts.Repos.HostRepo != "" { + trialLog.Print("Using direct trial mode (no simulation)") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Direct trial mode: Workflows will be installed and run directly in the specified repository")) + return runWorkflowTrialsMode{directTrialMode: true}, nil } - // Step 1: Determine host repository slug - var hostRepoSlug string + logicalRepoSlug, err := GetCurrentRepoSlug() + if err != nil { + return runWorkflowTrialsMode{}, fmt.Errorf("failed to determine simulated host repository: %w", err) + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Target repository (current): "+logicalRepoSlug)) + return runWorkflowTrialsMode{logicalRepoSlug: logicalRepoSlug}, nil +} + +func runWorkflowTrialsHostRepo(ctx context.Context, opts TrialOptions) (string, error) { if opts.Repos.HostRepo != "" { hostRepo, err := parseRepoSpec(opts.Repos.HostRepo) if err != nil { - return fmt.Errorf("invalid --host-repo specification '%s': %w", opts.Repos.HostRepo, err) - } - hostRepoSlug = hostRepo.RepoSlug - trialLog.Printf("Using specified host repository: %s", hostRepoSlug) - } else { - // Use default trial repo with current username - username, err := getCurrentGitHubUsername(ctx) - if err != nil { - return fmt.Errorf("failed to get GitHub username for default trial repo: %w", err) - } - hostRepoSlug = username + "/gh-aw-trial" - trialLog.Printf("Using default host repository: %s", hostRepoSlug) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Host repository (default): "+hostRepoSlug)) - } - - // Step 1.5: Show confirmation unless quiet mode - if !opts.Quiet { - if err := showTrialConfirmation(trialConfirmationOptions{ - parsedSpecs: parsedSpecs, - logicalRepoSlug: logicalRepoSlug, - cloneRepoSlug: cloneRepoSlug, - hostRepoSlug: hostRepoSlug, - deleteHostRepo: opts.DeleteHostRepo, - forceDeleteHostRepo: opts.ForceDelete, - autoMergePRs: opts.AutoMergePRs, - repeatCount: opts.RepeatCount, - directTrialMode: directTrialMode, - engineOverride: opts.EngineOverride, - }); err != nil { - return err + return "", fmt.Errorf("invalid --host-repo specification '%s': %w", opts.Repos.HostRepo, err) } + trialLog.Printf("Using specified host repository: %s", hostRepo.RepoSlug) + return hostRepo.RepoSlug, nil } - // Step 2: Create or reuse host repository - trialLog.Printf("Ensuring trial repository exists: %s", hostRepoSlug) - if err := ensureTrialRepository(hostRepoSlug, cloneRepoSlug, opts.ForceDelete, opts.DryRun, opts.Verbose); err != nil { - return fmt.Errorf("failed to ensure host repository: %w", err) + username, err := getCurrentGitHubUsername(ctx) + if err != nil { + return "", fmt.Errorf("failed to get GitHub username for default trial repo: %w", err) } + hostRepoSlug := username + "/gh-aw-trial" + trialLog.Printf("Using default host repository: %s", hostRepoSlug) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Host repository (default): "+hostRepoSlug)) + return hostRepoSlug, nil +} - // In dry-run mode, stop here after showing what would be done - if opts.DryRun { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("[DRY RUN] Stopping here. No actual changes were made.")) +func runWorkflowTrialsConfirm(parsedSpecs []*WorkflowSpec, mode runWorkflowTrialsMode, hostRepoSlug string, opts TrialOptions) error { + if opts.Quiet { return nil } + return showTrialConfirmation(trialConfirmationOptions{ + parsedSpecs: parsedSpecs, + logicalRepoSlug: mode.logicalRepoSlug, + cloneRepoSlug: mode.cloneRepoSlug, + hostRepoSlug: hostRepoSlug, + deleteHostRepo: opts.DeleteHostRepo, + forceDeleteHostRepo: opts.ForceDelete, + autoMergePRs: opts.AutoMergePRs, + repeatCount: opts.RepeatCount, + directTrialMode: mode.directTrialMode, + engineOverride: opts.EngineOverride, + }) +} - // Step 2.5: Ensure engine secrets are configured when an explicit engine override is provided - // When no override is specified, the workflow will use its frontmatter engine and handle secrets during compilation - if opts.EngineOverride != "" { - // Check what secrets already exist in the repository - existingSecrets, err := getExistingSecretsInRepo(hostRepoSlug) - if err != nil { - trialLog.Printf("Warning: could not check existing secrets: %v", err) - existingSecrets = make(map[string]struct { - }) - } +func runWorkflowTrialsEnsureEngineSecrets(ctx context.Context, hostRepoSlug string, opts TrialOptions) error { + if opts.EngineOverride == "" { + return nil + } - // Ensure the required engine secret is available (prompts interactively if needed) - secretConfig := EngineSecretConfig{ - Ctx: ctx, - RepoSlug: hostRepoSlug, - Engine: opts.EngineOverride, - Verbose: opts.Verbose, - ExistingSecrets: existingSecrets, - IncludeSystemSecrets: false, - IncludeOptional: false, - } - if err := checkAndEnsureEngineSecretsForEngine(secretConfig); err != nil { - return fmt.Errorf("failed to configure engine secret: %w", err) - } + existingSecrets, err := getExistingSecretsInRepo(hostRepoSlug) + if err != nil { + trialLog.Printf("Warning: could not check existing secrets: %v", err) + existingSecrets = make(map[string]struct{}) } - // Set up cleanup if requested - if opts.DeleteHostRepo { - defer func() { - if err := cleanupTrialRepository(hostRepoSlug, opts.Verbose); err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to cleanup host repository: %v", err))) - } - }() + secretConfig := EngineSecretConfig{ + Ctx: ctx, + RepoSlug: hostRepoSlug, + Engine: opts.EngineOverride, + Verbose: opts.Verbose, + ExistingSecrets: existingSecrets, + IncludeSystemSecrets: false, + IncludeOptional: false, } + if err := checkAndEnsureEngineSecretsForEngine(secretConfig); err != nil { + return fmt.Errorf("failed to configure engine secret: %w", err) + } + return nil +} - // Step 2.7: Clone source repository contents if in clone-repo mode - if cloneRepoSlug != "" { - if err := cloneRepoContentsIntoHost(cloneRepoSlug, cloneRepoVersion, hostRepoSlug, opts.Verbose); err != nil { - return fmt.Errorf("failed to clone repository contents: %w", err) +func runWorkflowTrialsSetupCleanup(hostRepoSlug string, opts TrialOptions) func() { + if !opts.DeleteHostRepo { + return nil + } + return func() { + if err := cleanupTrialRepository(hostRepoSlug, opts.Verbose); err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to cleanup host repository: %v", err))) } } +} - // Step 2.8: Disable all workflows except the ones being trialled (only in clone-repo mode, done once before all trials) - if cloneRepoSlug != "" { - // Build list of workflow names to keep enabled - var workflowsToKeep []string - for _, spec := range parsedSpecs { - workflowsToKeep = append(workflowsToKeep, spec.WorkflowName) - } +func runWorkflowTrialsDisableClonedWorkflows(parsedSpecs []*WorkflowSpec, cloneRepoSlug, hostRepoSlug string, opts TrialOptions) error { + if cloneRepoSlug == "" { + return nil + } - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Disabling workflows in cloned repository (keeping: %s)", strings.Join(workflowsToKeep, ", ")))) - } + workflowsToKeep := sliceutil.Map(parsedSpecs, func(spec *WorkflowSpec) string { return spec.WorkflowName }) + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Disabling workflows in cloned repository (keeping: %s)", strings.Join(workflowsToKeep, ", ")))) + } - // Clone host repository temporarily to access workflows - tempDirForDisable, err := cloneTrialHostRepository(hostRepoSlug, opts.Verbose) - if err != nil { - return fmt.Errorf("failed to clone host repository for workflow disabling: %w", err) + tempDirForDisable, err := cloneTrialHostRepository(hostRepoSlug, opts.Verbose) + if err != nil { + return fmt.Errorf("failed to clone host repository for workflow disabling: %w", err) + } + defer func() { + if err := os.RemoveAll(tempDirForDisable); err != nil { + trialLog.Printf("Failed to cleanup temp directory for workflow disabling: %v", err) } - defer func() { - if err := os.RemoveAll(tempDirForDisable); err != nil { - trialLog.Printf("Failed to cleanup temp directory for workflow disabling: %v", err) - } - }() + }() + return runWorkflowTrialsDisableInDir(tempDirForDisable, workflowsToKeep, opts.Verbose) +} - // Change to temp directory to access local .github/workflows - originalDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) +func runWorkflowTrialsDisableInDir(tempDirForDisable string, workflowsToKeep []string, verbose bool) error { + originalDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + if err := os.Chdir(tempDirForDisable); err != nil { + return fmt.Errorf("failed to change to temp directory: %w", err) + } + defer func() { + if err := os.Chdir(originalDir); err != nil { + trialLog.Printf("Failed to change back to original directory: %v", err) } + }() - if err := os.Chdir(tempDirForDisable); err != nil { - return fmt.Errorf("failed to change to temp directory: %w", err) - } - // Always attempt to change back to the original directory - defer func() { - if err := os.Chdir(originalDir); err != nil { - trialLog.Printf("Failed to change back to original directory: %v", err) - } - }() - - // Disable workflows (pass empty string for repoSlug since we're working locally) - disableErr := DisableAllWorkflowsExcept("", workflowsToKeep, opts.Verbose) - // Check for disable errors after changing back - if disableErr != nil { - // Log warning but don't fail the trial - workflow disabling is not critical - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to disable workflows: %v", disableErr))) - } + disableErr := DisableAllWorkflowsExcept("", workflowsToKeep, verbose) + if disableErr != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to disable workflows: %v", disableErr))) } + return nil +} - // Execute trials with optional repeat functionality +func runWorkflowTrialsExecute(ctx context.Context, parsedSpecs []*WorkflowSpec, mode runWorkflowTrialsMode, hostRepoSlug string, opts TrialOptions) error { return ExecuteWithRepeat(RepeatOptions{ RepeatCount: opts.RepeatCount, RepeatMessage: "Repeating trial run", ExecuteFunc: func() error { - return executeTrialRun(ctx, parsedSpecs, hostRepoSlug, logicalRepoSlug, cloneRepoSlug, directTrialMode, opts) + return executeTrialRun(ctx, parsedSpecs, hostRepoSlug, mode.logicalRepoSlug, mode.cloneRepoSlug, mode.directTrialMode, opts) }, CleanupFunc: func() { if opts.DeleteHostRepo { @@ -254,7 +295,6 @@ func RunWorkflowTrials(ctx context.Context, workflowSpecs []string, opts TrialOp }, UseStderr: true, }) - } // getCurrentGitHubUsername gets the current GitHub username from gh CLI diff --git a/pkg/cli/trial_support.go b/pkg/cli/trial_support.go index e906e95d33c..213a9a67459 100644 --- a/pkg/cli/trial_support.go +++ b/pkg/cli/trial_support.go @@ -26,8 +26,6 @@ type TrialArtifacts struct { // downloadAllArtifacts downloads and parses all available artifacts from a workflow run func downloadAllArtifacts(hostRepoSlug, runID string, verbose bool) (*TrialArtifacts, error) { trialSupportLog.Printf("Downloading artifacts: repo=%s, runID=%s", hostRepoSlug, runID) - // Use the repository slug directly (should already be in user/repo format) - repoSlug := hostRepoSlug // Create temp directory for artifact download tempDir, err := os.MkdirTemp("", "trial-artifacts-*") @@ -36,26 +34,39 @@ func downloadAllArtifacts(hostRepoSlug, runID string, verbose bool) (*TrialArtif } defer os.RemoveAll(tempDir) - // Download all artifacts for this run + if ok := downloadAllArtifactsRunDownload(hostRepoSlug, runID, tempDir, verbose); !ok { + return &TrialArtifacts{}, nil + } + + artifacts := &TrialArtifacts{ + AdditionalArtifacts: make(map[string]any), + } + + downloadAllArtifactsWalk(tempDir, artifacts, verbose) + return artifacts, nil +} + +func downloadAllArtifactsRunDownload(hostRepoSlug, runID, tempDir string, verbose bool) bool { + // Use the repository slug directly (should already be in user/repo format) + repoSlug := hostRepoSlug + output, err := workflow.RunGHCombined("Downloading artifacts...", "run", "download", runID, "--repo", repoSlug, "--dir", tempDir) if err != nil { // If no artifacts exist, that's okay - some workflows don't generate artifacts if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("No artifacts found for run %s: %s", runID, string(output)))) } - return &TrialArtifacts{}, nil + return false } if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Downloaded all artifacts for run %s to %s", runID, tempDir))) } + return true +} - artifacts := &TrialArtifacts{ - AdditionalArtifacts: make(map[string]any), - } - - // Walk through all downloaded artifacts - err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { +func downloadAllArtifactsWalk(tempDir string, artifacts *TrialArtifacts, verbose bool) { + err := filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -73,41 +84,7 @@ func downloadAllArtifacts(hostRepoSlug, runID string, verbose bool) (*TrialArtif return nil } - // Handle specific artifact types - switch { - case strings.HasSuffix(path, constants.AgentOutputFilename): - // Parse safe outputs - trialSupportLog.Printf("Processing safe outputs artifact: %s", relPath) - if safeOutputs := parseJSONArtifact(path, verbose); safeOutputs != nil { - artifacts.SafeOutputs = safeOutputs - } - - case strings.HasSuffix(path, "aw_info.json"): - // Parse agentic run information - trialSupportLog.Printf("Processing agentic run info artifact: %s", relPath) - if runInfo := parseJSONArtifact(path, verbose); runInfo != nil { - artifacts.AgenticRunInfo = runInfo - } - - // case strings.Contains(relPath, "agent") && strings.HasSuffix(path, ".log"): - // // Collect agent stdio logs - // if logContent := readTextArtifact(path, verbose); logContent != "" { - // artifacts.AgentStdioLogs = append(artifacts.AgentStdioLogs, logContent) - // } - - case strings.HasSuffix(path, ".json") || strings.HasSuffix(path, ".jsonl") || strings.HasSuffix(path, ".log") || strings.HasSuffix(path, ".txt"): - // Handle other artifacts - if strings.HasSuffix(path, ".json") || strings.HasSuffix(path, ".jsonl") { - if content := parseJSONArtifact(path, verbose); content != nil { - artifacts.AdditionalArtifacts[relPath] = content - } - } else { - if content := readTextArtifact(path, verbose); content != "" { - artifacts.AdditionalArtifacts[relPath] = content - } - } - } - + downloadAllArtifactsProcess(path, relPath, artifacts, verbose) return nil }) @@ -116,8 +93,45 @@ func downloadAllArtifacts(hostRepoSlug, runID string, verbose bool) (*TrialArtif fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Error walking artifact directory: %v", err))) } } +} - return artifacts, nil +func downloadAllArtifactsProcess(path, relPath string, artifacts *TrialArtifacts, verbose bool) { + switch { + case strings.HasSuffix(path, constants.AgentOutputFilename): + // Parse safe outputs + trialSupportLog.Printf("Processing safe outputs artifact: %s", relPath) + if safeOutputs := parseJSONArtifact(path, verbose); safeOutputs != nil { + artifacts.SafeOutputs = safeOutputs + } + + case strings.HasSuffix(path, "aw_info.json"): + // Parse agentic run information + trialSupportLog.Printf("Processing agentic run info artifact: %s", relPath) + if runInfo := parseJSONArtifact(path, verbose); runInfo != nil { + artifacts.AgenticRunInfo = runInfo + } + + // case strings.Contains(relPath, "agent") && strings.HasSuffix(path, ".log"): + // // Collect agent stdio logs + // if logContent := readTextArtifact(path, verbose); logContent != "" { + // artifacts.AgentStdioLogs = append(artifacts.AgentStdioLogs, logContent) + // } + + case strings.HasSuffix(path, ".json") || strings.HasSuffix(path, ".jsonl") || strings.HasSuffix(path, ".log") || strings.HasSuffix(path, ".txt"): + downloadAllArtifactsProcessAdditional(path, relPath, artifacts, verbose) + } +} + +func downloadAllArtifactsProcessAdditional(path, relPath string, artifacts *TrialArtifacts, verbose bool) { + if strings.HasSuffix(path, ".json") || strings.HasSuffix(path, ".jsonl") { + if content := parseJSONArtifact(path, verbose); content != nil { + artifacts.AdditionalArtifacts[relPath] = content + } + return + } + if content := readTextArtifact(path, verbose); content != "" { + artifacts.AdditionalArtifacts[relPath] = content + } } // parseJSONArtifact parses a JSON artifact file and returns the parsed content diff --git a/pkg/cli/update_actions.go b/pkg/cli/update_actions.go index ba50cedfe99..0e879dba4f0 100644 --- a/pkg/cli/update_actions.go +++ b/pkg/cli/update_actions.go @@ -76,6 +76,59 @@ func updateActions(ctx context.Context, deps actionUpdateDeps, allowMajor, verbo fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Checking for GitHub Actions updates...")) } + actionCache, ok, err := updateActionsLoadCache(verbose) + if err != nil || !ok { + return err + } + + updateLog.Printf("Loaded %d action entries from actions-lock.json", len(actionCache.Entries)) + + result := &updateActionsResult{} + for _, s := range updateActionsSnapshot(actionCache) { + if ctx.Err() != nil { + return ctx.Err() + } + updateActionsProcessEntry(updateActionsProcessEntryParams{ + Ctx: ctx, + Deps: deps, + ActionCache: actionCache, + Snapshot: s, + Result: result, + AllowMajor: allowMajor, + Verbose: verbose, + DisableReleaseBump: disableReleaseBump, + CoolDown: coolDown, + }) + } + + updateActionsShowSummary(result, verbose) + return updateActionsSaveCache(actionCache, result.updatedActions) +} + +type updateActionsEntrySnapshot struct { + key string + entry workflow.ActionCacheEntry +} + +type updateActionsResult struct { + updatedActions []string + failedActions []actionUpdateFailure + skippedActions []string +} + +type updateActionsProcessEntryParams struct { + Ctx context.Context + Deps actionUpdateDeps + ActionCache *workflow.ActionCache + Snapshot updateActionsEntrySnapshot + Result *updateActionsResult + AllowMajor bool + Verbose bool + DisableReleaseBump bool + CoolDown time.Duration +} + +func updateActionsLoadCache(verbose bool) (*workflow.ActionCache, bool, error) { // Load the action cache (actions-lock.json) using the shared ActionCache helpers // so that cached inputs/descriptions for safe-outputs.actions entries are preserved. actionsLockPath := filepath.Join(".github", "aw", "actions-lock.json") @@ -83,187 +136,174 @@ func updateActions(ctx context.Context, deps actionUpdateDeps, allowMajor, verbo if verbose { fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Actions lock file not found: "+actionsLockPath)) } - return nil // Not an error, just skip + return nil, false, nil // Not an error, just skip } actionCache := workflow.NewActionCache(".") if err := actionCache.Load(); err != nil { - return fmt.Errorf("failed to parse actions lock file: %w", err) + return nil, false, fmt.Errorf("failed to parse actions lock file: %w", err) } + return actionCache, true, nil +} - updateLog.Printf("Loaded %d action entries from actions-lock.json", len(actionCache.Entries)) - - // Track updates - var updatedActions []string - var failedActions []actionUpdateFailure - var skippedActions []string - - // Snapshot entries before iteration to avoid mutating the map mid-loop. - type entrySnapshot struct { - key string - entry workflow.ActionCacheEntry - } - snapshot := make([]entrySnapshot, 0, len(actionCache.Entries)) +func updateActionsSnapshot(actionCache *workflow.ActionCache) []updateActionsEntrySnapshot { + snapshot := make([]updateActionsEntrySnapshot, 0, len(actionCache.Entries)) for key, entry := range actionCache.Entries { - snapshot = append(snapshot, entrySnapshot{key: key, entry: entry}) + snapshot = append(snapshot, updateActionsEntrySnapshot{key: key, entry: entry}) } + return snapshot +} - for _, s := range snapshot { - if ctx.Err() != nil { - return ctx.Err() +func updateActionsProcessEntry(p updateActionsProcessEntryParams) { + entry := p.Snapshot.entry + updateLog.Printf("Checking action: %s@%s", entry.Repo, entry.Version) + effectiveAllowMajor := !p.DisableReleaseBump || p.AllowMajor || isCoreAction(entry.Repo) + latestVersion, latestSHA, err := p.Deps.getLatestRelease(p.Ctx, entry.Repo, entry.Version, effectiveAllowMajor, p.Verbose) + if err != nil { + if p.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check %s: %v", entry.Repo, err))) } - entry := s.entry - updateLog.Printf("Checking action: %s@%s", entry.Repo, entry.Version) + p.Result.failedActions = append(p.Result.failedActions, actionUpdateFailure{name: entry.Repo, err: err.Error()}) + return + } - // By default all actions are force-updated to the latest major version. - // When disableReleaseBump is set, only core actions (actions/*) bypass the --major flag. - effectiveAllowMajor := !disableReleaseBump || allowMajor || isCoreAction(entry.Repo) + latestVersion, latestSHA, ok := updateActionsCapNative(p.Ctx, p.Deps, entry, latestVersion, latestSHA, p.Result, p.Verbose) + if !ok || updateActionsSkipOlder(entry, latestVersion, p.Result) || updateActionsSkipUnchanged(entry, latestVersion, latestSHA, p.Result, p.Verbose) { + return + } + if updateActionsInCoolDown(p.Ctx, p.ActionCache, entry.Repo, latestVersion, p.CoolDown, p.Result) { + return + } + updateActionsApply(p.ActionCache, p.Snapshot, latestVersion, latestSHA, p.Result) +} - // Check for latest release using the injectable function (also used by updateActionRefsInContent) - latestVersion, latestSHA, err := deps.getLatestRelease(ctx, entry.Repo, entry.Version, effectiveAllowMajor, verbose) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check %s: %v", entry.Repo, err))) - } - failedActions = append(failedActions, actionUpdateFailure{name: entry.Repo, err: err.Error()}) - continue - } +func updateActionsCapNative(ctx context.Context, deps actionUpdateDeps, entry workflow.ActionCacheEntry, latestVersion, latestSHA string, result *updateActionsResult, verbose bool) (string, string, bool) { + if !isGhAwNativeAction(entry.Repo) { + return latestVersion, latestSHA, true + } + cliVersion := GetVersion() + cliVer := parseVersion(cliVersion) + latestVer := parseVersion(latestVersion) + if cliVer == nil || latestVer == nil || !latestVer.IsNewer(cliVer) { + return latestVersion, latestSHA, true + } - // For gh-aw native actions (github/gh-aw/* and github/gh-aw-actions/*), the action - // versions are published in lock-step with the CLI. Never update these actions beyond - // the version of the currently running CLI — doing so would pin a newer (possibly - // pre-release) action that may be incompatible with the user's installed CLI. - if isGhAwNativeAction(entry.Repo) { - cliVersion := GetVersion() - cliVer := parseVersion(cliVersion) - latestVer := parseVersion(latestVersion) - if cliVer != nil && latestVer != nil && latestVer.IsNewer(cliVer) { - cappedVersion := semverutil.EnsureVPrefix(cliVersion) - updateLog.Printf("Capping %s update to CLI version %s (latest available %s exceeds running CLI)", entry.Repo, cappedVersion, latestVersion) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("%s: capping update target to CLI version %s (latest %s is newer than running CLI)", entry.Repo, cappedVersion, latestVersion))) - } - cappedSHA, shaErr := deps.getActionSHAForTag(ctx, gitutil.ExtractBaseRepo(entry.Repo), cappedVersion) - if shaErr != nil { - updateLog.Printf("Cannot resolve SHA for %s@%s (CLI version cap): %v; skipping update", entry.Repo, cappedVersion, shaErr) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: cannot resolve SHA for CLI version %s: %v", entry.Repo, cappedVersion, shaErr))) - failedActions = append(failedActions, actionUpdateFailure{ - name: entry.Repo, - err: fmt.Sprintf("cannot resolve SHA for CLI version %s: %v", cappedVersion, shaErr), - }) - continue - } - latestVersion = cappedVersion - latestSHA = cappedSHA - } - } + cappedVersion := semverutil.EnsureVPrefix(cliVersion) + updateLog.Printf("Capping %s update to CLI version %s (latest available %s exceeds running CLI)", entry.Repo, cappedVersion, latestVersion) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("%s: capping update target to CLI version %s (latest %s is newer than running CLI)", entry.Repo, cappedVersion, latestVersion))) + } + cappedSHA, shaErr := deps.getActionSHAForTag(ctx, gitutil.ExtractBaseRepo(entry.Repo), cappedVersion) + if shaErr != nil { + updateLog.Printf("Cannot resolve SHA for %s@%s (CLI version cap): %v; skipping update", entry.Repo, cappedVersion, shaErr) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: cannot resolve SHA for CLI version %s: %v", entry.Repo, cappedVersion, shaErr))) + result.failedActions = append(result.failedActions, actionUpdateFailure{ + name: entry.Repo, + err: fmt.Sprintf("cannot resolve SHA for CLI version %s: %v", cappedVersion, shaErr), + }) + return latestVersion, latestSHA, false + } + return cappedVersion, cappedSHA, true +} - // Prevent downgrades: if the proposed version is older than the current, skip. - // This can happen when GitHub Releases do not include every tag - // (e.g., v1.1.3 was pushed as a tag-only release without a formal GitHub - // Release, so the Releases API only returns v1.1.0 as the latest). - currentVer := parseVersion(entry.Version) - latestVer := parseVersion(latestVersion) - if currentVer != nil && latestVer != nil && currentVer.IsNewer(latestVer) { - updateLog.Printf("Skipping %s: proposed version %s is older than current %s (would be a downgrade)", entry.Repo, latestVersion, entry.Version) - msg := fmt.Sprintf("%s: skipping proposed update from %s to %s (would be a downgrade)", - entry.Repo, entry.Version, latestVersion) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(msg)) - skippedActions = append(skippedActions, entry.Repo) - continue - } +func updateActionsSkipOlder(entry workflow.ActionCacheEntry, latestVersion string, result *updateActionsResult) bool { + currentVer := parseVersion(entry.Version) + latestVer := parseVersion(latestVersion) + if currentVer == nil || latestVer == nil || !currentVer.IsNewer(latestVer) { + return false + } + updateLog.Printf("Skipping %s: proposed version %s is older than current %s (would be a downgrade)", entry.Repo, latestVersion, entry.Version) + msg := fmt.Sprintf("%s: skipping proposed update from %s to %s (would be a downgrade)", entry.Repo, entry.Version, latestVersion) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(msg)) + result.skippedActions = append(result.skippedActions, entry.Repo) + return true +} - // Check if update is available - if latestVersion == entry.Version && latestSHA == entry.SHA { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("%s@%s is up to date", entry.Repo, entry.Version))) - } - skippedActions = append(skippedActions, entry.Repo) - continue - } +func updateActionsSkipUnchanged(entry workflow.ActionCacheEntry, latestVersion, latestSHA string, result *updateActionsResult, verbose bool) bool { + if latestVersion != entry.Version || latestSHA != entry.SHA { + return false + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("%s@%s is up to date", entry.Repo, entry.Version))) + } + result.skippedActions = append(result.skippedActions, entry.Repo) + return true +} - // Apply cooldown: if the repo is not exempt and the release is too recent, skip. - if !isExemptFromCoolDown(entry.Repo) { - var coolDownResult coolDownCheckResult - if cachedDate, ok := actionCache.GetReleasedAt(entry.Repo, latestVersion); ok { - // Use cached release date to avoid an extra API call. - coolDownResult = checkReleaseCoolDownWithDate(entry.Repo, latestVersion, cachedDate, coolDown) - } else { - // Fetch from API and cache the date for future runs. - coolDownResult = checkReleaseCoolDown(ctx, entry.Repo, latestVersion, coolDown) - if !coolDownResult.PublishedAt.IsZero() { - actionCache.SetReleasedAt(entry.Repo, latestVersion, coolDownResult.PublishedAt) - } - } - if coolDownResult.InCoolDown { - cooldownLog.Printf("Action %s: %s", entry.Repo, coolDownResult.Message) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping update for %s: %s", entry.Repo, coolDownResult.Message))) - skippedActions = append(skippedActions, entry.Repo) - continue - } +func updateActionsInCoolDown(ctx context.Context, actionCache *workflow.ActionCache, repo, latestVersion string, coolDown time.Duration, result *updateActionsResult) bool { + if isExemptFromCoolDown(repo) { + return false + } + var coolDownResult coolDownCheckResult + if cachedDate, ok := actionCache.GetReleasedAt(repo, latestVersion); ok { + coolDownResult = checkReleaseCoolDownWithDate(repo, latestVersion, cachedDate, coolDown) + } else { + coolDownResult = checkReleaseCoolDown(ctx, repo, latestVersion, coolDown) + if !coolDownResult.PublishedAt.IsZero() { + actionCache.SetReleasedAt(repo, latestVersion, coolDownResult.PublishedAt) } + } + if coolDownResult.InCoolDown { + cooldownLog.Printf("Action %s: %s", repo, coolDownResult.Message) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping update for %s: %s", repo, coolDownResult.Message))) + result.skippedActions = append(result.skippedActions, repo) + } + return coolDownResult.InCoolDown +} - // Update the entry using ActionCache.Set which: - // - Preserves cached inputs/descriptions when the SHA is unchanged (moving tag) - // - Clears cached inputs/descriptions when the SHA changes, prompting a re-fetch - // of the updated action.yml on the next compile - oldSHAStr := entry.SHA - if len(oldSHAStr) > 7 { - oldSHAStr = oldSHAStr[:7] - } - newSHAStr := latestSHA - if len(newSHAStr) > 7 { - newSHAStr = newSHAStr[:7] - } - updateLog.Printf("Updating %s from %s (%s) to %s (%s)", entry.Repo, entry.Version, oldSHAStr, latestVersion, newSHAStr) - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Updated %s from %s to %s", entry.Repo, entry.Version, latestVersion))) +func updateActionsApply(actionCache *workflow.ActionCache, s updateActionsEntrySnapshot, latestVersion, latestSHA string, result *updateActionsResult) { + entry := s.entry + oldSHAStr := updateActionsShortSHA(entry.SHA) + newSHAStr := updateActionsShortSHA(latestSHA) + updateLog.Printf("Updating %s from %s (%s) to %s (%s)", entry.Repo, entry.Version, oldSHAStr, latestVersion, newSHAStr) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Updated %s from %s to %s", entry.Repo, entry.Version, latestVersion))) - // Remove the old key when the version changes, using the original map key from - // the snapshot to handle any key/version mismatches in the stored cache file. - if latestVersion != entry.Version { - actionCache.DeleteByKey(s.key) - } - // Set the new entry; ActionCache.Set handles inputs/description preservation. - actionCache.Set(entry.Repo, latestVersion, latestSHA) + if latestVersion != entry.Version { + actionCache.DeleteByKey(s.key) + } + actionCache.Set(entry.Repo, latestVersion, latestSHA) + result.updatedActions = append(result.updatedActions, entry.Repo) +} - updatedActions = append(updatedActions, entry.Repo) +func updateActionsShortSHA(sha string) string { + if len(sha) > 7 { + return sha[:7] } + return sha +} - // Show summary +func updateActionsShowSummary(result *updateActionsResult, verbose bool) { fmt.Fprintln(os.Stderr, "") - - if len(updatedActions) > 0 { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Updated %d action(s):", len(updatedActions)))) - for _, action := range updatedActions { + if len(result.updatedActions) > 0 { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Updated %d action(s):", len(result.updatedActions)))) + for _, action := range result.updatedActions { fmt.Fprintln(os.Stderr, console.FormatListItem(action)) } fmt.Fprintln(os.Stderr, "") } - - if len(skippedActions) > 0 && verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("%d action(s) already up to date", len(skippedActions)))) + if len(result.skippedActions) > 0 && verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("%d action(s) already up to date", len(result.skippedActions)))) fmt.Fprintln(os.Stderr, "") } - - if len(failedActions) > 0 { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check %d action(s):", len(failedActions)))) - for _, f := range failedActions { + if len(result.failedActions) > 0 { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to check %d action(s):", len(result.failedActions)))) + for _, f := range result.failedActions { fmt.Fprintf(os.Stderr, " %s: %s\n", f.name, f.err) } fmt.Fprintln(os.Stderr, "") } +} - // Save the updated actions lock file using ActionCache.Save which preserves - // all entry fields (including inputs/descriptions for safe-outputs actions). - if len(updatedActions) > 0 { - if err := actionCache.Save(); err != nil { - return fmt.Errorf("failed to save actions lock file: %w", err) - } - - updateLog.Printf("Successfully wrote updated actions-lock.json with %d updates", len(updatedActions)) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updated actions-lock.json file")) +func updateActionsSaveCache(actionCache *workflow.ActionCache, updatedActions []string) error { + if len(updatedActions) == 0 { + return nil + } + if err := actionCache.Save(); err != nil { + return fmt.Errorf("failed to save actions lock file: %w", err) } + updateLog.Printf("Successfully wrote updated actions-lock.json with %d updates", len(updatedActions)) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updated actions-lock.json file")) return nil } @@ -280,70 +320,93 @@ func getLatestActionReleaseWithDeps(ctx context.Context, deps actionUpdateDeps, baseRepo := gitutil.ExtractBaseRepo(repo) updateLog.Printf("Using base repository: %s for action: %s", baseRepo, repo) - // Use gh CLI to get releases + releases, err := getLatestActionReleaseWithDepsFetch(ctx, deps, repo, baseRepo, currentVersion, allowMajor, verbose) + if err != nil { + return "", "", err + } + if len(releases) == 1 && strings.Contains(releases[0], "\x00") { + latestRelease, latestSHA, _ := strings.Cut(releases[0], "\x00") + return latestRelease, latestSHA, nil + } + + // Parse current version + currentVer := parseVersion(currentVersion) + validReleases, err := getLatestActionReleaseWithDepsValidReleases(releases) + if err != nil { + return "", "", err + } + + latestCompatible, err := getLatestActionReleaseWithDepsSelect(currentVer, validReleases, allowMajor) + if err != nil { + return "", "", err + } + + sha, err := deps.getActionSHAForTag(ctx, baseRepo, latestCompatible) + if err != nil { + return "", "", fmt.Errorf("failed to get SHA for %s: %w", latestCompatible, err) + } + return latestCompatible, sha, nil +} + +type actionReleaseWithVersion struct { + tag string + version *semverutil.SemanticVersion +} + +func getLatestActionReleaseWithDepsFetch(ctx context.Context, deps actionUpdateDeps, repo, baseRepo, currentVersion string, allowMajor, verbose bool) ([]string, error) { output, err := deps.runGHReleasesAPI(ctx, baseRepo) if err != nil { - // Check if this is an authentication error outputStr := string(output) if gitutil.IsAuthError(outputStr) || gitutil.IsAuthError(err.Error()) { updateLog.Printf("GitHub API authentication failed, attempting git ls-remote fallback for %s", repo) - // Try fallback using git ls-remote latestRelease, latestSHA, gitErr := deps.getLatestReleaseViaGit(ctx, repo, currentVersion, allowMajor, verbose) if gitErr != nil { - return "", "", fmt.Errorf("failed to fetch releases via GitHub API and git: API error: %w, Git Error: %w", err, gitErr) + return nil, fmt.Errorf("failed to fetch releases via GitHub API and git: API error: %w, Git Error: %w", err, gitErr) } - return latestRelease, latestSHA, nil + return []string{latestRelease + "\x00" + latestSHA}, nil } - // Include the gh output in the error for better diagnostics if trimmed := strings.TrimSpace(outputStr); trimmed != "" { - return "", "", fmt.Errorf("failed to fetch releases: %w: %s", err, trimmed) + return nil, fmt.Errorf("failed to fetch releases: %w: %s", err, trimmed) } - return "", "", fmt.Errorf("failed to fetch releases: %w", err) + return nil, fmt.Errorf("failed to fetch releases: %w", err) } releases := strings.Split(strings.TrimSpace(string(output)), "\n") - if len(releases) == 0 || releases[0] == "" { - // No GitHub Releases found; fall back to tag scanning via git ls-remote. - // Some repositories publish tags without creating GitHub Releases — this is safe - // to use and the warning below is informational only. - updateLog.Printf("No releases found via GitHub API for %s, falling back to git ls-remote tag scan", baseRepo) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(baseRepo+": no GitHub Releases found, falling back to tag scanning (safe to ignore)")) - } - latestRelease, latestSHA, gitErr := deps.getLatestReleaseViaGit(ctx, repo, currentVersion, allowMajor, verbose) - if gitErr != nil { - return "", "", fmt.Errorf("no releases or tags found for %s: %w", baseRepo, gitErr) - } - return latestRelease, latestSHA, nil + if len(releases) != 0 && releases[0] != "" { + return releases, nil } + return getLatestActionReleaseWithDepsFallback(ctx, deps, repo, baseRepo, currentVersion, allowMajor, verbose) +} - // Parse current version - currentVer := parseVersion(currentVersion) - - // Find all valid stable semantic version releases (skip prereleases such as v1.0.0-beta.1). - // Per semver rules, v1.1.0-beta.1 > v1.0.0, so without this filter a prerelease of a - // higher base version could be incorrectly selected as the upgrade target. - type releaseWithVersion struct { - tag string - version *semverutil.SemanticVersion +func getLatestActionReleaseWithDepsFallback(ctx context.Context, deps actionUpdateDeps, repo, baseRepo, currentVersion string, allowMajor, verbose bool) ([]string, error) { + updateLog.Printf("No releases found via GitHub API for %s, falling back to git ls-remote tag scan", baseRepo) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(baseRepo+": no GitHub Releases found, falling back to tag scanning (safe to ignore)")) + } + latestRelease, latestSHA, gitErr := deps.getLatestReleaseViaGit(ctx, repo, currentVersion, allowMajor, verbose) + if gitErr != nil { + return nil, fmt.Errorf("no releases or tags found for %s: %w", baseRepo, gitErr) } - var validReleases []releaseWithVersion + return []string{latestRelease + "\x00" + latestSHA}, nil +} + +func getLatestActionReleaseWithDepsValidReleases(releases []string) ([]actionReleaseWithVersion, error) { + var validReleases []actionReleaseWithVersion for _, release := range releases { releaseVer := parseVersion(release) if releaseVer != nil && releaseVer.Pre == "" { - validReleases = append(validReleases, releaseWithVersion{ - tag: release, - version: releaseVer, - }) + validReleases = append(validReleases, actionReleaseWithVersion{tag: release, version: releaseVer}) } } - if len(validReleases) == 0 { - return "", "", errors.New("no valid semantic version releases found") + return nil, errors.New("no valid semantic version releases found") } + getLatestActionReleaseWithDepsSort(validReleases) + return validReleases, nil +} - // Sort releases by semver in descending order (highest first) - slices.SortFunc(validReleases, func(a, b releaseWithVersion) int { +func getLatestActionReleaseWithDepsSort(validReleases []actionReleaseWithVersion) { + slices.SortFunc(validReleases, func(a, b actionReleaseWithVersion) int { switch { case a.version.IsNewer(b.version): return -1 @@ -353,55 +416,34 @@ func getLatestActionReleaseWithDeps(ctx context.Context, deps actionUpdateDeps, return 0 } }) +} - // If current version is not valid, return the highest semver release +func getLatestActionReleaseWithDepsSelect(currentVer *semverutil.SemanticVersion, validReleases []actionReleaseWithVersion, allowMajor bool) (string, error) { if currentVer == nil { - latestRelease := validReleases[0].tag - sha, err := deps.getActionSHAForTag(ctx, baseRepo, latestRelease) - if err != nil { - return "", "", fmt.Errorf("failed to get SHA for %s: %w", latestRelease, err) - } - return latestRelease, sha, nil + return validReleases[0].tag, nil } - - // Find the highest compatible release (respecting major version if !allowMajor) var latestCompatible string var latestCompatibleVersion *semverutil.SemanticVersion - for _, rel := range validReleases { - // Check if compatible based on major version if !allowMajor && rel.version.Major != currentVer.Major { continue } - - // Since releases are sorted by semver descending, first match is highest if latestCompatibleVersion == nil || rel.version.IsNewer(latestCompatibleVersion) { latestCompatible = rel.tag latestCompatibleVersion = rel.version - } else if !rel.version.IsNewer(latestCompatibleVersion) && - rel.version.Major == latestCompatibleVersion.Major && - rel.version.Minor == latestCompatibleVersion.Minor && - rel.version.Patch == latestCompatibleVersion.Patch { - // If versions are equal, prefer the less precise one (e.g., "v8" over "v8.0.0") - // This follows GitHub Actions convention of using major version tags - if !rel.version.IsPreciseVersion() && latestCompatibleVersion.IsPreciseVersion() { - latestCompatible = rel.tag - latestCompatibleVersion = rel.version - } + } else if getLatestActionReleaseWithDepsEqualVersion(rel.version, latestCompatibleVersion) && !rel.version.IsPreciseVersion() && latestCompatibleVersion.IsPreciseVersion() { + latestCompatible = rel.tag + latestCompatibleVersion = rel.version } } - if latestCompatible == "" { - return "", "", errors.New("no compatible release found") - } - - // Get the SHA for the latest compatible release - sha, err := deps.getActionSHAForTag(ctx, baseRepo, latestCompatible) - if err != nil { - return "", "", fmt.Errorf("failed to get SHA for %s: %w", latestCompatible, err) + return "", errors.New("no compatible release found") } + return latestCompatible, nil +} - return latestCompatible, sha, nil +func getLatestActionReleaseWithDepsEqualVersion(a, b *semverutil.SemanticVersion) bool { + return !a.IsNewer(b) && a.Major == b.Major && a.Minor == b.Minor && a.Patch == b.Patch } // getLatestActionReleaseViaGit gets the latest release using git ls-remote (fallback) @@ -425,69 +467,16 @@ func getLatestActionReleaseViaGit(ctx context.Context, repo, currentVersion stri return "", "", fmt.Errorf("failed to fetch releases via git ls-remote: %w", err) } - lines := strings.Split(strings.TrimSpace(string(output)), "\n") - var releases []string - tagToSHA := make(map[string]string) - - for _, line := range lines { - // Parse: " refs/tags/" - parts := strings.Fields(line) - if len(parts) >= 2 { - sha := parts[0] - tagRef := parts[1] - // Skip ^{} annotations (they point to the commit object) - if strings.HasSuffix(tagRef, "^{}") { - continue - } - tag := strings.TrimPrefix(tagRef, "refs/tags/") - releases = append(releases, tag) - tagToSHA[tag] = sha - } + releases, tagToSHA, err := getLatestActionReleaseViaGitParseTags(output) + if err != nil { + return "", "", err } - - if len(releases) == 0 { - return "", "", errors.New("no releases found") + validReleases, err := getLatestActionReleaseViaGitValidReleases(releases) + if err != nil { + return "", "", err } - - // Parse current version currentVer := parseVersion(currentVersion) - // Find all valid stable semantic version releases (skip prereleases such as v1.0.0-beta.1). - // Per semver rules, v1.1.0-beta.1 > v1.0.0, so without this filter a prerelease of a - // higher base version could be incorrectly selected as the upgrade target. - // git ls-remote --tags returns every tag, so the prerelease check is especially important - // for this fallback path. - type releaseWithVersion struct { - tag string - version *semverutil.SemanticVersion - } - var validReleases []releaseWithVersion - for _, release := range releases { - releaseVer := parseVersion(release) - if releaseVer != nil && releaseVer.Pre == "" { - validReleases = append(validReleases, releaseWithVersion{ - tag: release, - version: releaseVer, - }) - } - } - - if len(validReleases) == 0 { - return "", "", errors.New("no valid semantic version releases found") - } - - // Sort releases by semver in descending order (highest first) - slices.SortFunc(validReleases, func(a, b releaseWithVersion) int { - switch { - case a.version.IsNewer(b.version): - return -1 - case b.version.IsNewer(a.version): - return 1 - default: - return 0 - } - }) - // If current version is not valid, return the highest semver release if currentVer == nil { latestRelease := validReleases[0].tag @@ -498,43 +487,65 @@ func getLatestActionReleaseViaGit(ctx context.Context, repo, currentVersion stri return latestRelease, sha, nil } - // Find the highest compatible release (respecting major version if !allowMajor) - var latestCompatible string - var latestCompatibleVersion *semverutil.SemanticVersion + latestCompatible, err := getLatestActionReleaseViaGitSelect(currentVer, validReleases, allowMajor) + if err != nil { + return "", "", err + } + sha := tagToSHA[latestCompatible] + if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Latest compatible release: %s (via git)", latestCompatible))) + } - for _, rel := range validReleases { - // Check if compatible based on major version - if !allowMajor && rel.version.Major != currentVer.Major { + return latestCompatible, sha, nil +} + +func getLatestActionReleaseViaGitParseTags(output []byte) ([]string, map[string]string, error) { + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + var releases []string + tagToSHA := make(map[string]string) + for _, line := range lines { + // Parse: " refs/tags/" + parts := strings.Fields(line) + if len(parts) < 2 { continue } - - // Since releases are sorted by semver descending, first match is highest - if latestCompatibleVersion == nil || rel.version.IsNewer(latestCompatibleVersion) { - latestCompatible = rel.tag - latestCompatibleVersion = rel.version - } else if !rel.version.IsNewer(latestCompatibleVersion) && - rel.version.Major == latestCompatibleVersion.Major && - rel.version.Minor == latestCompatibleVersion.Minor && - rel.version.Patch == latestCompatibleVersion.Patch { - // If versions are equal, prefer the less precise one (e.g., "v8" over "v8.0.0") - // This follows GitHub Actions convention of using major version tags - if !rel.version.IsPreciseVersion() && latestCompatibleVersion.IsPreciseVersion() { - latestCompatible = rel.tag - latestCompatibleVersion = rel.version - } + sha := parts[0] + tagRef := parts[1] + // Skip ^{} annotations (they point to the commit object) + if strings.HasSuffix(tagRef, "^{}") { + continue } + tag := strings.TrimPrefix(tagRef, "refs/tags/") + releases = append(releases, tag) + tagToSHA[tag] = sha } - - if latestCompatible == "" { - return "", "", errors.New("no compatible release found") + if len(releases) == 0 { + return nil, nil, errors.New("no releases found") } + return releases, tagToSHA, nil +} - sha := tagToSHA[latestCompatible] - if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Latest compatible release: %s (via git)", latestCompatible))) +func getLatestActionReleaseViaGitValidReleases(releases []string) ([]actionReleaseWithVersion, error) { + var validReleases []actionReleaseWithVersion + for _, release := range releases { + releaseVer := parseVersion(release) + if releaseVer != nil && releaseVer.Pre == "" { + validReleases = append(validReleases, actionReleaseWithVersion{tag: release, version: releaseVer}) + } } + if len(validReleases) == 0 { + return nil, errors.New("no valid semantic version releases found") + } + getLatestActionReleaseWithDepsSort(validReleases) + return validReleases, nil +} - return latestCompatible, sha, nil +func getLatestActionReleaseViaGitSelect(currentVer *semverutil.SemanticVersion, validReleases []actionReleaseWithVersion, allowMajor bool) (string, error) { + latestCompatible, err := getLatestActionReleaseWithDepsSelect(currentVer, validReleases, allowMajor) + if err != nil { + return "", err + } + return latestCompatible, nil } // getActionSHAForTag gets the commit SHA for a given tag in an action repository. @@ -641,71 +652,104 @@ func updateActionsInWorkflowFiles(ctx context.Context, deps actionUpdateDeps, op coolDownCache := make(map[string]coolDownCheckResult) var updatedFiles []string - err := filepath.WalkDir(opts.workflowsDir, func(path string, d os.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr - } - if ctx.Err() != nil { - return ctx.Err() - } - if d.IsDir() || !strings.HasSuffix(d.Name(), ".md") { - return nil - } + return updateActionsInWorkflowFilesVisit(updateActionsInWorkflowFilesVisitParams{ + Ctx: ctx, + Deps: deps, + Opts: opts, + Cache: cache, + CoolDownCache: coolDownCache, + UpdatedFiles: &updatedFiles, + Path: path, + DirEntry: d, + WalkErr: walkErr, + }) + }) + if err != nil { + return fmt.Errorf("failed to walk workflows directory: %w", err) + } - content, err := os.ReadFile(path) - if err != nil { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read %s: %v", path, err))) - } - return nil - } + if len(updatedFiles) == 0 && opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No action references needed updating in workflow files")) + } - updatedActions, newContent, err := updateActionRefsInContentWithDeps(ctx, deps, string(content), cache, coolDownCache, !opts.disableReleaseBump, opts.verbose, opts.coolDown) - if err != nil { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to update action refs in %s: %v", path, err))) - } - return nil - } - updatedSkills, newContent, err := updateSkillRefsInContent(ctx, newContent, !opts.disableReleaseBump, opts.verbose, opts.coolDown) - if err != nil { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to update skill refs in %s: %v", path, err))) - } - return nil - } + return nil +} - if !updatedActions && !updatedSkills { - return nil - } +type updateActionsInWorkflowFilesVisitParams struct { + Ctx context.Context + Deps actionUpdateDeps + Opts updateActionsOptions + Cache map[string]latestReleaseResult + CoolDownCache map[string]coolDownCheckResult + UpdatedFiles *[]string + Path string + DirEntry os.DirEntry + WalkErr error +} - if err := os.WriteFile(path, []byte(newContent), constants.FilePermPublic); err != nil { - return fmt.Errorf("failed to write updated workflow %s: %w", path, err) - } +func updateActionsInWorkflowFilesVisit(p updateActionsInWorkflowFilesVisitParams) error { + if p.WalkErr != nil { + return p.WalkErr + } + if p.Ctx.Err() != nil { + return p.Ctx.Err() + } + if p.DirEntry.IsDir() || !strings.HasSuffix(p.DirEntry.Name(), ".md") { + return nil + } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Updated action/skill references in "+d.Name())) - updatedFiles = append(updatedFiles, path) + newContent, updated, err := updateActionsInWorkflowFilesUpdateContent(p.Ctx, p.Deps, p.Opts, p.Cache, p.CoolDownCache, p.Path) + if err != nil || !updated { + return err + } - // Recompile the updated workflow (unless --no-compile is set) - if !opts.noCompile { - if err := compileWorkflowWithRefresh(ctx, path, opts.verbose, false, opts.engineOverride, false, opts.approve); err != nil { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to recompile %s: %v", path, err))) - } - } - } - return nil - }) + if err := os.WriteFile(p.Path, []byte(newContent), constants.FilePermPublic); err != nil { + return fmt.Errorf("failed to write updated workflow %s: %w", p.Path, err) + } + + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Updated action/skill references in "+p.DirEntry.Name())) + *p.UpdatedFiles = append(*p.UpdatedFiles, p.Path) + updateActionsInWorkflowFilesCompile(p.Ctx, p.Opts, p.Path) + return nil +} + +func updateActionsInWorkflowFilesUpdateContent(ctx context.Context, deps actionUpdateDeps, opts updateActionsOptions, cache map[string]latestReleaseResult, coolDownCache map[string]coolDownCheckResult, path string) (string, bool, error) { + content, err := os.ReadFile(path) if err != nil { - return fmt.Errorf("failed to walk workflows directory: %w", err) + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read %s: %v", path, err))) + } + return "", false, nil } - if len(updatedFiles) == 0 && opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("No action references needed updating in workflow files")) + updatedActions, newContent, err := updateActionRefsInContentWithDeps(ctx, deps, string(content), cache, coolDownCache, !opts.disableReleaseBump, opts.verbose, opts.coolDown) + if err != nil { + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to update action refs in %s: %v", path, err))) + } + return "", false, nil + } + updatedSkills, newContent, err := updateSkillRefsInContent(ctx, newContent, !opts.disableReleaseBump, opts.verbose, opts.coolDown) + if err != nil { + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to update skill refs in %s: %v", path, err))) + } + return "", false, nil } + return newContent, updatedActions || updatedSkills, nil +} - return nil +func updateActionsInWorkflowFilesCompile(ctx context.Context, opts updateActionsOptions, path string) { + // Recompile the updated workflow (unless --no-compile is set) + if opts.noCompile { + return + } + if err := compileWorkflowWithRefresh(ctx, path, opts.verbose, false, opts.engineOverride, false, opts.approve); err != nil { + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to recompile %s: %v", path, err))) + } + } } type skillRefUpdateResolver func(ctx context.Context, repo, currentRef string, allowMajor, verbose bool, coolDown time.Duration) (string, error) @@ -820,111 +864,155 @@ func updateActionRefsInContentWithDeps(ctx context.Context, deps actionUpdateDep lines := strings.Split(content, "\n") for i, line := range lines { - match := actionRefPattern.FindStringSubmatchIndex(line) - if match == nil { - continue + newLine, lineChanged := updateActionRefsInContentWithDepsLine(updateActionRefsInContentWithDepsLineParams{ + Ctx: ctx, + Deps: deps, + Line: line, + Index: i, + Cache: cache, + CoolDownCache: coolDownCache, + AllowMajor: allowMajor, + Verbose: verbose, + CoolDown: coolDown, + }) + if lineChanged { + lines[i] = newLine + changed = true } + } - // Extract matched groups - prefix := line[match[2]:match[3]] // "uses: " - repo := line[match[4]:match[5]] // e.g. "actions/checkout" - ref := line[match[6]:match[7]] // SHA or version tag - comment := "" - if match[8] >= 0 { - comment = line[match[8]:match[9]] // e.g. " # v6.0.2" - } - trailing := "" - if match[10] >= 0 { - trailing = line[match[10]:match[11]] - } + return changed, strings.Join(lines, "\n"), nil +} - // When release bumps are disabled, skip non-core (non actions/*) action refs. - effectiveAllowMajor := allowMajor || isCoreAction(repo) - if !effectiveAllowMajor { - continue - } +type updateActionRefsInContentWithDepsMatch struct { + prefix string + repo string + ref string + comment string + trailing string + start string +} - // Determine the "current version" to pass to the latest-release resolver. - isSHA := IsCommitSHA(ref) - currentVersion := ref - if isSHA { - // Extract version from comment (e.g., " # v6.0.2" -> "v6.0.2") - if comment != "" { - commentVersion := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(comment), "#")) - if commentVersion != "" { - currentVersion = commentVersion - } else { - currentVersion = "" - } - } else { - currentVersion = "" - } - } +type updateActionRefsInContentWithDepsLineParams struct { + Ctx context.Context + Deps actionUpdateDeps + Line string + Index int + Cache map[string]latestReleaseResult + CoolDownCache map[string]coolDownCheckResult + AllowMajor bool + Verbose bool + CoolDown time.Duration +} - // Resolve latest version/SHA, using the cache to avoid redundant API calls. - // Use "|" as separator since GitHub repo names cannot contain "|". - cacheKey := repo + "|" + currentVersion - result, cached := cache[cacheKey] - if !cached { - latestVersion, latestSHA, err := deps.getLatestRelease(ctx, repo, currentVersion, effectiveAllowMajor, verbose) - if err != nil { - updateLog.Printf("Failed to get latest release for %s: %v", repo, err) - continue - } - result = latestReleaseResult{version: latestVersion, sha: latestSHA} - cache[cacheKey] = result - } - latestVersion := result.version - latestSHA := result.sha +func updateActionRefsInContentWithDepsLine(p updateActionRefsInContentWithDepsLineParams) (string, bool) { + match := actionRefPattern.FindStringSubmatchIndex(p.Line) + if match == nil { + return p.Line, false + } + refMatch := updateActionRefsInContentWithDepsParseMatch(p.Line, match) + effectiveAllowMajor := p.AllowMajor || isCoreAction(refMatch.repo) + if !effectiveAllowMajor { + return p.Line, false + } - if isSHA { - if latestSHA == ref { - continue // SHA unchanged - } - } else { - if latestVersion == ref { - continue // Version tag unchanged - } - // Prevent downgrades: if the proposed version is older than the current, skip. - currentVer := parseVersion(ref) - proposedVer := parseVersion(latestVersion) - if currentVer != nil && proposedVer != nil && currentVer.IsNewer(proposedVer) { - updateLog.Printf("Skipping %s in workflow file: proposed version %s is older than current %s (would be a downgrade)", repo, latestVersion, ref) - continue - } - } + isSHA := IsCommitSHA(refMatch.ref) + currentVersion := updateActionRefsInContentWithDepsCurrentVersion(refMatch.ref, refMatch.comment, isSHA) + result, ok := updateActionRefsInContentWithDepsResolve(p.Ctx, p.Deps, refMatch.repo, currentVersion, effectiveAllowMajor, p.Verbose, p.Cache) + if !ok || updateActionRefsInContentWithDepsUnchangedOrDowngrade(refMatch, result, isSHA) { + return p.Line, false + } + if updateActionRefsInContentWithDepsInCooldown(p.Ctx, refMatch.repo, result.version, p.CoolDownCache, p.Verbose, p.CoolDown) { + return p.Line, false + } - // Apply cooldown: if the repo is not exempt and the release is too recent, skip. - if !isExemptFromCoolDown(repo) { - coolDownKey := repo + "@" + latestVersion - coolDownResult, coolDownCached := coolDownCache[coolDownKey] - if !coolDownCached { - coolDownResult = checkReleaseCoolDown(ctx, repo, latestVersion, coolDown) - coolDownCache[coolDownKey] = coolDownResult - } - if coolDownResult.InCoolDown { - cooldownLog.Printf("Action ref %s in workflow: %s", repo, coolDownResult.Message) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping update for %s: %s", repo, coolDownResult.Message))) - } - continue - } - } + updateLog.Printf("Updating %s from %s to %s in line %d", refMatch.repo, refMatch.ref, result.version, p.Index+1) + return updateActionRefsInContentWithDepsBuildLine(refMatch, result, isSHA), true +} - // Build the new uses line - var newRef string - if isSHA { - // SHA-pinned references stay SHA-pinned, updated to latest SHA + version comment - newRef = fmt.Sprintf("%s%s%s@%s # %s%s", line[:match[2]], prefix, repo, latestSHA, latestVersion, trailing) - } else { - // Version tag references just get the new version tag - newRef = fmt.Sprintf("%s%s%s@%s%s%s", line[:match[2]], prefix, repo, latestVersion, comment, trailing) - } +func updateActionRefsInContentWithDepsParseMatch(line string, match []int) updateActionRefsInContentWithDepsMatch { + refMatch := updateActionRefsInContentWithDepsMatch{ + start: line[:match[2]], + prefix: line[match[2]:match[3]], + repo: line[match[4]:match[5]], + ref: line[match[6]:match[7]], + } + if match[8] >= 0 { + refMatch.comment = line[match[8]:match[9]] + } + if match[10] >= 0 { + refMatch.trailing = line[match[10]:match[11]] + } + return refMatch +} - updateLog.Printf("Updating %s from %s to %s in line %d", repo, ref, latestVersion, i+1) - lines[i] = newRef - changed = true +func updateActionRefsInContentWithDepsCurrentVersion(ref, comment string, isSHA bool) string { + if !isSHA { + return ref } + if comment == "" { + return "" + } + commentVersion := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(comment), "#")) + return commentVersion +} - return changed, strings.Join(lines, "\n"), nil +func updateActionRefsInContentWithDepsResolve(ctx context.Context, deps actionUpdateDeps, repo, currentVersion string, allowMajor, verbose bool, cache map[string]latestReleaseResult) (latestReleaseResult, bool) { + cacheKey := repo + "|" + currentVersion + result, cached := cache[cacheKey] + if cached { + return result, true + } + latestVersion, latestSHA, err := deps.getLatestRelease(ctx, repo, currentVersion, allowMajor, verbose) + if err != nil { + updateLog.Printf("Failed to get latest release for %s: %v", repo, err) + return latestReleaseResult{}, false + } + result = latestReleaseResult{version: latestVersion, sha: latestSHA} + cache[cacheKey] = result + return result, true +} + +func updateActionRefsInContentWithDepsUnchangedOrDowngrade(refMatch updateActionRefsInContentWithDepsMatch, result latestReleaseResult, isSHA bool) bool { + if isSHA { + return result.sha == refMatch.ref + } + if result.version == refMatch.ref { + return true + } + currentVer := parseVersion(refMatch.ref) + proposedVer := parseVersion(result.version) + if currentVer != nil && proposedVer != nil && currentVer.IsNewer(proposedVer) { + updateLog.Printf("Skipping %s in workflow file: proposed version %s is older than current %s (would be a downgrade)", refMatch.repo, result.version, refMatch.ref) + return true + } + return false +} + +func updateActionRefsInContentWithDepsInCooldown(ctx context.Context, repo, latestVersion string, coolDownCache map[string]coolDownCheckResult, verbose bool, coolDown time.Duration) bool { + if isExemptFromCoolDown(repo) { + return false + } + coolDownKey := repo + "@" + latestVersion + coolDownResult, coolDownCached := coolDownCache[coolDownKey] + if !coolDownCached { + coolDownResult = checkReleaseCoolDown(ctx, repo, latestVersion, coolDown) + coolDownCache[coolDownKey] = coolDownResult + } + if coolDownResult.InCoolDown { + cooldownLog.Printf("Action ref %s in workflow: %s", repo, coolDownResult.Message) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping update for %s: %s", repo, coolDownResult.Message))) + } + } + return coolDownResult.InCoolDown +} + +func updateActionRefsInContentWithDepsBuildLine(refMatch updateActionRefsInContentWithDepsMatch, result latestReleaseResult, isSHA bool) string { + if isSHA { + // SHA-pinned references stay SHA-pinned, updated to latest SHA + version comment + return fmt.Sprintf("%s%s%s@%s # %s%s", refMatch.start, refMatch.prefix, refMatch.repo, result.sha, result.version, refMatch.trailing) + } + // Version tag references just get the new version tag + return fmt.Sprintf("%s%s%s@%s%s%s", refMatch.start, refMatch.prefix, refMatch.repo, result.version, refMatch.comment, refMatch.trailing) } diff --git a/pkg/cli/update_command.go b/pkg/cli/update_command.go index 626b74e5528..c0e6547efe7 100644 --- a/pkg/cli/update_command.go +++ b/pkg/cli/update_command.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/github/gh-aw/pkg/console" "github.com/github/gh-aw/pkg/constants" @@ -22,6 +23,13 @@ const updateTargetRepoCheckoutDir = ".github/aw/updates" // NewUpdateCommand creates the update command func NewUpdateCommand(validateEngine func(string) error) *cobra.Command { + cmd := newUpdateCommandBase(validateEngine) + newUpdateCommandFlags(cmd) + newUpdateCommandCompletions(cmd) + return cmd +} + +func newUpdateCommandBase(validateEngine func(string) error) *cobra.Command { cmd := &cobra.Command{ Use: "update [workflow]...", Short: "Update agentic workflows from their source repositories", @@ -67,101 +75,13 @@ Note: In GitHub Enterprise repos, shorthand source specs resolve on your enterpr ` + string(constants.CLIExtensionPrefix) + ` update --cool-down 0 # Disable cooldown and apply all pending releases immediately ` + string(constants.CLIExtensionPrefix) + ` update --cool-down 3d # Apply a custom 3-day cooldown period`, RunE: func(cmd *cobra.Command, args []string) error { - majorFlag, _ := cmd.Flags().GetBool("major") - forceFlag, _ := cmd.Flags().GetBool("force") - engineOverride, _ := cmd.Flags().GetString("engine") - verbose, _ := cmd.Flags().GetBool("verbose") - workflowDir, _ := cmd.Flags().GetString("dir") - noStopAfter, _ := cmd.Flags().GetBool("no-stop-after") - stopAfter, _ := cmd.Flags().GetString("stop-after") - noMergeFlag, _ := cmd.Flags().GetBool("no-merge") - disableReleaseBump, _ := cmd.Flags().GetBool("no-release-bump") - disableReleaseBumpLegacy, _ := cmd.Flags().GetBool("disable-release-bump") - disableReleaseBump = disableReleaseBump || disableReleaseBumpLegacy - noCompile, _ := cmd.Flags().GetBool("no-compile") - noRedirect, _ := cmd.Flags().GetBool("no-redirect") - disableSecurityScanner, _ := cmd.Flags().GetBool("no-security-scanner") - disableSecurityScannerLegacy, _ := cmd.Flags().GetBool("disable-security-scanner") - disableSecurityScanner = disableSecurityScanner || disableSecurityScannerLegacy - approveFlag, _ := cmd.Flags().GetBool("approve") - createPRFlag, _ := cmd.Flags().GetBool("create-pull-request") - prFlagAlias, _ := cmd.Flags().GetBool("pr") - createPR := createPRFlag || prFlagAlias - createIssue, _ := cmd.Flags().GetBool("create-issue") - yes, _ := cmd.Flags().GetBool("yes") - coolDownStr, _ := cmd.Flags().GetString("cool-down") - targetRepo, _ := cmd.Flags().GetString("repo") - targetOrg, _ := cmd.Flags().GetString("org") - repoGlobs, _ := cmd.Flags().GetStringSlice("repos") - - if err := validateEngine(engineOverride); err != nil { - return err - } - - coolDown, err := parseCoolDownFlag(coolDownStr) - if err != nil { - return fmt.Errorf("invalid --cool-down value: %w", err) - } - - if targetRepo != "" && targetOrg != "" { - return errors.New("cannot specify both --repo and --org flags; use --repo for a single repository or --org for organization-wide updates") - } - - if createIssue && targetOrg == "" { - return errors.New("--create-issue requires --org to be specified") - } - - if createPR && createIssue { - return errors.New("cannot specify both --create-pull-request and --create-issue") - } - - if createPR && targetRepo == "" && targetOrg == "" { - if err := PreflightCheckForCreatePR(verbose); err != nil { - return err - } - } - - opts := UpdateWorkflowsOptions{ - WorkflowNames: args, - AllowMajor: majorFlag, - Force: forceFlag, - Yes: yes, - Verbose: verbose, - EngineOverride: engineOverride, - WorkflowsDir: workflowDir, - NoStopAfter: noStopAfter, - StopAfter: stopAfter, - NoMerge: noMergeFlag, - DisableReleaseBump: disableReleaseBump, - NoCompile: noCompile, - NoRedirect: noRedirect, - DisableSecurityScanner: disableSecurityScanner, - CoolDown: coolDown, - Approve: approveFlag, - } - - if targetRepo != "" { - return runUpdateForTargetRepo(cmd.Context(), targetRepo, opts, createPR, verbose) - } - - if targetOrg != "" { - return runUpdateForOrg(cmd.Context(), targetOrg, repoGlobs, opts, createPR, createIssue, verbose) - } - - if err := RunUpdateWorkflows(cmd.Context(), opts); err != nil { - return err - } - - if createPR { - prBody := "This PR updates agentic workflows from their source repositories." - _, err := CreatePRWithChanges("update-workflows", "chore: update workflows", - "Update workflows from source", prBody, verbose) - return err - } - return nil + return runUpdateCommand(cmd, args, validateEngine) }, } + return cmd +} +func newUpdateCommandFlags(cmd *cobra.Command) { cmd.Flags().Bool("major", false, "Allow major version updates when updating tagged releases") cmd.Flags().BoolP("force", "f", false, "Force update even if no changes are detected") addEngineFlag(cmd) @@ -187,13 +107,140 @@ Note: In GitHub Enterprise repos, shorthand source specs resolve on your enterpr cmd.Flags().BoolP("yes", "y", false, "Auto-accept org-mode update confirmations (required in CI)") cmd.Flags().String("cool-down", "7d", coolDownFlagUsage) _ = cmd.Flags().MarkHidden("pr") // Hide the short alias from help output +} +func newUpdateCommandCompletions(cmd *cobra.Command) { // Register completions for update command cmd.ValidArgsFunction = CompleteWorkflowNames RegisterEngineFlagCompletion(cmd) RegisterDirFlagCompletion(cmd, "dir") +} - return cmd +type runUpdateCommandFlags struct { + majorFlag bool + forceFlag bool + engineOverride string + verbose bool + workflowDir string + noStopAfter bool + stopAfter string + noMergeFlag bool + disableReleaseBump bool + noCompile bool + noRedirect bool + disableSecurityScanner bool + approveFlag bool + createPR bool + createIssue bool + yes bool + coolDownStr string + targetRepo string + targetOrg string + repoGlobs []string +} + +func runUpdateCommand(cmd *cobra.Command, args []string, validateEngine func(string) error) error { + flags := runUpdateCommandReadFlags(cmd) + if err := runUpdateCommandValidate(flags, validateEngine); err != nil { + return err + } + coolDown, err := parseCoolDownFlag(flags.coolDownStr) + if err != nil { + return fmt.Errorf("invalid --cool-down value: %w", err) + } + + opts := runUpdateCommandOptions(args, flags, coolDown) + if flags.targetRepo != "" { + return runUpdateForTargetRepo(cmd.Context(), flags.targetRepo, opts, flags.createPR, flags.verbose) + } + if flags.targetOrg != "" { + return runUpdateForOrg(cmd.Context(), flags.targetOrg, flags.repoGlobs, opts, flags.createPR, flags.createIssue, flags.verbose) + } + return runUpdateCommandLocal(cmd.Context(), opts, flags.createPR, flags.verbose) +} + +func runUpdateCommandReadFlags(cmd *cobra.Command) runUpdateCommandFlags { + flags := runUpdateCommandFlags{} + flags.majorFlag, _ = cmd.Flags().GetBool("major") + flags.forceFlag, _ = cmd.Flags().GetBool("force") + flags.engineOverride, _ = cmd.Flags().GetString("engine") + flags.verbose, _ = cmd.Flags().GetBool("verbose") + flags.workflowDir, _ = cmd.Flags().GetString("dir") + flags.noStopAfter, _ = cmd.Flags().GetBool("no-stop-after") + flags.stopAfter, _ = cmd.Flags().GetString("stop-after") + flags.noMergeFlag, _ = cmd.Flags().GetBool("no-merge") + flags.disableReleaseBump, _ = cmd.Flags().GetBool("no-release-bump") + disableReleaseBumpLegacy, _ := cmd.Flags().GetBool("disable-release-bump") + flags.disableReleaseBump = flags.disableReleaseBump || disableReleaseBumpLegacy + flags.noCompile, _ = cmd.Flags().GetBool("no-compile") + flags.noRedirect, _ = cmd.Flags().GetBool("no-redirect") + flags.disableSecurityScanner, _ = cmd.Flags().GetBool("no-security-scanner") + disableSecurityScannerLegacy, _ := cmd.Flags().GetBool("disable-security-scanner") + flags.disableSecurityScanner = flags.disableSecurityScanner || disableSecurityScannerLegacy + flags.approveFlag, _ = cmd.Flags().GetBool("approve") + createPRFlag, _ := cmd.Flags().GetBool("create-pull-request") + prFlagAlias, _ := cmd.Flags().GetBool("pr") + flags.createPR = createPRFlag || prFlagAlias + flags.createIssue, _ = cmd.Flags().GetBool("create-issue") + flags.yes, _ = cmd.Flags().GetBool("yes") + flags.coolDownStr, _ = cmd.Flags().GetString("cool-down") + flags.targetRepo, _ = cmd.Flags().GetString("repo") + flags.targetOrg, _ = cmd.Flags().GetString("org") + flags.repoGlobs, _ = cmd.Flags().GetStringSlice("repos") + return flags +} + +func runUpdateCommandValidate(flags runUpdateCommandFlags, validateEngine func(string) error) error { + if err := validateEngine(flags.engineOverride); err != nil { + return err + } + if flags.targetRepo != "" && flags.targetOrg != "" { + return errors.New("cannot specify both --repo and --org flags; use --repo for a single repository or --org for organization-wide updates") + } + if flags.createIssue && flags.targetOrg == "" { + return errors.New("--create-issue requires --org to be specified") + } + if flags.createPR && flags.createIssue { + return errors.New("cannot specify both --create-pull-request and --create-issue") + } + if flags.createPR && flags.targetRepo == "" && flags.targetOrg == "" { + return PreflightCheckForCreatePR(flags.verbose) + } + return nil +} + +func runUpdateCommandOptions(args []string, flags runUpdateCommandFlags, coolDown time.Duration) UpdateWorkflowsOptions { + return UpdateWorkflowsOptions{ + WorkflowNames: args, + AllowMajor: flags.majorFlag, + Force: flags.forceFlag, + Yes: flags.yes, + Verbose: flags.verbose, + EngineOverride: flags.engineOverride, + WorkflowsDir: flags.workflowDir, + NoStopAfter: flags.noStopAfter, + StopAfter: flags.stopAfter, + NoMerge: flags.noMergeFlag, + DisableReleaseBump: flags.disableReleaseBump, + NoCompile: flags.noCompile, + NoRedirect: flags.noRedirect, + DisableSecurityScanner: flags.disableSecurityScanner, + CoolDown: coolDown, + Approve: flags.approveFlag, + } +} + +func runUpdateCommandLocal(ctx context.Context, opts UpdateWorkflowsOptions, createPR bool, verbose bool) error { + if err := RunUpdateWorkflows(ctx, opts); err != nil { + return err + } + if createPR { + prBody := "This PR updates agentic workflows from their source repositories." + _, err := CreatePRWithChanges("update-workflows", "chore: update workflows", + "Update workflows from source", prBody, verbose) + return err + } + return nil } // RunUpdateWorkflows updates workflows from their source repositories. @@ -278,70 +325,86 @@ func recompileAllWorkflows(ctx context.Context, workflowsDir, engineOverride str } func runUpdateForTargetRepo(ctx context.Context, targetRepo string, opts UpdateWorkflowsOptions, createPR bool, verbose bool) error { + checkoutDir, err := runUpdateForTargetRepoCheckout(ctx, targetRepo, verbose) + if err != nil { + return err + } + + restoreDir, err := runUpdateForTargetRepoChdir(checkoutDir) + if err != nil { + return err + } + defer restoreDir() + + if createPR { + if err := PreflightCheckForCreatePR(verbose); err != nil { + return err + } + } + + if err := RunUpdateWorkflows(ctx, opts); err != nil { + return err + } + + if createPR { + return runUpdateForTargetRepoCreatePR(ctx, targetRepo, verbose) + } + return nil +} + +func runUpdateForTargetRepoCheckout(ctx context.Context, targetRepo string, verbose bool) (string, error) { gitRoot, err := gitutil.FindGitRoot() if err != nil { - return fmt.Errorf("--repo requires running inside a git repository: %w", err) + return "", fmt.Errorf("--repo requires running inside a git repository: %w", err) } updatesDir, err := ensureUpdateTargetRepoGitignore(gitRoot) if err != nil { - return err + return "", err } checkoutDir := filepath.Join(updatesDir, sanitizeRepoPath(targetRepo)) if err := shallowCloneTargetRepo(ctx, targetRepo, checkoutDir); err != nil { - return err + return "", err } - if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Checked out "+targetRepo+" at "+checkoutDir)) } + return checkoutDir, nil +} +func runUpdateForTargetRepoChdir(checkoutDir string) (func(), error) { originalDir, err := os.Getwd() if err != nil { - return fmt.Errorf("failed to read current directory: %w", err) + return nil, fmt.Errorf("failed to read current directory: %w", err) } - defer func() { - _ = os.Chdir(originalDir) - }() - if err := os.Chdir(checkoutDir); err != nil { - return fmt.Errorf("failed to change directory to checkout %s: %w", checkoutDir, err) + return nil, fmt.Errorf("failed to change directory to checkout %s: %w", checkoutDir, err) } + return func() { + _ = os.Chdir(originalDir) + }, nil +} - if createPR { - if err := PreflightCheckForCreatePR(verbose); err != nil { - return err - } +func runUpdateForTargetRepoCreatePR(ctx context.Context, targetRepo string, verbose bool) error { + releaseTag, releaseURL := getGhawReleaseInfo() + xmlMarker := buildOrgXMLMarker(ghawUpdateMarkerPrefix, releaseTag) + closeExistingOrgPRsByMarker(ctx, targetRepo, ghawUpdateMarkerPrefix, verbose) + + var releaseLine string + if releaseURL != "" { + releaseLine = fmt.Sprintf("\n[View gh-aw release %s](%s)\n", releaseTag, releaseURL) } + prBody := "This PR updates agentic workflows from their source repositories." + + releaseLine + "\n" + xmlMarker - if err := RunUpdateWorkflows(ctx, opts); err != nil { + prURL, err := CreatePRWithChanges("update-workflows", "chore: update workflows", + "Update workflows from source", prBody, verbose) + if err != nil { return err } - - if createPR { - releaseTag, releaseURL := getGhawReleaseInfo() - xmlMarker := buildOrgXMLMarker(ghawUpdateMarkerPrefix, releaseTag) - - // Close any stale update PRs in the target repo before creating the new one. - closeExistingOrgPRsByMarker(ctx, targetRepo, ghawUpdateMarkerPrefix, verbose) - - var releaseLine string - if releaseURL != "" { - releaseLine = fmt.Sprintf("\n[View gh-aw release %s](%s)\n", releaseTag, releaseURL) - } - prBody := "This PR updates agentic workflows from their source repositories." + - releaseLine + "\n" + xmlMarker - - prURL, err := CreatePRWithChanges("update-workflows", "chore: update workflows", - "Update workflows from source", prBody, verbose) - if err != nil { - return err - } - if prURL != "" { - addLabelToOrgPR(ctx, prURL, agenticWorkflowsLabel, verbose) - } - return nil + if prURL != "" { + addLabelToOrgPR(ctx, prURL, agenticWorkflowsLabel, verbose) } return nil } diff --git a/pkg/cli/update_container_pins.go b/pkg/cli/update_container_pins.go index cf53ebfa3d4..13d1e48c5ba 100644 --- a/pkg/cli/update_container_pins.go +++ b/pkg/cli/update_container_pins.go @@ -63,136 +63,153 @@ func UpdateContainerPins(ctx context.Context, workflowDir string, verbose bool) fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updating container image pins...")) } - // Collect all container images referenced in the compiled lock files. + images, ok, err := updateContainerPinsCollectImages(workflowDir, verbose) + if err != nil || !ok { + return false, err + } + + actionCache, err := updateContainerPinsLoadCache() + if err != nil { + return false, err + } + prunedCount := updateContainerPinsPrune(actionCache, images) + result := updateContainerPinsResolve(ctx, actionCache, images, verbose) + + updateContainerPinsSummary(result, prunedCount, verbose) + if err := updateContainerPinsSave(actionCache, len(result.updatedImages), prunedCount); err != nil { + return false, err + } + return len(result.updatedImages) > 0, nil +} + +type updateContainerPinsPinnedEntry struct { + image string + pinnedImage string // image@sha256:... +} + +type updateContainerPinsResult struct { + updatedImages []updateContainerPinsPinnedEntry + failedImages []imageFailure + skippedImages []string +} + +func updateContainerPinsCollectImages(workflowDir string, verbose bool) ([]string, bool, error) { images, err := collectImagesFromLockFiles(workflowDir) if err != nil { containerPinsLog.Printf("Failed to collect images from lock files: %v", err) - // Non-fatal — just skip if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to collect container images: %v", err))) } - return false, nil + return nil, false, nil } - if len(images) == 0 { if verbose { fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("No container images found in lock files")) } - return false, nil + return nil, false, nil } - containerPinsLog.Printf("Found %d unique container image(s) across lock files", len(images)) + return images, true, nil +} - // Load the action cache. +func updateContainerPinsLoadCache() (*workflow.ActionCache, error) { actionsLockPath := filepath.Join(".github", "aw", "actions-lock.json") actionCache := workflow.NewActionCache(".") if fileutil.FileExists(actionsLockPath) { if loadErr := actionCache.Load(); loadErr != nil { - return false, fmt.Errorf("failed to load actions-lock.json: %w", loadErr) + return nil, fmt.Errorf("failed to load actions-lock.json: %w", loadErr) } } + return actionCache, nil +} - // Build a set of base image tags (without @sha256: digest suffix) currently - // referenced in the compiled lock files so that stale entries (e.g. superseded - // AWF versions) can be pruned. Lock files that were previously compiled may - // already embed pinned references (image:tag@sha256:...), so we strip the - // digest before comparing against container pin keys, which always use the - // base tag as the key. - imageSet := make(map[string]struct { - }, len(images)) +func updateContainerPinsPrune(actionCache *workflow.ActionCache, images []string) int { + imageSet := make(map[string]struct{}, len(images)) for _, img := range images { base, _, _ := strings.Cut(img, "@sha256:") - imageSet[base] = struct { - }{} + imageSet[base] = struct{}{} } - - // Remove any container pin entries that are no longer referenced by the - // compiled lock files. This keeps actions-lock.json consistent with what - // compile actually emits and prevents stale version accumulation. prunedCount := actionCache.PruneStaleContainerPins(imageSet) if prunedCount > 0 { containerPinsLog.Printf("Pruned %d stale container pin(s) from actions-lock.json", prunedCount) } + return prunedCount +} - // Resolve digests for images that are not yet pinned. - type pinnedEntry struct { - image string - pinnedImage string // image@sha256:... - } - var updatedImages []pinnedEntry - var failedImages []imageFailure - var skippedImages []string - +func updateContainerPinsResolve(ctx context.Context, actionCache *workflow.ActionCache, images []string, verbose bool) updateContainerPinsResult { + var result updateContainerPinsResult for _, image := range images { - // Images already containing @sha256: are immutably pinned — skip them. - if strings.Contains(image, "@sha256:") { - skippedImages = append(skippedImages, image) - continue - } - - // Check if we already have a valid pin for this image in the cache. - if pin, ok := actionCache.GetContainerPin(image); ok && pin.Digest != "" { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("%s already pinned: %s", image, pin.Digest))) - } - skippedImages = append(skippedImages, image) - continue - } - - // Attempt to resolve the digest without pulling. - digest, resolveErr := fetchContainerDigest(ctx, image, verbose) - if resolveErr != nil { - containerPinsLog.Printf("Failed to resolve digest for %s: %v", image, resolveErr) - failedImages = append(failedImages, imageFailure{image: image, reason: resolveErr.Error()}) - continue - } - - pinnedImage := image + "@" + digest - actionCache.SetContainerPin(image, digest, pinnedImage) - updatedImages = append(updatedImages, pinnedEntry{image: image, pinnedImage: pinnedImage}) + updateContainerPinsResolveImage(ctx, actionCache, image, &result, verbose) + } + return result +} +func updateContainerPinsResolveImage(ctx context.Context, actionCache *workflow.ActionCache, image string, result *updateContainerPinsResult, verbose bool) { + if strings.Contains(image, "@sha256:") { + result.skippedImages = append(result.skippedImages, image) + return + } + if pin, ok := actionCache.GetContainerPin(image); ok && pin.Digest != "" { if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Pinned %s → %s", image, digest))) + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("%s already pinned: %s", image, pin.Digest))) } + result.skippedImages = append(result.skippedImages, image) + return + } + digest, resolveErr := fetchContainerDigest(ctx, image, verbose) + if resolveErr != nil { + containerPinsLog.Printf("Failed to resolve digest for %s: %v", image, resolveErr) + result.failedImages = append(result.failedImages, imageFailure{image: image, reason: resolveErr.Error()}) + return } + pinnedImage := image + "@" + digest + actionCache.SetContainerPin(image, digest, pinnedImage) + result.updatedImages = append(result.updatedImages, updateContainerPinsPinnedEntry{image: image, pinnedImage: pinnedImage}) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Pinned %s → %s", image, digest))) + } +} - // Print summary. +func updateContainerPinsSummary(result updateContainerPinsResult, prunedCount int, verbose bool) { fmt.Fprintln(os.Stderr, "") - - if len(updatedImages) > 0 { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Pinned %d container image(s):", len(updatedImages)))) - for _, entry := range updatedImages { + if len(result.updatedImages) > 0 { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Pinned %d container image(s):", len(result.updatedImages)))) + for _, entry := range result.updatedImages { fmt.Fprintln(os.Stderr, console.FormatListItem(entry.pinnedImage)) } fmt.Fprintln(os.Stderr, "") } - if prunedCount > 0 { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Pruned %d stale container pin(s) from actions-lock.json", prunedCount))) fmt.Fprintln(os.Stderr, "") } - - if len(skippedImages) > 0 && verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("%d container image(s) already up to date", len(skippedImages)))) + if len(result.skippedImages) > 0 && verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("%d container image(s) already up to date", len(result.skippedImages)))) fmt.Fprintln(os.Stderr, "") } + updateContainerPinsFailedSummary(result.failedImages) +} - if len(failedImages) > 0 { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to resolve digest for %d image(s) (Docker/crane may be unavailable):", len(failedImages)))) - for _, f := range failedImages { - fmt.Fprintf(os.Stderr, " %s: %s\n", f.image, f.reason) - } - fmt.Fprintln(os.Stderr, "") +func updateContainerPinsFailedSummary(failedImages []imageFailure) { + if len(failedImages) == 0 { + return } - - if len(updatedImages) > 0 || prunedCount > 0 { - if err := actionCache.Save(); err != nil { - return false, fmt.Errorf("failed to save actions-lock.json: %w", err) - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updated container pins in actions-lock.json")) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to resolve digest for %d image(s) (Docker/crane may be unavailable):", len(failedImages)))) + for _, f := range failedImages { + fmt.Fprintf(os.Stderr, " %s: %s\n", f.image, f.reason) } + fmt.Fprintln(os.Stderr, "") +} - return len(updatedImages) > 0, nil +func updateContainerPinsSave(actionCache *workflow.ActionCache, updatedCount, prunedCount int) error { + if updatedCount == 0 && prunedCount == 0 { + return nil + } + if err := actionCache.Save(); err != nil { + return fmt.Errorf("failed to save actions-lock.json: %w", err) + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updated container pins in actions-lock.json")) + return nil } // collectImagesFromLockFiles scans all .lock.yml files under workflowDir and returns diff --git a/pkg/cli/update_extension_check.go b/pkg/cli/update_extension_check.go index 2fb9612b05e..628d3fea3d4 100644 --- a/pkg/cli/update_extension_check.go +++ b/pkg/cli/update_extension_check.go @@ -49,148 +49,152 @@ func upgradeExtensionIfOutdated(verbose bool, includePrereleases bool) (bool, st currentVersion := GetVersion() updateExtensionCheckLog.Printf("Checking if extension needs upgrade (current: %s)", currentVersion) - // Skip for non-release versions (dev builds) + latestVersion, ok, err := upgradeExtensionIfOutdatedLatest(currentVersion, verbose, includePrereleases) + if err != nil || !ok { + return false, "", err + } + if upgradeExtensionIfOutdatedAlreadyCurrent(currentVersion, latestVersion, verbose, includePrereleases) { + return false, "", nil + } + + upgradeExtensionIfOutdatedAnnounce(currentVersion, latestVersion) + if includePrereleases && !needsRenameWorkaround() { + return upgradeExtensionIfOutdatedInstallPinned(latestVersion) + } + + firstErr, firstAttemptBuf, upgraded, err := upgradeExtensionIfOutdatedFirstAttempt(latestVersion) + if err != nil || upgraded { + return upgraded, "", err + } + if !needsRenameWorkaround() { + return false, "", fmt.Errorf("failed to upgrade gh-aw extension: %w", firstErr) + } + return upgradeExtensionIfOutdatedRetryWithRename(latestVersion, firstAttemptBuf) +} + +func upgradeExtensionIfOutdatedLatest(currentVersion string, verbose bool, includePrereleases bool) (string, bool, error) { if !workflow.IsReleasedVersion(currentVersion) { updateExtensionCheckLog.Print("Not a released version, skipping upgrade check") if verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping extension upgrade check (development build)")) } - return false, "", nil + return "", false, nil } - - // Query GitHub API for latest release latestVersion, err := getLatestRelease(includePrereleases) if err != nil { - // Fail silently - don't block the upgrade command if we can't reach GitHub updateExtensionCheckLog.Printf("Failed to check for latest release (silently ignoring): %v", err) if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Could not check for extension updates: %v", err))) } - return false, "", nil + return "", false, nil } - if latestVersion == "" { updateExtensionCheckLog.Print("Could not determine latest version, skipping upgrade") - return false, "", nil + return "", false, nil } - updateExtensionCheckLog.Printf("Latest version: %s", latestVersion) + return latestVersion, true, nil +} - // Ensure both versions have the 'v' prefix required by the semver package. +func upgradeExtensionIfOutdatedAlreadyCurrent(currentVersion, latestVersion string, verbose bool, includePrereleases bool) bool { currentSV := "v" + strings.TrimPrefix(currentVersion, "v") latestSV := "v" + strings.TrimPrefix(latestVersion, "v") - - // Already on the latest (or newer) version – use proper semver comparison so - // that e.g. "0.10.0" is correctly treated as newer than "0.9.0". - if semver.IsValid(currentSV) && semver.IsValid(latestSV) { - if semver.Compare(currentSV, latestSV) >= 0 { - updateExtensionCheckLog.Print("Extension is already up to date") - if notice := prereleaseChannelNotice(currentVersion, latestVersion, includePrereleases); len(notice) > 0 { - for _, line := range notice { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(line)) - } - } else if verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("gh-aw extension is up to date")) - } - return false, "", nil - } - } else { - // Versions are not valid semver; skip unreliable string comparison and - // proceed with the upgrade to avoid incorrectly treating an outdated - // version as up to date (lexicographic comparison breaks for e.g. "0.9.0" vs "0.10.0"). + if !semver.IsValid(currentSV) || !semver.IsValid(latestSV) { updateExtensionCheckLog.Printf("Non-semver versions detected (current=%q, latest=%q); proceeding with upgrade", currentVersion, latestVersion) + return false } + if semver.Compare(currentSV, latestSV) < 0 { + return false + } + updateExtensionCheckLog.Print("Extension is already up to date") + if notice := prereleaseChannelNotice(currentVersion, latestVersion, includePrereleases); len(notice) > 0 { + for _, line := range notice { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(line)) + } + } else if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("gh-aw extension is up to date")) + } + return true +} - // A newer version is available – upgrade automatically +func upgradeExtensionIfOutdatedAnnounce(currentVersion, latestVersion string) { updateExtensionCheckLog.Printf("Upgrading extension from %s to %s", currentVersion, latestVersion) fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Upgrading gh-aw extension from %s to %s...", renderReleaseVersion(currentVersion), renderReleaseVersion(latestVersion)))) +} - // When targeting a prerelease version on platforms that do not lock running - // binaries (i.e., not Linux or Windows), gh extension upgrade --force resolves - // the upgrade target via /releases/latest which excludes prereleases. On - // those platforms the first attempt would silently install an older stable - // release instead of the desired prerelease. Skip directly to the pin-based - // install to ensure the exact target version is installed. - // Note: on Linux/Windows the binary is locked, so gh extension upgrade --force - // fails with ETXTBSY/Access-denied and we fall through to the rename+retry path - // which already uses pin-based install; the check below is only needed for - // platforms (e.g. macOS) where the first attempt would "succeed" with the wrong - // version. - if includePrereleases && !needsRenameWorkaround() { - updateExtensionCheckLog.Printf("Prerelease upgrade on macOS: skipping gh extension upgrade (uses /releases/latest, ignores prereleases), using pin-based install for %s", latestVersion) - removeCmd := ghCmdForExtension("extension", "remove", extensionRepo) - removeCmd.Stdout = os.Stderr - removeCmd.Stderr = os.Stderr - if removeErr := removeCmd.Run(); removeErr != nil { - updateExtensionCheckLog.Printf("Could not remove extension before pin-based install (continuing anyway): %v", removeErr) - } - pinCmd := ghCmdForExtension("extension", "install", extensionRepo, "--pin", latestVersion) - pinCmd.Stdout = os.Stderr - pinCmd.Stderr = os.Stderr - if pinErr := pinCmd.Run(); pinErr != nil { - return false, "", fmt.Errorf("failed to install gh-aw extension at version %s: %w", renderReleaseVersion(latestVersion), pinErr) - } - _, versionErr := verifyInstalledVersion(latestVersion) - if versionErr != nil { - return false, "", fmt.Errorf("failed to verify gh-aw extension version after upgrade: %w", versionErr) - } - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("gh-aw extension upgraded to "+renderReleaseVersion(latestVersion))) - return true, "", nil - } - - // First attempt: run the upgrade without touching the filesystem. - // On most systems this will succeed. On Linux with WSL the kernel may - // return ETXTBSY when gh tries to open the currently-executing binary for - // writing; on Windows the OS returns "Access is denied" for the same - // reason. In both cases we fall through to the rename+retry path below. - // - // On Linux and Windows we buffer the first attempt's output rather than - // printing it directly, so that the error message is suppressed when the - // rename+retry path succeeds and the user is not shown a confusing failure. +func upgradeExtensionIfOutdatedInstallPinned(latestVersion string) (bool, string, error) { + updateExtensionCheckLog.Printf("Prerelease upgrade on macOS: skipping gh extension upgrade (uses /releases/latest, ignores prereleases), using pin-based install for %s", latestVersion) + removeCmd := ghCmdForExtension("extension", "remove", extensionRepo) + removeCmd.Stdout = os.Stderr + removeCmd.Stderr = os.Stderr + if removeErr := removeCmd.Run(); removeErr != nil { + updateExtensionCheckLog.Printf("Could not remove extension before pin-based install (continuing anyway): %v", removeErr) + } + pinCmd := ghCmdForExtension("extension", "install", extensionRepo, "--pin", latestVersion) + pinCmd.Stdout = os.Stderr + pinCmd.Stderr = os.Stderr + if pinErr := pinCmd.Run(); pinErr != nil { + return false, "", fmt.Errorf("failed to install gh-aw extension at version %s: %w", renderReleaseVersion(latestVersion), pinErr) + } + _, versionErr := verifyInstalledVersion(latestVersion) + if versionErr != nil { + return false, "", fmt.Errorf("failed to verify gh-aw extension version after upgrade: %w", versionErr) + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("gh-aw extension upgraded to "+renderReleaseVersion(latestVersion))) + return true, "", nil +} + +func upgradeExtensionIfOutdatedFirstAttempt(latestVersion string) (error, bytes.Buffer, bool, error) { var firstAttemptBuf bytes.Buffer firstAttemptOut := firstAttemptWriter(os.Stderr, &firstAttemptBuf) firstCmd := ghCmdForExtension(extensionUpgradeArgs()...) firstCmd.Stdout = firstAttemptOut firstCmd.Stderr = firstAttemptOut firstErr := firstCmd.Run() - if firstErr == nil { - // First attempt succeeded without any file manipulation. - if needsRenameWorkaround() { - // Replay the buffered output that was not shown during the attempt. - _, _ = io.Copy(os.Stderr, &firstAttemptBuf) - } - installedVersion, versionErr := installedExtensionVersion() - if versionErr != nil { - return false, "", fmt.Errorf("failed to verify gh-aw extension version after upgrade: %w", versionErr) - } - if normalizeVersion(installedVersion) != normalizeVersion(latestVersion) { - updateExtensionCheckLog.Printf("First upgrade attempt reported success but installed version is %s (expected %s)", renderReleaseVersion(installedVersion), renderReleaseVersion(latestVersion)) - mismatchErr := fmt.Errorf("failed to upgrade gh-aw extension: expected %s, got %s", renderReleaseVersion(latestVersion), renderReleaseVersion(installedVersion)) - if !needsRenameWorkaround() { - return false, "", mismatchErr - } - firstErr = fmt.Errorf("gh extension upgrade reported success without installing target version: %w", mismatchErr) - } else { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("gh-aw extension upgraded to "+renderReleaseVersion(latestVersion))) - return true, "", nil - } + if firstErr != nil { + return firstErr, firstAttemptBuf, false, nil } - - // First attempt failed. - if !needsRenameWorkaround() { - // On platforms other than Linux and Windows there is nothing more to try. - return false, "", fmt.Errorf("failed to upgrade gh-aw extension: %w", firstErr) + if needsRenameWorkaround() { + _, _ = io.Copy(os.Stderr, &firstAttemptBuf) } + installedVersion, versionErr := installedExtensionVersion() + if versionErr != nil { + return nil, firstAttemptBuf, false, fmt.Errorf("failed to verify gh-aw extension version after upgrade: %w", versionErr) + } + if normalizeVersion(installedVersion) != normalizeVersion(latestVersion) { + updateExtensionCheckLog.Printf("First upgrade attempt reported success but installed version is %s (expected %s)", renderReleaseVersion(installedVersion), renderReleaseVersion(latestVersion)) + mismatchErr := fmt.Errorf("failed to upgrade gh-aw extension: expected %s, got %s", renderReleaseVersion(latestVersion), renderReleaseVersion(installedVersion)) + if !needsRenameWorkaround() { + return nil, firstAttemptBuf, false, mismatchErr + } + firstErr = fmt.Errorf("gh extension upgrade reported success without installing target version: %w", mismatchErr) + return firstErr, firstAttemptBuf, false, nil + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("gh-aw extension upgraded to "+renderReleaseVersion(latestVersion))) + return nil, firstAttemptBuf, true, nil +} - // On Linux the failure is likely ETXTBSY; on Windows it is likely - // "Access is denied". Both arise because the OS prevents overwriting a - // running binary. Attempt the rename+retry workaround: rename the - // currently-running binary away to free up its path, then retry the - // upgrade so that gh can write the new binary at the original location. +func upgradeExtensionIfOutdatedRetryWithRename(latestVersion string, firstAttemptBuf bytes.Buffer) (bool, string, error) { updateExtensionCheckLog.Printf("First upgrade attempt failed (likely locked binary); retrying with rename workaround. First attempt output: %s", firstAttemptBuf.String()) + installPath, backupPath := upgradeExtensionIfOutdatedRenameCurrentExecutable() + if err := upgradeExtensionIfOutdatedRemoveForRetry(&backupPath); err != nil { + updateExtensionCheckLog.Print(err) + } + if err := upgradeExtensionIfOutdatedRunRetry(latestVersion, installPath, backupPath, firstAttemptBuf); err != nil { + return false, "", err + } + _, versionErr := verifyInstalledVersion(latestVersion) + if versionErr != nil { + return false, "", fmt.Errorf("failed to verify gh-aw extension version after upgrade: %w", versionErr) + } + if backupPath != "" { + cleanupExecutableBackup(backupPath) + } + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("gh-aw extension upgraded to "+renderReleaseVersion(latestVersion))) + return true, installPath, nil +} - // Resolve the current executable path before renaming; after the rename - // os.Executable() returns a "(deleted)"-suffixed path on Linux. +func upgradeExtensionIfOutdatedRenameCurrentExecutable() (string, string) { var installPath string var backupPath string if exe, exeErr := os.Executable(); exeErr == nil { @@ -198,121 +202,76 @@ func upgradeExtensionIfOutdated(verbose bool, includePrereleases bool) (bool, st exe = resolved } if iPath, bPath, renameErr := renamePathForUpgrade(exe); renameErr != nil { - // Rename failed; the retry will likely fail again. updateExtensionCheckLog.Printf("Could not rename executable for retry (upgrade will likely fail): %v", renameErr) } else { installPath = iPath backupPath = bPath - // On Windows, gh extension remove cannot delete the extension directory - // while it still contains a running binary (even a renamed one). Move - // the backup to a location outside the extension directory so that - // gh extension remove can succeed. - // - // We first try os.TempDir(); if that fails because TEMP is on a - // different drive (common on GitHub Actions runners where the extension - // lives on C: but TEMP is on D:), we fall back to the parent of the - // extension directory which is guaranteed to be on the same drive. - if runtime.GOOS == "windows" { - extDir := filepath.Dir(backupPath) - moved := false - - // Attempt 1: OS temp directory - tmpBackup := filepath.Join(os.TempDir(), filepath.Base(backupPath)) - if moveErr := os.Rename(backupPath, tmpBackup); moveErr == nil { - updateExtensionCheckLog.Printf("Moved Windows backup %s -> %s to free extension directory for removal", backupPath, tmpBackup) - backupPath = tmpBackup - moved = true - } else { - updateExtensionCheckLog.Printf("Could not move backup to %s (cross-drive?): %v; trying same-drive fallback", tmpBackup, moveErr) - } - - // Attempt 2: parent of the extension directory (same drive as backup) - if !moved { - sameDriveDir := filepath.Dir(extDir) - sameDriveTmp := filepath.Join(sameDriveDir, filepath.Base(backupPath)) - if moveErr2 := os.Rename(backupPath, sameDriveTmp); moveErr2 == nil { - updateExtensionCheckLog.Printf("Moved Windows backup %s -> %s (same-drive fallback) to free extension directory for removal", backupPath, sameDriveTmp) - backupPath = sameDriveTmp - } else { - updateExtensionCheckLog.Printf("Could not move backup out of extension directory (gh extension remove may fail): %v", moveErr2) - } - } - - // After moving our own backup out of the extension directory, try to - // remove any stale .bak files left by previous upgrade attempts or the - // gh CLI's own rename mechanism. These may be temporarily locked by - // Windows Defender; retry a few times with short delays. - cleanupStaleWindowsBackups(extDir, backupPath) - } + backupPath = upgradeExtensionIfOutdatedMoveWindowsBackup(backupPath) } } + return installPath, backupPath +} + +func upgradeExtensionIfOutdatedMoveWindowsBackup(backupPath string) string { + if runtime.GOOS != "windows" || backupPath == "" { + return backupPath + } + extDir := filepath.Dir(backupPath) + backupPath = upgradeExtensionIfOutdatedMoveWindowsBackupOut(backupPath, extDir) + cleanupStaleWindowsBackups(extDir, backupPath) + return backupPath +} + +func upgradeExtensionIfOutdatedMoveWindowsBackupOut(backupPath, extDir string) string { + tmpBackup := filepath.Join(os.TempDir(), filepath.Base(backupPath)) + if moveErr := os.Rename(backupPath, tmpBackup); moveErr == nil { + updateExtensionCheckLog.Printf("Moved Windows backup %s -> %s to free extension directory for removal", backupPath, tmpBackup) + return tmpBackup + } + sameDriveDir := filepath.Dir(extDir) + sameDriveTmp := filepath.Join(sameDriveDir, filepath.Base(backupPath)) + if moveErr2 := os.Rename(backupPath, sameDriveTmp); moveErr2 == nil { + updateExtensionCheckLog.Printf("Moved Windows backup %s -> %s (same-drive fallback) to free extension directory for removal", backupPath, sameDriveTmp) + return sameDriveTmp + } + updateExtensionCheckLog.Printf("Could not move backup out of extension directory (gh extension remove may fail)") + return backupPath +} - // Retry path: remove + reinstall at the exact target version. - // - // Using "gh extension upgrade --force" again would call fetchLatestRelease - // (/releases/latest) internally, which returns 404 for prerelease-only repos - // and causes "unable to retrieve latest version for extension" errors. - // Using "gh extension install --pin VERSION" instead calls fetchReleaseFromTag, - // which accepts any tag (stable or prerelease). - // - // We must remove the extension first because "gh extension install" checks - // whether the extension is already present via its manifest.yml. With the - // manifest in place the install command takes the "already installed" code - // path and does nothing; removing the extension clears that guard. - // - // Note: on Linux the backup file lives inside the extension directory and is - // gone once the remove step succeeds (unlink frees the directory entry even - // though the process still holds the file open). On Windows the backup has - // been moved to the OS temp directory (above) so the remove step can always - // succeed. In both cases we clear backupPath after a successful remove to - // avoid a misleading restore attempt on subsequent failures. +func upgradeExtensionIfOutdatedRemoveForRetry(backupPath *string) error { removeCmd := ghCmdForExtension("extension", "remove", extensionRepo) removeCmd.Stdout = os.Stderr removeCmd.Stderr = os.Stderr - if removeErr := removeCmd.Run(); removeErr == nil { - // Extension directory has been deleted. - backupPath = "" - } else { - updateExtensionCheckLog.Printf("Could not remove extension before reinstall (will attempt install anyway): %v", removeErr) + if removeErr := removeCmd.Run(); removeErr != nil { + return fmt.Errorf("Could not remove extension before reinstall (will attempt install anyway): %w", removeErr) } + *backupPath = "" + return nil +} +func upgradeExtensionIfOutdatedRunRetry(latestVersion, installPath, backupPath string, firstAttemptBuf bytes.Buffer) error { retryCmd := ghCmdForExtension("extension", "install", extensionRepo, "--pin", latestVersion) retryCmd.Stdout = os.Stderr retryCmd.Stderr = os.Stderr if retryErr := retryCmd.Run(); retryErr != nil { - // Retry also failed. Restore the backup so the user still has gh-aw - // (only possible when the remove step above did not succeed). if backupPath != "" { restoreExecutableBackup(installPath, backupPath) } if runtime.GOOS == "windows" && isWindowsLockError(firstAttemptBuf.String(), retryErr) { - // On Windows, self-upgrade may not be possible while the binary is - // running. Guide the user to upgrade manually from a separate shell. - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("On Windows, gh-aw cannot self-upgrade while it is running.")) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please upgrade manually by running one of the following:")) - fmt.Fprintln(os.Stderr, " "+extensionUpgradeHelpCommand(latestVersion)) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("If that does not work, try reinstalling:")) - fmt.Fprintln(os.Stderr, " gh extension remove gh-aw") - fmt.Fprintln(os.Stderr, " "+extensionInstallHelpCommand(latestVersion)) + upgradeExtensionIfOutdatedPrintWindowsHelp(latestVersion) } - return false, "", fmt.Errorf("failed to upgrade gh-aw extension: %w", retryErr) - } - - _, versionErr := verifyInstalledVersion(latestVersion) - if versionErr != nil { - // Verification failed; leave the backup in place so the user can roll - // back manually if needed. - return false, "", fmt.Errorf("failed to verify gh-aw extension version after upgrade: %w", versionErr) - } - - // Verification passed. Clean up the backup now that it is safe to do so - // (it will already be gone when the remove step above succeeded). - if backupPath != "" { - cleanupExecutableBackup(backupPath) + return fmt.Errorf("failed to upgrade gh-aw extension: %w", retryErr) } + return nil +} - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("gh-aw extension upgraded to "+renderReleaseVersion(latestVersion))) - return true, installPath, nil +func upgradeExtensionIfOutdatedPrintWindowsHelp(latestVersion string) { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("On Windows, gh-aw cannot self-upgrade while it is running.")) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Please upgrade manually by running one of the following:")) + fmt.Fprintln(os.Stderr, " "+extensionUpgradeHelpCommand(latestVersion)) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("If that does not work, try reinstalling:")) + fmt.Fprintln(os.Stderr, " gh extension remove gh-aw") + fmt.Fprintln(os.Stderr, " "+extensionInstallHelpCommand(latestVersion)) } // needsRenameWorkaround reports whether the current platform requires the diff --git a/pkg/cli/update_manifest.go b/pkg/cli/update_manifest.go index 2ef306cf2c6..517ee82b538 100644 --- a/pkg/cli/update_manifest.go +++ b/pkg/cli/update_manifest.go @@ -68,17 +68,39 @@ func updateManifestWorkflowGroup(ctx context.Context, source string, grouped []* return successes, failures } + groupInfo, ok, failures := resolveUpdateManifestWorkflowGroupInfo(ctx, source, grouped, opts) + if !ok { + return successes, failures + } + + existingByName := make(map[string]*workflowWithSource, len(grouped)) + for _, wf := range grouped { + existingByName[wf.Name] = wf + } + + successes, failures = updateManifestWorkflowGroupExisting(ctx, groupInfo, existingByName, opts, successes, failures) + targetDir := filepath.Dir(grouped[0].Path) + successes, failures = updateManifestWorkflowGroupAdditions(ctx, groupInfo, existingByName, targetDir, opts, successes, failures) + return successes, failures +} + +type updateManifestWorkflowGroupInfo struct { + repoSpec *RepoSpec + currentRef string + latestRef string + manifestSource string + currentByName map[string]string + latestByName map[string]string +} + +func resolveUpdateManifestWorkflowGroupInfo(ctx context.Context, source string, grouped []*workflowWithSource, opts UpdateWorkflowsOptions) (updateManifestWorkflowGroupInfo, bool, []updateFailure) { repoSpec, _, err := parseManifestSourceSpec(source) if err != nil { - for _, wf := range grouped { - failures = append(failures, updateFailure{Name: wf.Name, Error: err.Error()}) - } - return successes, failures + return updateManifestWorkflowGroupInfo{}, false, updateManifestWorkflowGroupFailures(grouped, err.Error()) } if repoSpec == nil { - return successes, failures + return updateManifestWorkflowGroupInfo{}, false, nil } - currentRef := repoSpec.Version if currentRef == "" { currentRef = "main" @@ -86,53 +108,50 @@ func updateManifestWorkflowGroup(ctx context.Context, source string, grouped []* latestRef, err := resolveLatestRefFn(ctx, repoSpec.RepoSlug, currentRef, opts.AllowMajor, opts.Verbose, opts.CoolDown) if err != nil { updateManifestLog.Printf("Failed to resolve latest manifest ref for %s: %v", repoSpec.RepoSlug, err) - for _, wf := range grouped { - failures = append(failures, updateFailure{Name: wf.Name, Error: fmt.Sprintf("failed to resolve latest manifest ref: %v", err)}) - } - return successes, failures + return updateManifestWorkflowGroupInfo{}, false, updateManifestWorkflowGroupFailures(grouped, fmt.Sprintf("failed to resolve latest manifest ref: %v", err)) + } + currentPkg, latestPkg, failures := updateManifestWorkflowGroupPackages(ctx, repoSpec, currentRef, latestRef, grouped) + if len(failures) > 0 { + return updateManifestWorkflowGroupInfo{}, false, failures } - updateManifestLog.Printf("Resolved manifest refs: current=%s, latest=%s", currentRef, latestRef) sourceFieldRef := latestRef - // Preserve branch-tracking behavior: when source points to a branch, keep the - // branch name in source so future updates continue following that branch. - // For tags/SHAs, pin to the resolved latest ref. if isBranchRef(currentRef) { sourceFieldRef = currentRef } + return updateManifestWorkflowGroupInfo{ + repoSpec: repoSpec, + currentRef: currentRef, + latestRef: latestRef, + manifestSource: manifestSourceWithRef(repoSpec, sourceFieldRef), + currentByName: manifestWorkflowPathByName(currentPkg.InstallationSource), + latestByName: manifestWorkflowPathByName(latestPkg.InstallationSource), + }, true, nil +} - currentPkg, err := resolveRepositoryPackage(ctx, &RepoSpec{ - RepoSlug: repoSpec.RepoSlug, - PackagePath: repoSpec.PackagePath, - Version: currentRef, - }, "") +func updateManifestWorkflowGroupPackages(ctx context.Context, repoSpec *RepoSpec, currentRef, latestRef string, grouped []*workflowWithSource) (*resolvedRepositoryPackage, *resolvedRepositoryPackage, []updateFailure) { + updateManifestLog.Printf("Resolved manifest refs: current=%s, latest=%s", currentRef, latestRef) + currentPkg, err := resolveRepositoryPackage(ctx, &RepoSpec{RepoSlug: repoSpec.RepoSlug, PackagePath: repoSpec.PackagePath, Version: currentRef}, "") if err != nil { - for _, wf := range grouped { - failures = append(failures, updateFailure{Name: wf.Name, Error: fmt.Sprintf("failed to resolve current manifest package: %v", err)}) - } - return successes, failures + return nil, nil, updateManifestWorkflowGroupFailures(grouped, fmt.Sprintf("failed to resolve current manifest package: %v", err)) } - latestPkg, err := resolveRepositoryPackage(ctx, &RepoSpec{ - RepoSlug: repoSpec.RepoSlug, - PackagePath: repoSpec.PackagePath, - Version: latestRef, - }, "") + latestPkg, err := resolveRepositoryPackage(ctx, &RepoSpec{RepoSlug: repoSpec.RepoSlug, PackagePath: repoSpec.PackagePath, Version: latestRef}, "") if err != nil { - for _, wf := range grouped { - failures = append(failures, updateFailure{Name: wf.Name, Error: fmt.Sprintf("failed to resolve latest manifest package: %v", err)}) - } - return successes, failures + return nil, nil, updateManifestWorkflowGroupFailures(grouped, fmt.Sprintf("failed to resolve latest manifest package: %v", err)) } + return currentPkg, latestPkg, nil +} - currentByName := manifestWorkflowPathByName(currentPkg.InstallationSource) - latestByName := manifestWorkflowPathByName(latestPkg.InstallationSource) - existingByName := make(map[string]*workflowWithSource, len(grouped)) +func updateManifestWorkflowGroupFailures(grouped []*workflowWithSource, msg string) []updateFailure { + var failures []updateFailure for _, wf := range grouped { - existingByName[wf.Name] = wf + failures = append(failures, updateFailure{Name: wf.Name, Error: msg}) } + return failures +} - manifestSource := manifestSourceWithRef(repoSpec, sourceFieldRef) +func updateManifestWorkflowGroupExisting(ctx context.Context, info updateManifestWorkflowGroupInfo, existingByName map[string]*workflowWithSource, opts UpdateWorkflowsOptions, successes []string, failures []updateFailure) ([]string, []updateFailure) { for name, wf := range existingByName { - latestPath, exists := latestByName[name] + latestPath, exists := info.latestByName[name] if !exists { if err := removeManifestManagedWorkflow(wf.Path); err != nil { failures = append(failures, updateFailure{Name: wf.Name, Error: err.Error()}) @@ -141,39 +160,43 @@ func updateManifestWorkflowGroup(ctx context.Context, source string, grouped []* successes = append(successes, wf.Name) continue } - - oldPath := currentByName[name] - if oldPath == "" { - oldPath = latestPath - } - update := manifestManagedWorkflowUpdate{ - wf: wf, - repo: repoSpec.RepoSlug, - currentPath: oldPath, - latestPath: latestPath, - currentRef: currentRef, - latestRef: latestRef, - manifestSource: manifestSource, - } + update := updateManifestWorkflowGroupUpdate(info, wf, name, latestPath) if err := updateManifestManagedWorkflow(ctx, update, opts); err != nil { failures = append(failures, updateFailure{Name: wf.Name, Error: err.Error()}) continue } successes = append(successes, wf.Name) } + return successes, failures +} - targetDir := filepath.Dir(grouped[0].Path) - for name, latestPath := range latestByName { +func updateManifestWorkflowGroupUpdate(info updateManifestWorkflowGroupInfo, wf *workflowWithSource, name, latestPath string) manifestManagedWorkflowUpdate { + oldPath := info.currentByName[name] + if oldPath == "" { + oldPath = latestPath + } + return manifestManagedWorkflowUpdate{ + wf: wf, + repo: info.repoSpec.RepoSlug, + currentPath: oldPath, + latestPath: latestPath, + currentRef: info.currentRef, + latestRef: info.latestRef, + manifestSource: info.manifestSource, + } +} + +func updateManifestWorkflowGroupAdditions(ctx context.Context, info updateManifestWorkflowGroupInfo, existingByName map[string]*workflowWithSource, targetDir string, opts UpdateWorkflowsOptions, successes []string, failures []updateFailure) ([]string, []updateFailure) { + for name, latestPath := range info.latestByName { if _, exists := existingByName[name]; exists { continue } - if err := addManifestManagedWorkflow(ctx, targetDir, name, repoSpec.RepoSlug, latestPath, latestRef, manifestSource, opts); err != nil { + if err := addManifestManagedWorkflow(ctx, targetDir, name, info.repoSpec.RepoSlug, latestPath, info.latestRef, info.manifestSource, opts); err != nil { failures = append(failures, updateFailure{Name: name, Error: err.Error()}) continue } successes = append(successes, name) } - return successes, failures } @@ -198,74 +221,18 @@ func updateManifestManagedWorkflow(ctx context.Context, update manifestManagedWo return fmt.Errorf("failed to download workflow %s/%s@%s: %w", update.repo, update.latestPath, update.latestRef, err) } - if !opts.Force && update.currentRef == update.latestRef && update.currentPath == update.latestPath { - sourceContent, err := downloadWorkflowContentFn(ctx, update.repo, update.currentPath, update.currentRef, opts.Verbose) - if err == nil { - currentContent, readErr := os.ReadFile(update.wf.Path) - if readErr == nil && !hasLocalModifications(string(sourceContent), string(currentContent), sourceSpecCurrent, filepath.Dir(update.wf.Path), opts.Verbose) { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow %s is already up to date (%s)", update.wf.Name, shortRef(update.currentRef)))) - return nil - } - } + upToDate, err := updateManifestManagedWorkflowAlreadyCurrent(ctx, update, sourceSpecCurrent, opts) + if err != nil || upToDate { + return err } - merge := !opts.NoMerge - var finalContent string - var hasConflicts bool - if merge { - baseContent, err := downloadWorkflowContentFn(ctx, update.repo, update.currentPath, update.currentRef, opts.Verbose) - if err != nil { - updateManifestLog.Printf("Cannot fetch base for 3-way merge of %s, falling back to overwrite: %v", update.wf.Name, err) - merge = false - } else { - currentContent, err := os.ReadFile(update.wf.Path) - if err != nil { - return fmt.Errorf("failed to read current workflow: %w", err) - } - newSourceSpec := sourceSpecWithRef(&SourceSpec{Repo: update.repo, Path: update.latestPath}, update.latestRef) - mergedContent, conflicts, mergeErr := MergeWorkflowContent(string(baseContent), string(currentContent), string(newContent), sourceSpecCurrent, newSourceSpec, update.wf.Path, opts.Verbose) - if mergeErr != nil { - return fmt.Errorf("failed to merge workflow content: %w", mergeErr) - } - finalContent = mergedContent - hasConflicts = conflicts - } - } - if !merge { - finalContent = string(newContent) - processedContent, err := processIncludesInContent(finalContent, &WorkflowSpec{ - RepoSpec: RepoSpec{ - RepoSlug: update.repo, - Version: update.latestRef, - }, - WorkflowPath: update.latestPath, - }, update.latestRef, filepath.Dir(update.wf.Path), opts.Verbose) - if err == nil { - finalContent = processedContent - } - } - - finalContent, err = UpdateFieldInFrontmatter(finalContent, "source", update.manifestSource) + finalContent, hasConflicts, err := updateManifestManagedWorkflowContent(ctx, update, sourceSpecCurrent, string(newContent), opts) if err != nil { - return fmt.Errorf("failed to update source frontmatter: %w", err) - } - - if opts.NoStopAfter { - cleanedContent, err := RemoveFieldFromOnTrigger(finalContent, "stop-after") - if err == nil { - finalContent = cleanedContent - } - } else if opts.StopAfter != "" { - updatedContent, err := SetFieldInOnTrigger(finalContent, "stop-after", opts.StopAfter) - if err == nil { - finalContent = updatedContent - } + return err } - - if !opts.DisableSecurityScanner { - if findings := workflow.ScanMarkdownSecurity(finalContent); len(findings) > 0 { - return fmt.Errorf("workflow '%s' failed security scan: %d issue(s) detected", update.wf.Name, len(findings)) - } + finalContent, err = updateManifestManagedWorkflowFinalizeContent(update, finalContent, opts) + if err != nil { + return err } if err := os.WriteFile(update.wf.Path, []byte(finalContent), constants.FilePermPublic); err != nil { @@ -284,6 +251,87 @@ func updateManifestManagedWorkflow(ctx context.Context, update manifestManagedWo return nil } +func updateManifestManagedWorkflowAlreadyCurrent(ctx context.Context, update manifestManagedWorkflowUpdate, sourceSpecCurrent string, opts UpdateWorkflowsOptions) (bool, error) { + if opts.Force || update.currentRef != update.latestRef || update.currentPath != update.latestPath { + return false, nil + } + sourceContent, err := downloadWorkflowContentFn(ctx, update.repo, update.currentPath, update.currentRef, opts.Verbose) + if err != nil { + return false, nil + } + currentContent, readErr := os.ReadFile(update.wf.Path) + if readErr == nil && !hasLocalModifications(string(sourceContent), string(currentContent), sourceSpecCurrent, filepath.Dir(update.wf.Path), opts.Verbose) { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow %s is already up to date (%s)", update.wf.Name, shortRef(update.currentRef)))) + return true, nil + } + return false, nil +} + +func updateManifestManagedWorkflowContent(ctx context.Context, update manifestManagedWorkflowUpdate, sourceSpecCurrent string, newContent string, opts UpdateWorkflowsOptions) (string, bool, error) { + if opts.NoMerge { + return updateManifestManagedWorkflowOverwrite(update, newContent, opts) + } + baseContent, err := downloadWorkflowContentFn(ctx, update.repo, update.currentPath, update.currentRef, opts.Verbose) + if err != nil { + updateManifestLog.Printf("Cannot fetch base for 3-way merge of %s, falling back to overwrite: %v", update.wf.Name, err) + return updateManifestManagedWorkflowOverwrite(update, newContent, opts) + } + currentContent, err := os.ReadFile(update.wf.Path) + if err != nil { + return "", false, fmt.Errorf("failed to read current workflow: %w", err) + } + newSourceSpec := sourceSpecWithRef(&SourceSpec{Repo: update.repo, Path: update.latestPath}, update.latestRef) + mergedContent, conflicts, mergeErr := MergeWorkflowContent(string(baseContent), string(currentContent), newContent, sourceSpecCurrent, newSourceSpec, update.wf.Path, opts.Verbose) + if mergeErr != nil { + return "", false, fmt.Errorf("failed to merge workflow content: %w", mergeErr) + } + return mergedContent, conflicts, nil +} + +func updateManifestManagedWorkflowOverwrite(update manifestManagedWorkflowUpdate, newContent string, opts UpdateWorkflowsOptions) (string, bool, error) { + finalContent := newContent + processedContent, err := processIncludesInContent(finalContent, &WorkflowSpec{ + RepoSpec: RepoSpec{ + RepoSlug: update.repo, + Version: update.latestRef, + }, + WorkflowPath: update.latestPath, + }, update.latestRef, filepath.Dir(update.wf.Path), opts.Verbose) + if err == nil { + finalContent = processedContent + } + return finalContent, false, nil +} + +func updateManifestManagedWorkflowFinalizeContent(update manifestManagedWorkflowUpdate, finalContent string, opts UpdateWorkflowsOptions) (string, error) { + finalContent, err := UpdateFieldInFrontmatter(finalContent, "source", update.manifestSource) + if err != nil { + return "", fmt.Errorf("failed to update source frontmatter: %w", err) + } + finalContent = updateManifestManagedWorkflowStopAfter(finalContent, opts) + if !opts.DisableSecurityScanner { + if findings := workflow.ScanMarkdownSecurity(finalContent); len(findings) > 0 { + return "", fmt.Errorf("workflow '%s' failed security scan: %d issue(s) detected", update.wf.Name, len(findings)) + } + } + return finalContent, nil +} + +func updateManifestManagedWorkflowStopAfter(finalContent string, opts UpdateWorkflowsOptions) string { + if opts.NoStopAfter { + cleanedContent, err := RemoveFieldFromOnTrigger(finalContent, "stop-after") + if err == nil { + return cleanedContent + } + } else if opts.StopAfter != "" { + updatedContent, err := SetFieldInOnTrigger(finalContent, "stop-after", opts.StopAfter) + if err == nil { + return updatedContent + } + } + return finalContent +} + func addManifestManagedWorkflow(ctx context.Context, targetDir, name, repo, latestPath, latestRef, manifestSource string, opts UpdateWorkflowsOptions) error { updateManifestLog.Printf("Adding new manifest-managed workflow %s from %s/%s@%s", name, repo, latestPath, latestRef) newContent, err := downloadWorkflowContentFn(ctx, repo, latestPath, latestRef, opts.Verbose) diff --git a/pkg/cli/update_merge.go b/pkg/cli/update_merge.go index 704e85b1e2f..c6f379b9416 100644 --- a/pkg/cli/update_merge.go +++ b/pkg/cli/update_merge.go @@ -24,13 +24,7 @@ var updateMergeLog = logger.New("cli:update_merge") func hasLocalModifications(sourceContent, localContent, sourceSpec, localWorkflowDir string, verbose bool) bool { updateMergeLog.Printf("Checking for local modifications: source_spec=%s", sourceSpec) // Normalize both contents - sourceNormalized := stringutil.NormalizeWhitespace(sourceContent) - localNormalized := stringutil.NormalizeWhitespace(localContent) - - // Remove stop-after field from both contents for comparison - // This field is deployment-specific and should not trigger "local modifications" warnings - sourceNormalized, _ = RemoveFieldFromOnTrigger(sourceNormalized, "stop-after") - localNormalized, _ = RemoveFieldFromOnTrigger(localNormalized, "stop-after") + sourceNormalized, localNormalized := hasLocalModificationsNormalize(sourceContent, localContent) // Parse the source spec to get repo and ref information parsedSourceSpec, err := parseSourceSpec(sourceSpec) @@ -42,17 +36,33 @@ func hasLocalModifications(sourceContent, localContent, sourceSpec, localWorkflo return sourceNormalized != localNormalized } - // Add the source field to the remote content + sourceResolvedNormalized, localNormalized := hasLocalModificationsResolveSource(sourceNormalized, localNormalized, sourceSpec, parsedSourceSpec, localWorkflowDir, verbose) + hasModifications := sourceResolvedNormalized != localNormalized + updateMergeLog.Printf("Local modifications detected: %v", hasModifications) + + if verbose && hasModifications { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Local modifications detected")) + } + return hasModifications +} + +func hasLocalModificationsNormalize(sourceContent, localContent string) (string, string) { + sourceNormalized := stringutil.NormalizeWhitespace(sourceContent) + localNormalized := stringutil.NormalizeWhitespace(localContent) + sourceNormalized, _ = RemoveFieldFromOnTrigger(sourceNormalized, "stop-after") + localNormalized, _ = RemoveFieldFromOnTrigger(localNormalized, "stop-after") + return sourceNormalized, localNormalized +} + +func hasLocalModificationsResolveSource(sourceNormalized, localNormalized, sourceSpec string, parsedSourceSpec *SourceSpec, localWorkflowDir string, verbose bool) (string, string) { sourceWithSource, err := UpdateFieldInFrontmatter(sourceNormalized, "source", sourceSpec) if err != nil { if verbose { fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Failed to add source field to remote content: %v", err))) } - // Fall back to simple comparison - return sourceNormalized != localNormalized + return sourceNormalized, localNormalized } - // Resolve imports on the remote content workflow := &WorkflowSpec{ RepoSpec: RepoSpec{ RepoSlug: parsedSourceSpec.Repo, @@ -60,7 +70,6 @@ func hasLocalModifications(sourceContent, localContent, sourceSpec, localWorkflo }, WorkflowPath: parsedSourceSpec.Path, } - sourceResolved, err := processIncludesInContent(sourceWithSource, workflow, parsedSourceSpec.Ref, localWorkflowDir, verbose) if err != nil { if verbose { @@ -78,17 +87,7 @@ func hasLocalModifications(sourceContent, localContent, sourceSpec, localWorkflo if normalized, normalizeErr := UpdateFieldInFrontmatter(localNormalized, "source", "__gh_aw_source__"); normalizeErr == nil { localNormalized = normalized } - - // Compare the normalized contents - hasModifications := sourceResolvedNormalized != localNormalized - - updateMergeLog.Printf("Local modifications detected: %v", hasModifications) - - if verbose && hasModifications { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Local modifications detected")) - } - - return hasModifications + return sourceResolvedNormalized, localNormalized } // MergeWorkflowContent performs a 3-way merge of workflow content using git merge-file @@ -103,47 +102,73 @@ func MergeWorkflowContent(base, current, new, oldSourceSpec, newRefOrSourceSpec, fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Performing 3-way merge using git merge-file")) } - // Parse the old source spec to get the current ref + specs, err := mergeWorkflowContentSpecs(oldSourceSpec, newRefOrSourceSpec) + if err != nil { + return "", false, err + } + baseNormalized, currentNormalized, newNormalized := mergeWorkflowContentNormalize(base, current, new, specs.currentSourceSpec, specs.newSourceSpec, verbose) + + currentFile, cleanup, err := mergeWorkflowContentWriteTempFiles(baseNormalized, currentNormalized, newNormalized) + if err != nil { + return "", false, err + } + defer cleanup() + hasConflicts, err := mergeWorkflowContentRunGitMerge(currentFile, verbose) + if err != nil { + return "", false, err + } + + updateMergeLog.Printf("Merge completed: has_conflicts=%v", hasConflicts) + mergedContent, err := os.ReadFile(currentFile) + if err != nil { + return "", false, fmt.Errorf("failed to read merged content: %w", err) + } + + mergedStr := string(mergedContent) + mergedStr = mergeWorkflowContentProcessIncludes(mergedStr, hasConflicts, specs.parsedNewSourceSpec, specs.newRef, localWorkflowPath, verbose) + return mergedStr, hasConflicts, nil +} + +type mergeWorkflowContentSpecInfo struct { + currentSourceSpec string + newSourceSpec string + parsedNewSourceSpec *SourceSpec + newRef string +} + +func mergeWorkflowContentSpecs(oldSourceSpec, newRefOrSourceSpec string) (mergeWorkflowContentSpecInfo, error) { sourceSpec, err := parseSourceSpec(oldSourceSpec) if err != nil { updateMergeLog.Printf("Failed to parse source spec: %v", err) - return "", false, fmt.Errorf("failed to parse source spec: %w", err) + return mergeWorkflowContentSpecInfo{}, fmt.Errorf("failed to parse source spec: %w", err) } currentSourceSpec := fmt.Sprintf("%s/%s@%s", sourceSpec.Repo, sourceSpec.Path, sourceSpec.Ref) - - // Support both legacy ref-only and full source spec for the merge target. newSourceSpec := fmt.Sprintf("%s/%s@%s", sourceSpec.Repo, sourceSpec.Path, newRefOrSourceSpec) if tentativeSourceSpec, parseErr := parseSourceSpec(newRefOrSourceSpec); parseErr == nil { newSourceSpec = sourceSpecWithRef(tentativeSourceSpec, tentativeSourceSpec.Ref) } parsedNewSourceSpec, err := parseSourceSpec(newSourceSpec) if err != nil { - return "", false, fmt.Errorf("failed to parse new source spec: %w", err) + return mergeWorkflowContentSpecInfo{}, fmt.Errorf("failed to parse new source spec: %w", err) } - newRef := parsedNewSourceSpec.Ref + return mergeWorkflowContentSpecInfo{currentSourceSpec: currentSourceSpec, newSourceSpec: newSourceSpec, parsedNewSourceSpec: parsedNewSourceSpec, newRef: parsedNewSourceSpec.Ref}, nil +} - // Fix the base version by adding the source field to match what both current and new have - // This prevents unnecessary conflicts over the source field +func mergeWorkflowContentNormalize(base, current, new, currentSourceSpec, newSourceSpec string, verbose bool) (string, string, string) { baseWithSource, err := UpdateFieldInFrontmatter(base, "source", currentSourceSpec) if err != nil { if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to add source to base content: %v", err))) } - // Continue with original base content baseWithSource = base } - - // Update the source field in the new content with the new ref newWithUpdatedSource, err := UpdateFieldInFrontmatter(new, "source", newSourceSpec) if err != nil { if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to update source in new content: %v", err))) } - // Continue with original new content newWithUpdatedSource = new } - - // Normalize whitespace in all three versions to reduce spurious conflicts baseNormalized := stringutil.NormalizeWhitespace(baseWithSource) currentNormalized := stringutil.NormalizeWhitespace(current) newNormalized := stringutil.NormalizeWhitespace(newWithUpdatedSource) @@ -152,102 +177,78 @@ func MergeWorkflowContent(base, current, new, oldSourceSpec, newRefOrSourceSpec, } else if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to normalize source in current content: %v", normalizeErr))) } + return baseNormalized, currentNormalized, newNormalized +} - // Create temporary directory for merge files +func mergeWorkflowContentWriteTempFiles(baseNormalized, currentNormalized, newNormalized string) (string, func(), error) { tmpDir, err := os.MkdirTemp("", "gh-aw-merge-*") if err != nil { - return "", false, fmt.Errorf("failed to create temp directory: %w", err) + return "", nil, fmt.Errorf("failed to create temp directory: %w", err) } - defer os.RemoveAll(tmpDir) - - // Write base, current, and new versions to temporary files + cleanup := func() { _ = os.RemoveAll(tmpDir) } baseFile := filepath.Join(tmpDir, "base.md") currentFile := filepath.Join(tmpDir, "current.md") newFile := filepath.Join(tmpDir, "new.md") - if err := os.WriteFile(baseFile, []byte(baseNormalized), constants.FilePermPublic); err != nil { - return "", false, fmt.Errorf("failed to write base file: %w", err) + cleanup() + return "", nil, fmt.Errorf("failed to write base file: %w", err) } if err := os.WriteFile(currentFile, []byte(currentNormalized), constants.FilePermPublic); err != nil { - return "", false, fmt.Errorf("failed to write current file: %w", err) + cleanup() + return "", nil, fmt.Errorf("failed to write current file: %w", err) } if err := os.WriteFile(newFile, []byte(newNormalized), constants.FilePermPublic); err != nil { - return "", false, fmt.Errorf("failed to write new file: %w", err) + cleanup() + return "", nil, fmt.Errorf("failed to write new file: %w", err) } + return currentFile, cleanup, nil +} - // Execute git merge-file - // Format: git merge-file - cmd := exec.Command("git", "merge-file", - "-L", "current (local changes)", - "-L", "base (original)", - "-L", "new (upstream)", - "--diff3", // Use diff3 style conflict markers for better context - currentFile, baseFile, newFile) - +func mergeWorkflowContentRunGitMerge(currentFile string, verbose bool) (bool, error) { + baseFile := filepath.Join(filepath.Dir(currentFile), "base.md") + newFile := filepath.Join(filepath.Dir(currentFile), "new.md") + cmd := exec.Command("git", "merge-file", "-L", "current (local changes)", "-L", "base (original)", "-L", "new (upstream)", "--diff3", currentFile, baseFile, newFile) output, err := cmd.CombinedOutput() - - // git merge-file returns: - // - 0 if merge was successful without conflicts - // - >0 if conflicts were found (appears to return number of conflicts, but file is still updated) - // The exit code can be >1 for multiple conflicts, not just errors - hasConflicts := false - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - exitCode := exitErr.ExitCode() - if exitCode > 0 && exitCode < 128 { - // Conflicts found (exit codes 1-127 indicate conflicts) - // Exit codes >= 128 typically indicate system errors - hasConflicts = true - updateMergeLog.Printf("Merge conflicts detected: exit_code=%d", exitCode) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Merge conflicts detected (exit code: %d)", exitCode))) - } - } else { - // Real error (exit code >= 128) - updateMergeLog.Printf("Git merge-file failed: exit_code=%d", exitCode) - return "", false, fmt.Errorf("git merge-file failed: %w\nOutput: %s", err, output) - } - } else { - return "", false, fmt.Errorf("failed to execute git merge-file: %w", err) - } + if err == nil { + return false, nil } - - updateMergeLog.Printf("Merge completed: has_conflicts=%v", hasConflicts) - - // Read the merged content from the current file (git merge-file updates it in-place) - mergedContent, err := os.ReadFile(currentFile) - if err != nil { - return "", false, fmt.Errorf("failed to read merged content: %w", err) + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + return false, fmt.Errorf("failed to execute git merge-file: %w", err) } + exitCode := exitErr.ExitCode() + if exitCode <= 0 || exitCode >= 128 { + updateMergeLog.Printf("Git merge-file failed: exit_code=%d", exitCode) + return false, fmt.Errorf("git merge-file failed: %w\nOutput: %s", err, output) + } + updateMergeLog.Printf("Merge conflicts detected: exit_code=%d", exitCode) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Merge conflicts detected (exit code: %d)", exitCode))) + } + return true, nil +} - mergedStr := string(mergedContent) - - // Process @include directives if present and no conflicts - // Skip include processing if there are conflicts to avoid errors - if !hasConflicts { - workflow := &WorkflowSpec{ - RepoSpec: RepoSpec{ - RepoSlug: parsedNewSourceSpec.Repo, - Version: newRef, - }, - WorkflowPath: parsedNewSourceSpec.Path, - } - - localWorkflowDir := "" - if localWorkflowPath != "" { - localWorkflowDir = filepath.Dir(localWorkflowPath) - } - processedContent, err := processIncludesInContent(mergedStr, workflow, newRef, localWorkflowDir, verbose) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to process includes: %v", err))) - } - // Return unprocessed content on error - } else { - mergedStr = processedContent +func mergeWorkflowContentProcessIncludes(mergedStr string, hasConflicts bool, parsedNewSourceSpec *SourceSpec, newRef, localWorkflowPath string, verbose bool) string { + if hasConflicts { + return mergedStr + } + workflow := &WorkflowSpec{ + RepoSpec: RepoSpec{ + RepoSlug: parsedNewSourceSpec.Repo, + Version: newRef, + }, + WorkflowPath: parsedNewSourceSpec.Path, + } + localWorkflowDir := "" + if localWorkflowPath != "" { + localWorkflowDir = filepath.Dir(localWorkflowPath) + } + processedContent, err := processIncludesInContent(mergedStr, workflow, newRef, localWorkflowDir, verbose) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to process includes: %v", err))) } + return mergedStr } - - return mergedStr, hasConflicts, nil + return processedContent } diff --git a/pkg/cli/update_org.go b/pkg/cli/update_org.go index cdf46cd45eb..62752f4f202 100644 --- a/pkg/cli/update_org.go +++ b/pkg/cli/update_org.go @@ -166,21 +166,11 @@ func renderOrgPreviewReport(previewByRepo []orgRepoPreview, applying bool) { } func previewOrgRepoUpdates(ctx context.Context, repo string, opts UpdateWorkflowsOptions, verbose bool) (orgRepoPreview, error) { - gitRoot, err := gitutil.FindGitRoot() - if err != nil { - return orgRepoPreview{}, fmt.Errorf("--org requires running inside a git repository: %w", err) - } - - updatesDir, err := ensureUpdateTargetRepoGitignore(gitRoot) + checkoutDir, err := previewOrgRepoUpdatesCheckout(ctx, repo) if err != nil { return orgRepoPreview{}, err } - checkoutDir := filepath.Join(updatesDir, sanitizeRepoPath(repo)) - if err := shallowCloneTargetRepo(ctx, repo, checkoutDir); err != nil { - return orgRepoPreview{}, err - } - workflowsDir := filepath.Join(checkoutDir, constants.GetWorkflowDir()) workflows, err := findWorkflowsWithSource(workflowsDir, opts.WorkflowNames, verbose) if err != nil { @@ -193,46 +183,75 @@ func previewOrgRepoUpdates(ctx context.Context, repo string, opts UpdateWorkflow Workflows: make([]orgWorkflowPreview, 0, len(workflows)), } for _, wf := range workflows { - sourceSpec, err := parseSourceSpec(wf.SourceSpec) - if err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s/%s: failed to parse source: %v", repo, wf.Name, err))) - orgUpdateLog.Printf("Failed to parse source for %s/%s: %v", repo, wf.Name, err) + item, ok := previewOrgRepoUpdatesWorkflow(ctx, repo, checkoutDir, wf, opts, verbose, &preview) + if !ok { continue } - name := normalizeWorkflowID(wf.Name) - resolved, err := resolveRedirectedUpdateLocation(ctx, name, sourceSpec, opts.AllowMajor, verbose, opts.NoRedirect, opts.CoolDown) - if err != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s/%s: %v", repo, wf.Name, err))) - orgUpdateLog.Printf("Failed to resolve update location for %s/%s: %v", repo, wf.Name, err) - continue - } - if resolved.currentRef == resolved.latestRef && len(resolved.redirectHistory) == 0 && !opts.Force { - continue - } - - editedAt, err := getLatestWorkflowEditTimeFromCheckout(ctx, checkoutDir, wf.Path) - if err == nil { - if preview.OldestEdit.IsZero() || editedAt.Before(preview.OldestEdit) { - preview.OldestEdit = editedAt - } - } - // Resolve SHAs to version-tag labels so the preview shows human-readable - // identifiers (e.g. v1.2.3) instead of bare short SHAs when possible. - currentLabel := resolveVersionLabel(ctx, resolved.sourceSpec.Repo, resolved.currentRef) - latestLabel := resolveVersionLabel(ctx, resolved.sourceSpec.Repo, resolved.latestRef) - preview.Workflows = append(preview.Workflows, orgWorkflowPreview{ - Name: name, - Path: wf.Path, - CurrentRef: currentLabel, - LatestRef: latestLabel, - Redirected: len(resolved.redirectHistory) > 0, - EditedAt: editedAt, - }) + preview.Workflows = append(preview.Workflows, item) } return preview, nil } +func previewOrgRepoUpdatesCheckout(ctx context.Context, repo string) (string, error) { + gitRoot, err := gitutil.FindGitRoot() + if err != nil { + return "", fmt.Errorf("--org requires running inside a git repository: %w", err) + } + + updatesDir, err := ensureUpdateTargetRepoGitignore(gitRoot) + if err != nil { + return "", err + } + + checkoutDir := filepath.Join(updatesDir, sanitizeRepoPath(repo)) + if err := shallowCloneTargetRepo(ctx, repo, checkoutDir); err != nil { + return "", err + } + return checkoutDir, nil +} + +func previewOrgRepoUpdatesWorkflow(ctx context.Context, repo, checkoutDir string, wf *workflowWithSource, opts UpdateWorkflowsOptions, verbose bool, preview *orgRepoPreview) (orgWorkflowPreview, bool) { + sourceSpec, err := parseSourceSpec(wf.SourceSpec) + if err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s/%s: failed to parse source: %v", repo, wf.Name, err))) + orgUpdateLog.Printf("Failed to parse source for %s/%s: %v", repo, wf.Name, err) + return orgWorkflowPreview{}, false + } + name := normalizeWorkflowID(wf.Name) + resolved, err := resolveRedirectedUpdateLocation(ctx, name, sourceSpec, opts.AllowMajor, verbose, opts.NoRedirect, opts.CoolDown) + if err != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s/%s: %v", repo, wf.Name, err))) + orgUpdateLog.Printf("Failed to resolve update location for %s/%s: %v", repo, wf.Name, err) + return orgWorkflowPreview{}, false + } + if resolved.currentRef == resolved.latestRef && len(resolved.redirectHistory) == 0 && !opts.Force { + return orgWorkflowPreview{}, false + } + + editedAt := previewOrgRepoUpdatesEditedAt(ctx, checkoutDir, wf.Path, preview) + currentLabel := resolveVersionLabel(ctx, resolved.sourceSpec.Repo, resolved.currentRef) + latestLabel := resolveVersionLabel(ctx, resolved.sourceSpec.Repo, resolved.latestRef) + return orgWorkflowPreview{ + Name: name, + Path: wf.Path, + CurrentRef: currentLabel, + LatestRef: latestLabel, + Redirected: len(resolved.redirectHistory) > 0, + EditedAt: editedAt, + }, true +} + +func previewOrgRepoUpdatesEditedAt(ctx context.Context, checkoutDir, workflowPath string, preview *orgRepoPreview) time.Time { + editedAt, err := getLatestWorkflowEditTimeFromCheckout(ctx, checkoutDir, workflowPath) + if err == nil { + if preview.OldestEdit.IsZero() || editedAt.Before(preview.OldestEdit) { + preview.OldestEdit = editedAt + } + } + return editedAt +} + func getLatestWorkflowEditTimeFromCheckout(ctx context.Context, checkoutDir, workflowPath string) (time.Time, error) { relativePath := workflowPath if filepath.IsAbs(workflowPath) { diff --git a/pkg/cli/update_redirects.go b/pkg/cli/update_redirects.go index b490c4297b2..9bdb910e976 100644 --- a/pkg/cli/update_redirects.go +++ b/pkg/cli/update_redirects.go @@ -40,74 +40,109 @@ func resolveRedirectedUpdateLocation(ctx context.Context, workflowName string, i history := make([]string, 0, 2) for range maxRedirectDepth { - currentRef := current.Ref - if currentRef == "" { - currentRef = resolveDefaultBranchRef(ctx, current.Repo) - } - - locationKey := sourceSpecWithRef(current, currentRef) - if _, exists := visited[locationKey]; exists { - updateRedirectsLog.Printf("Redirect loop detected: workflow=%s, location=%s", workflowName, locationKey) - return nil, fmt.Errorf("redirect loop detected while updating %s at %s", workflowName, locationKey) - } - visited[locationKey] = struct{}{} - - latestRef, err := resolveLatestRefFn(ctx, current.Repo, currentRef, allowMajor, verbose, coolDown) - if err != nil { - return nil, fmt.Errorf("failed to resolve latest ref for %s: %w", sourceSpecWithRef(current, currentRef), err) - } - - content, err := downloadWorkflowContentFn(ctx, current.Repo, current.Path, latestRef, verbose) - if err != nil { - return nil, fmt.Errorf("failed to download workflow %s: %w", sourceSpecWithRef(current, latestRef), err) - } - - redirect, err := extractRedirectFromContent(string(content)) + result, next, err := resolveRedirectedUpdateLocationStep(resolveRedirectedUpdateLocationStepParams{ + Ctx: ctx, + WorkflowName: workflowName, + Current: current, + Visited: visited, + History: history, + AllowMajor: allowMajor, + Verbose: verbose, + NoRedirect: noRedirect, + CoolDown: coolDown, + }) if err != nil { return nil, err } - - sourceFieldRef := latestRef - if isBranchRef(currentRef) { - sourceFieldRef = currentRef + if result != nil { + return result, nil } + history = append(history, next.message) + current = next.source + } - if redirect == "" { - updateRedirectsLog.Printf("Resolved update location: workflow=%s, repo=%s, latestRef=%s, redirects=%d", workflowName, current.Repo, latestRef, len(history)) - return &resolvedUpdateLocation{ - sourceSpec: current, - currentRef: currentRef, - latestRef: latestRef, - sourceFieldRef: sourceFieldRef, - content: content, - redirectHistory: history, - }, nil - } + updateRedirectsLog.Printf("Redirect chain exceeded max depth: workflow=%s, depth=%d", workflowName, maxRedirectDepth) + return nil, fmt.Errorf("redirect chain exceeded maximum depth (%d) while updating %s", maxRedirectDepth, workflowName) +} - if noRedirect { - updateRedirectsLog.Printf("Redirect blocked by --no-redirect: workflow=%s, redirect=%s", workflowName, redirect) - return nil, fmt.Errorf("redirect is disabled by --no-redirect for %s: %s declares redirect to %s (remove redirect frontmatter or run update without --no-redirect)", workflowName, sourceSpecWithRef(current, latestRef), redirect) - } +type resolveRedirectedUpdateLocationNext struct { + source *SourceSpec + message string +} - redirectedSource, err := normalizeRedirectToSourceSpec(redirect) - if err != nil { - return nil, fmt.Errorf("invalid redirect %q in %s: %w", redirect, sourceSpecWithRef(current, latestRef), err) - } +type resolveRedirectedUpdateLocationStepParams struct { + Ctx context.Context + WorkflowName string + Current *SourceSpec + Visited map[string]struct{} + History []string + AllowMajor bool + Verbose bool + NoRedirect bool + CoolDown time.Duration +} - nextRef := redirectedSource.Ref - if nextRef == "" { - nextRef = resolveDefaultBranchRef(ctx, redirectedSource.Repo) - } +func resolveRedirectedUpdateLocationStep(p resolveRedirectedUpdateLocationStepParams) (*resolvedUpdateLocation, resolveRedirectedUpdateLocationNext, error) { + currentRef := p.Current.Ref + if currentRef == "" { + currentRef = resolveDefaultBranchRef(p.Ctx, p.Current.Repo) + } + locationKey := sourceSpecWithRef(p.Current, currentRef) + if _, exists := p.Visited[locationKey]; exists { + updateRedirectsLog.Printf("Redirect loop detected: workflow=%s, location=%s", p.WorkflowName, locationKey) + return nil, resolveRedirectedUpdateLocationNext{}, fmt.Errorf("redirect loop detected while updating %s at %s", p.WorkflowName, locationKey) + } + p.Visited[locationKey] = struct{}{} - updateRedirectsLog.Printf("Following redirect: workflow=%s, from=%s, to=%s", workflowName, sourceSpecWithRef(current, latestRef), sourceSpecWithRef(redirectedSource, nextRef)) - redirectMessage := fmt.Sprintf("Workflow %s redirect: %s → %s", workflowName, sourceSpecWithRef(current, latestRef), sourceSpecWithRef(redirectedSource, nextRef)) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(redirectMessage)) - history = append(history, redirectMessage) - current = redirectedSource + latestRef, content, redirect, err := resolveRedirectedUpdateLocationFetch(p.Ctx, p.Current, currentRef, p.AllowMajor, p.Verbose, p.CoolDown) + if err != nil { + return nil, resolveRedirectedUpdateLocationNext{}, err + } + sourceFieldRef := latestRef + if isBranchRef(currentRef) { + sourceFieldRef = currentRef + } + if redirect == "" { + updateRedirectsLog.Printf("Resolved update location: workflow=%s, repo=%s, latestRef=%s, redirects=%d", p.WorkflowName, p.Current.Repo, latestRef, len(p.History)) + return &resolvedUpdateLocation{sourceSpec: p.Current, currentRef: currentRef, latestRef: latestRef, sourceFieldRef: sourceFieldRef, content: content, redirectHistory: p.History}, resolveRedirectedUpdateLocationNext{}, nil } + next, err := resolveRedirectedUpdateLocationRedirect(p.Ctx, p.WorkflowName, p.Current, latestRef, redirect, p.NoRedirect) + return nil, next, err +} - updateRedirectsLog.Printf("Redirect chain exceeded max depth: workflow=%s, depth=%d", workflowName, maxRedirectDepth) - return nil, fmt.Errorf("redirect chain exceeded maximum depth (%d) while updating %s", maxRedirectDepth, workflowName) +func resolveRedirectedUpdateLocationFetch(ctx context.Context, current *SourceSpec, currentRef string, allowMajor, verbose bool, coolDown time.Duration) (string, []byte, string, error) { + latestRef, err := resolveLatestRefFn(ctx, current.Repo, currentRef, allowMajor, verbose, coolDown) + if err != nil { + return "", nil, "", fmt.Errorf("failed to resolve latest ref for %s: %w", sourceSpecWithRef(current, currentRef), err) + } + content, err := downloadWorkflowContentFn(ctx, current.Repo, current.Path, latestRef, verbose) + if err != nil { + return "", nil, "", fmt.Errorf("failed to download workflow %s: %w", sourceSpecWithRef(current, latestRef), err) + } + redirect, err := extractRedirectFromContent(string(content)) + if err != nil { + return "", nil, "", err + } + return latestRef, content, redirect, nil +} + +func resolveRedirectedUpdateLocationRedirect(ctx context.Context, workflowName string, current *SourceSpec, latestRef, redirect string, noRedirect bool) (resolveRedirectedUpdateLocationNext, error) { + if noRedirect { + updateRedirectsLog.Printf("Redirect blocked by --no-redirect: workflow=%s, redirect=%s", workflowName, redirect) + return resolveRedirectedUpdateLocationNext{}, fmt.Errorf("redirect is disabled by --no-redirect for %s: %s declares redirect to %s (remove redirect frontmatter or run update without --no-redirect)", workflowName, sourceSpecWithRef(current, latestRef), redirect) + } + redirectedSource, err := normalizeRedirectToSourceSpec(redirect) + if err != nil { + return resolveRedirectedUpdateLocationNext{}, fmt.Errorf("invalid redirect %q in %s: %w", redirect, sourceSpecWithRef(current, latestRef), err) + } + nextRef := redirectedSource.Ref + if nextRef == "" { + nextRef = resolveDefaultBranchRef(ctx, redirectedSource.Repo) + } + updateRedirectsLog.Printf("Following redirect: workflow=%s, from=%s, to=%s", workflowName, sourceSpecWithRef(current, latestRef), sourceSpecWithRef(redirectedSource, nextRef)) + redirectMessage := fmt.Sprintf("Workflow %s redirect: %s → %s", workflowName, sourceSpecWithRef(current, latestRef), sourceSpecWithRef(redirectedSource, nextRef)) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(redirectMessage)) + return resolveRedirectedUpdateLocationNext{source: redirectedSource, message: redirectMessage}, nil } // resolveDefaultBranchRef returns the repository's default branch name via the diff --git a/pkg/cli/update_workflows.go b/pkg/cli/update_workflows.go index d1a2abc2bdb..c4e9901cdeb 100644 --- a/pkg/cli/update_workflows.go +++ b/pkg/cli/update_workflows.go @@ -76,76 +76,91 @@ func UpdateWorkflows(ctx context.Context, opts UpdateWorkflowsOptions) error { clearUpdateResolutionCaches() updateLog.Printf("Scanning for workflows with source field: dir=%s, filter=%v, noMerge=%v, noCompile=%v, noRedirect=%v, disableSecurityScanner=%v, coolDown=%v", opts.WorkflowsDir, opts.WorkflowNames, opts.NoMerge, opts.NoCompile, opts.NoRedirect, opts.DisableSecurityScanner, opts.CoolDown) - // Use provided workflows directory or default - workflowsDir := opts.WorkflowsDir - if workflowsDir == "" { - workflowsDir = getWorkflowsDir() - } - // Find all workflows with source field + workflowsDir := updateWorkflowsDir(opts.WorkflowsDir) workflows, err := findWorkflowsWithSource(workflowsDir, opts.WorkflowNames, opts.Verbose) if err != nil { return err } - - updateLog.Printf("Found %d workflows with source field", len(workflows)) - - if len(workflows) == 0 { - if len(opts.WorkflowNames) > 0 { - return errors.New("no workflows found matching the specified names with source field") - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("no workflows found with source field")) - return nil + if ok, err := updateWorkflowsValidateFound(workflows, opts.WorkflowNames); err != nil || !ok { + return err } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d workflow(s) to update", len(workflows)))) - // Track update results var successfulUpdates []string var failedUpdates []updateFailure + manifestGroups, directWorkflows := updateWorkflowsPartition(workflows, &failedUpdates) + successfulUpdates, failedUpdates = updateWorkflowsDirect(ctx, directWorkflows, opts, successfulUpdates, failedUpdates) + successfulUpdates, failedUpdates = updateWorkflowsManifestGroups(ctx, manifestGroups, opts, successfulUpdates, failedUpdates) + + // Show summary + showUpdateSummary(successfulUpdates, failedUpdates) + return updateWorkflowsResult(successfulUpdates, failedUpdates) +} + +func updateWorkflowsDir(workflowsDir string) string { + if workflowsDir == "" { + return getWorkflowsDir() + } + return workflowsDir +} + +func updateWorkflowsValidateFound(workflows []*workflowWithSource, filterNames []string) (bool, error) { + updateLog.Printf("Found %d workflows with source field", len(workflows)) + if len(workflows) > 0 { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d workflow(s) to update", len(workflows)))) + return true, nil + } + if len(filterNames) > 0 { + return false, errors.New("no workflows found matching the specified names with source field") + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("no workflows found with source field")) + return false, nil +} + +func updateWorkflowsPartition(workflows []*workflowWithSource, failedUpdates *[]updateFailure) (map[string][]*workflowWithSource, []*workflowWithSource) { manifestGroups := make(map[string][]*workflowWithSource) var directWorkflows []*workflowWithSource for _, wf := range workflows { if _, ok, err := parseManifestSourceSpec(wf.SourceSpec); err != nil { - failedUpdates = append(failedUpdates, updateFailure{ - Name: wf.Name, - Error: err.Error(), - }) + *failedUpdates = append(*failedUpdates, updateFailure{Name: wf.Name, Error: err.Error()}) continue } else if ok { - manifestGroups[strings.TrimSpace(wf.SourceSpec)] = append(manifestGroups[strings.TrimSpace(wf.SourceSpec)], wf) + source := strings.TrimSpace(wf.SourceSpec) + manifestGroups[source] = append(manifestGroups[source], wf) continue } directWorkflows = append(directWorkflows, wf) } + return manifestGroups, directWorkflows +} - // Update each workflow +func updateWorkflowsDirect(ctx context.Context, directWorkflows []*workflowWithSource, opts UpdateWorkflowsOptions, successfulUpdates []string, failedUpdates []updateFailure) ([]string, []updateFailure) { for _, wf := range directWorkflows { updateLog.Printf("Updating workflow: %s (source: %s)", wf.Name, wf.SourceSpec) if err := updateWorkflow(ctx, wf, opts); err != nil { updateLog.Printf("Failed to update workflow %s: %v", wf.Name, err) - failedUpdates = append(failedUpdates, updateFailure{ - Name: wf.Name, - Error: err.Error(), - }) + failedUpdates = append(failedUpdates, updateFailure{Name: wf.Name, Error: err.Error()}) continue } updateLog.Printf("Successfully updated workflow: %s", wf.Name) successfulUpdates = append(successfulUpdates, wf.Name) } + return successfulUpdates, failedUpdates +} + +func updateWorkflowsManifestGroups(ctx context.Context, manifestGroups map[string][]*workflowWithSource, opts UpdateWorkflowsOptions, successfulUpdates []string, failedUpdates []updateFailure) ([]string, []updateFailure) { for source, grouped := range manifestGroups { groupSuccesses, groupFailures := updateManifestWorkflowGroup(ctx, source, grouped, opts) successfulUpdates = append(successfulUpdates, groupSuccesses...) failedUpdates = append(failedUpdates, groupFailures...) } + return successfulUpdates, failedUpdates +} - // Show summary - showUpdateSummary(successfulUpdates, failedUpdates) - +func updateWorkflowsResult(successfulUpdates []string, failedUpdates []updateFailure) error { if len(successfulUpdates) == 0 { - // If all failures were due to GitHub API rate limiting, treat as non-fatal. - // Rate limiting is a transient infrastructure condition, not a code error. if len(failedUpdates) > 0 && allFailuresAreRateLimited(failedUpdates) { fmt.Fprintln(os.Stderr, console.FormatWarningMessage("All workflow updates skipped due to GitHub API rate limiting")) return nil @@ -181,77 +196,79 @@ func findWorkflowsWithSource(workflowsDir string, filterNames []string, verbose updateLog.Printf("Found %d entries in workflows directory", len(entries)) for _, entry := range entries { - if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") { - continue + workflow, ok := findWorkflowsWithSourceEntry(workflowsDir, entry, filterNames, verbose) + if ok { + workflows = append(workflows, workflow) } + } - // Skip .lock.yml files - if strings.HasSuffix(entry.Name(), ".lock.yml") { - continue - } + return workflows, nil +} - workflowPath := filepath.Join(workflowsDir, entry.Name()) - workflowName := normalizeWorkflowID(entry.Name()) +func findWorkflowsWithSourceEntry(workflowsDir string, entry os.DirEntry, filterNames []string, verbose bool) (*workflowWithSource, bool) { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".md") || strings.HasSuffix(entry.Name(), ".lock.yml") { + return nil, false + } - // Filter by name if specified - if len(filterNames) > 0 { - matched := false - for _, filterName := range filterNames { - // Normalize filter name to handle both "workflow" and "workflow.md" formats - filterName = normalizeWorkflowID(filterName) - if workflowName == filterName { - matched = true - break - } - } - if !matched { - continue - } - } + workflowPath := filepath.Join(workflowsDir, entry.Name()) + workflowName := normalizeWorkflowID(entry.Name()) + if !findWorkflowsWithSourceMatchesFilter(workflowName, filterNames) { + return nil, false + } - // Read the workflow file and extract source field - content, err := os.ReadFile(workflowPath) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read %s: %v", workflowPath, err))) - } - continue - } + source, ok := findWorkflowsWithSourceRead(workflowPath, workflowName, verbose) + if !ok { + return nil, false + } + return &workflowWithSource{ + Name: workflowName, + Path: workflowPath, + SourceSpec: strings.TrimSpace(source), + }, true +} - // Parse frontmatter - result, err := parser.ExtractFrontmatterFromContent(string(content)) - if err != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse frontmatter in %s: %v", workflowPath, err))) - } - continue +func findWorkflowsWithSourceMatchesFilter(workflowName string, filterNames []string) bool { + if len(filterNames) == 0 { + return true + } + for _, filterName := range filterNames { + if workflowName == normalizeWorkflowID(filterName) { + return true } + } + return false +} - // Check for source field - sourceRaw, ok := result.Frontmatter["source"] - if !ok { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Skipping %s: no source field", workflowName))) - } - continue +func findWorkflowsWithSourceRead(workflowPath, workflowName string, verbose bool) (string, bool) { + content, err := os.ReadFile(workflowPath) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read %s: %v", workflowPath, err))) } - - source, ok := sourceRaw.(string) - if !ok || source == "" { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: invalid source field", workflowName))) - } - continue + return "", false + } + result, err := parser.ExtractFrontmatterFromContent(string(content)) + if err != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to parse frontmatter in %s: %v", workflowPath, err))) } - - workflows = append(workflows, &workflowWithSource{ - Name: workflowName, - Path: workflowPath, - SourceSpec: strings.TrimSpace(source), - }) + return "", false } - - return workflows, nil + sourceRaw, ok := result.Frontmatter["source"] + if !ok { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Skipping %s: no source field", workflowName))) + } + return "", false + } + source, ok := sourceRaw.(string) + if !ok || source == "" { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Skipping %s: invalid source field", workflowName))) + } + return "", false + } + return source, true } // resolveLatestRef resolves the latest ref for a workflow source @@ -500,91 +517,99 @@ func resolveLatestReleaseWithDeps(ctx context.Context, deps workflowUpdateDeps, fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Checking for latest release (current: %s, allow major: %v)", currentRef, allowMajor))) } - // Get all releases using gh CLI - output, err := deps.runReleasesAPI(ctx, repo) + releases, err := resolveLatestReleaseWithDepsFetch(ctx, deps, repo) if err != nil { - return "", fmt.Errorf("failed to fetch releases: %w", err) - } - - releases := strings.Split(strings.TrimSpace(string(output)), "\n") - if len(releases) == 0 || releases[0] == "" { - return "", errors.New("no releases found") + return "", err } // Parse current version currentVer := parseVersion(currentRef) if currentVer == nil { - // If current version is not a valid semantic version, select the latest stable release - // by semantic version so we are not sensitive to the ordering of the API response. - var latestStable string - var latestStableVersion *semverutil.SemanticVersion - - for _, release := range releases { - releaseVer := parseVersion(release) - if releaseVer == nil || releaseVer.Pre != "" { - continue - } - if latestStableVersion == nil || releaseVer.IsNewer(latestStableVersion) { - latestStable = release - latestStableVersion = releaseVer - } - } + return resolveLatestReleaseWithDepsLatestStable(repo, releases, verbose) + } - if latestStable == "" { - return "", fmt.Errorf("no stable releases found for %s", repo) - } + latestCompatible, err := resolveLatestReleaseWithDepsCompatible(releases, currentVer, allowMajor) + if err != nil { + return "", err + } - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Current version is not valid, using latest stable release: "+latestStable)) - } + if resolveLatestReleaseWithDepsInCoolDown(ctx, repo, currentRef, latestCompatible, coolDown) { + return currentRef, nil + } - return latestStable, nil + if verbose && latestCompatible != currentRef { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found newer release: "+latestCompatible)) } - // Find the latest compatible non-prerelease release. - // Per semver rules, v1.1.0-beta.1 > v1.0.0, so without this filter a prerelease - // of a higher base version could be incorrectly selected as the upgrade target. - var latestCompatible string - var latestCompatibleVersion *semverutil.SemanticVersion + return latestCompatible, nil +} +func resolveLatestReleaseWithDepsFetch(ctx context.Context, deps workflowUpdateDeps, repo string) ([]string, error) { + output, err := deps.runReleasesAPI(ctx, repo) + if err != nil { + return nil, fmt.Errorf("failed to fetch releases: %w", err) + } + releases := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(releases) == 0 || releases[0] == "" { + return nil, errors.New("no releases found") + } + return releases, nil +} + +func resolveLatestReleaseWithDepsLatestStable(repo string, releases []string, verbose bool) (string, error) { + var latestStable string + var latestStableVersion *semverutil.SemanticVersion for _, release := range releases { releaseVer := parseVersion(release) if releaseVer == nil || releaseVer.Pre != "" { continue } + if latestStableVersion == nil || releaseVer.IsNewer(latestStableVersion) { + latestStable = release + latestStableVersion = releaseVer + } + } + if latestStable == "" { + return "", fmt.Errorf("no stable releases found for %s", repo) + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Current version is not valid, using latest stable release: "+latestStable)) + } + return latestStable, nil +} - // Check if compatible based on major version +func resolveLatestReleaseWithDepsCompatible(releases []string, currentVer *semverutil.SemanticVersion, allowMajor bool) (string, error) { + var latestCompatible string + var latestCompatibleVersion *semverutil.SemanticVersion + for _, release := range releases { + releaseVer := parseVersion(release) + if releaseVer == nil || releaseVer.Pre != "" { + continue + } if !allowMajor && releaseVer.Major != currentVer.Major { continue } - - // Check if this is newer than what we have if latestCompatibleVersion == nil || releaseVer.IsNewer(latestCompatibleVersion) { latestCompatible = release latestCompatibleVersion = releaseVer } } - if latestCompatible == "" { return "", errors.New("no compatible release found") } + return latestCompatible, nil +} - // Apply cooldown: if the latest release is newer than the current and the repo is not - // exempt from cooldown, check whether the release is recent enough to be held back. - if latestCompatible != currentRef && !isExemptFromCoolDown(repo) { - if result := checkReleaseCoolDown(ctx, repo, latestCompatible, coolDown); result.InCoolDown { - cooldownLog.Printf("Workflow source %s: %s", repo, result.Message) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping update for %s: %s", repo, result.Message))) - // Return the current ref — no update until cooldown expires. - return currentRef, nil - } +func resolveLatestReleaseWithDepsInCoolDown(ctx context.Context, repo, currentRef, latestCompatible string, coolDown time.Duration) bool { + if latestCompatible == currentRef || isExemptFromCoolDown(repo) { + return false } - - if verbose && latestCompatible != currentRef { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found newer release: "+latestCompatible)) + if result := checkReleaseCoolDown(ctx, repo, latestCompatible, coolDown); result.InCoolDown { + cooldownLog.Printf("Workflow source %s: %s", repo, result.Message) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Skipping update for %s: %s", repo, result.Message))) + return true } - - return latestCompatible, nil + return false } // updateWorkflow updates a single workflow from its source @@ -597,236 +622,225 @@ func updateWorkflow(ctx context.Context, wf *workflowWithSource, opts UpdateWork fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Source: "+wf.SourceSpec)) } - // Parse source spec - initialSourceSpec, err := parseSourceSpec(wf.SourceSpec) + state, err := updateWorkflowResolve(ctx, wf, opts) if err != nil { - updateLog.Printf("Failed to parse source spec: %v", err) - return fmt.Errorf("failed to parse source spec: %w", err) + return err } - - resolvedLocation, err := resolveRedirectedUpdateLocation(ctx, wf.Name, initialSourceSpec, opts.AllowMajor, opts.Verbose, opts.NoRedirect, opts.CoolDown) + if done, err := updateWorkflowAlreadyUpToDate(ctx, state); done || err != nil { + return err + } + finalContent, hasConflicts, err := updateWorkflowContent(ctx, state) if err != nil { return err } + finalContent = updateWorkflowStopAfter(finalContent, opts) + if err := updateWorkflowSecurityScan(wf, finalContent, opts); err != nil { + return err + } + return updateWorkflowWriteCompile(ctx, state, finalContent, hasConflicts) +} - sourceSpec := resolvedLocation.sourceSpec - currentRef := resolvedLocation.currentRef - latestRef := resolvedLocation.latestRef - sourceFieldRef := resolvedLocation.sourceFieldRef - newContent := resolvedLocation.content +type updateWorkflowState struct { + wf *workflowWithSource + opts UpdateWorkflowsOptions + sourceSpec *SourceSpec + currentRef string + latestRef string + sourceFieldRef string + newContent []byte + redirectHistory []string +} +func updateWorkflowResolve(ctx context.Context, wf *workflowWithSource, opts UpdateWorkflowsOptions) (*updateWorkflowState, error) { + initialSourceSpec, err := parseSourceSpec(wf.SourceSpec) + if err != nil { + updateLog.Printf("Failed to parse source spec: %v", err) + return nil, fmt.Errorf("failed to parse source spec: %w", err) + } + resolvedLocation, err := resolveRedirectedUpdateLocation(ctx, wf.Name, initialSourceSpec, opts.AllowMajor, opts.Verbose, opts.NoRedirect, opts.CoolDown) + if err != nil { + return nil, err + } if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Current ref: "+currentRef)) - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Latest ref: "+latestRef)) + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Current ref: "+resolvedLocation.currentRef)) + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Latest ref: "+resolvedLocation.latestRef)) } + return &updateWorkflowState{wf: wf, opts: opts, sourceSpec: resolvedLocation.sourceSpec, currentRef: resolvedLocation.currentRef, latestRef: resolvedLocation.latestRef, sourceFieldRef: resolvedLocation.sourceFieldRef, newContent: resolvedLocation.content, redirectHistory: resolvedLocation.redirectHistory}, nil +} - // Check if update is needed - if !opts.Force && currentRef == latestRef && len(resolvedLocation.redirectHistory) == 0 { - updateLog.Printf("Workflow already at latest ref: %s, checking for local modifications", currentRef) - - // Download the source content to check if local file has been modified - sourceContent, err := downloadWorkflowContentFn(ctx, sourceSpec.Repo, sourceSpec.Path, currentRef, opts.Verbose) - if err != nil { - // If we can't download for comparison, just show the up-to-date message - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to download source for comparison: %v", err))) - } - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow %s is already up to date (%s)", wf.Name, shortRef(currentRef)))) - return nil - } - - // Read current workflow content - currentContent, err := os.ReadFile(wf.Path) - if err != nil { - return fmt.Errorf("failed to read current workflow: %w", err) +func updateWorkflowAlreadyUpToDate(ctx context.Context, state *updateWorkflowState) (bool, error) { + if state.opts.Force || state.currentRef != state.latestRef || len(state.redirectHistory) > 0 { + if len(state.redirectHistory) > 0 { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Workflow %s source location changed; updating source to %s/%s@%s", state.wf.Name, state.sourceSpec.Repo, state.sourceSpec.Path, state.sourceFieldRef))) } - - // Check if local file differs from source - if hasLocalModifications(string(sourceContent), string(currentContent), wf.SourceSpec, filepath.Dir(wf.Path), opts.Verbose) { - updateLog.Printf("Local modifications detected in workflow: %s", wf.Name) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow %s is already up to date (%s)", wf.Name, shortRef(currentRef)))) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("⚠️ Local copy of %s has been modified from source", wf.Name))) - return nil + return false, nil + } + updateLog.Printf("Workflow already at latest ref: %s, checking for local modifications", state.currentRef) + sourceContent, err := downloadWorkflowContentFn(ctx, state.sourceSpec.Repo, state.sourceSpec.Path, state.currentRef, state.opts.Verbose) + if err != nil { + if state.opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to download source for comparison: %v", err))) } - - updateLog.Printf("Workflow %s is up to date with no local modifications", wf.Name) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow %s is already up to date (%s)", wf.Name, shortRef(currentRef)))) - return nil + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow %s is already up to date (%s)", state.wf.Name, shortRef(state.currentRef)))) + return true, nil } - - if len(resolvedLocation.redirectHistory) > 0 { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Workflow %s source location changed; updating source to %s/%s@%s", wf.Name, sourceSpec.Repo, sourceSpec.Path, sourceFieldRef))) + currentContent, err := os.ReadFile(state.wf.Path) + if err != nil { + return true, fmt.Errorf("failed to read current workflow: %w", err) } - - // Determine merge mode. Merge is the default behaviour — it detects - // local modifications and performs a 3-way merge to preserve them. - // When --no-merge is used, local changes are overridden with upstream. - merge := !opts.NoMerge - if len(resolvedLocation.redirectHistory) > 0 { - merge = false + if hasLocalModifications(string(sourceContent), string(currentContent), state.wf.SourceSpec, filepath.Dir(state.wf.Path), state.opts.Verbose) { + updateLog.Printf("Local modifications detected in workflow: %s", state.wf.Name) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow %s is already up to date (%s)", state.wf.Name, shortRef(state.currentRef)))) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("⚠️ Local copy of %s has been modified from source", state.wf.Name))) + return true, nil } + updateLog.Printf("Workflow %s is up to date with no local modifications", state.wf.Name) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Workflow %s is already up to date (%s)", state.wf.Name, shortRef(state.currentRef)))) + return true, nil +} - // When merge mode is on, detect local modifications to confirm we - // actually need to merge (if no local mods, override is fine either way). +func updateWorkflowContent(ctx context.Context, state *updateWorkflowState) (string, bool, error) { + merge := updateWorkflowShouldMerge(ctx, state) if merge { - baseContent, dlErr := downloadWorkflowContentFn(ctx, sourceSpec.Repo, sourceSpec.Path, currentRef, opts.Verbose) - if dlErr == nil { - localContent, readErr := os.ReadFile(wf.Path) - if readErr == nil && hasLocalModifications(string(baseContent), string(localContent), wf.SourceSpec, filepath.Dir(wf.Path), opts.Verbose) { - updateLog.Printf("Local modifications detected in %s, merging to preserve changes", wf.Name) - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Local modifications detected in %s, merging to preserve your changes", wf.Name))) - } else { - // No local modifications — no need to merge, just override - merge = false - } - } + return updateWorkflowMerge(ctx, state) } + return updateWorkflowOverride(state) +} - var finalContent string - var hasConflicts bool - - // Decide whether to merge or override - if merge { - // Merge mode: perform 3-way merge to preserve local changes - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Using merge mode to preserve local changes")) - } - - // Download the base version (current ref from source) - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Downloading base version from %s/%s@%s", sourceSpec.Repo, sourceSpec.Path, currentRef))) - } - - baseContent, err := downloadWorkflowContentFn(ctx, sourceSpec.Repo, sourceSpec.Path, currentRef, opts.Verbose) - if err != nil { - return fmt.Errorf("failed to download base workflow: %w", err) - } - - // Read current workflow content - currentContent, err := os.ReadFile(wf.Path) - if err != nil { - return fmt.Errorf("failed to read current workflow: %w", err) - } - - // Perform 3-way merge using git merge-file - updateLog.Printf("Performing 3-way merge for workflow: %s", wf.Name) - mergedContent, conflicts, err := MergeWorkflowContent(string(baseContent), string(currentContent), string(newContent), wf.SourceSpec, sourceSpecWithRef(sourceSpec, sourceFieldRef), wf.Path, opts.Verbose) - if err != nil { - updateLog.Printf("Merge failed for workflow %s: %v", wf.Name, err) - return fmt.Errorf("failed to merge workflow content: %w", err) - } +func updateWorkflowShouldMerge(ctx context.Context, state *updateWorkflowState) bool { + if state.opts.NoMerge || len(state.redirectHistory) > 0 { + return false + } + baseContent, dlErr := downloadWorkflowContentFn(ctx, state.sourceSpec.Repo, state.sourceSpec.Path, state.currentRef, state.opts.Verbose) + if dlErr != nil { + return true + } + localContent, readErr := os.ReadFile(state.wf.Path) + if readErr == nil && hasLocalModifications(string(baseContent), string(localContent), state.wf.SourceSpec, filepath.Dir(state.wf.Path), state.opts.Verbose) { + updateLog.Printf("Local modifications detected in %s, merging to preserve changes", state.wf.Name) + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Local modifications detected in %s, merging to preserve your changes", state.wf.Name))) + return true + } + return false +} - finalContent = mergedContent - hasConflicts = conflicts +func updateWorkflowMerge(ctx context.Context, state *updateWorkflowState) (string, bool, error) { + if state.opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Using merge mode to preserve local changes")) + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage(fmt.Sprintf("Downloading base version from %s/%s@%s", state.sourceSpec.Repo, state.sourceSpec.Path, state.currentRef))) + } + baseContent, err := downloadWorkflowContentFn(ctx, state.sourceSpec.Repo, state.sourceSpec.Path, state.currentRef, state.opts.Verbose) + if err != nil { + return "", false, fmt.Errorf("failed to download base workflow: %w", err) + } + currentContent, err := os.ReadFile(state.wf.Path) + if err != nil { + return "", false, fmt.Errorf("failed to read current workflow: %w", err) + } + updateLog.Printf("Performing 3-way merge for workflow: %s", state.wf.Name) + mergedContent, conflicts, err := MergeWorkflowContent(string(baseContent), string(currentContent), string(state.newContent), state.wf.SourceSpec, sourceSpecWithRef(state.sourceSpec, state.sourceFieldRef), state.wf.Path, state.opts.Verbose) + if err != nil { + updateLog.Printf("Merge failed for workflow %s: %v", state.wf.Name, err) + return "", false, fmt.Errorf("failed to merge workflow content: %w", err) + } + if conflicts { + updateLog.Printf("Merge conflicts detected in workflow: %s", state.wf.Name) + } + return mergedContent, conflicts, nil +} - if hasConflicts { - updateLog.Printf("Merge conflicts detected in workflow: %s", wf.Name) +func updateWorkflowOverride(state *updateWorkflowState) (string, bool, error) { + if state.opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Using override mode - local changes will be replaced")) + } + finalContent := string(state.newContent) + newWithUpdatedSource, err := UpdateFieldInFrontmatter(finalContent, "source", sourceSpecWithRef(state.sourceSpec, state.sourceFieldRef)) + if err != nil { + if state.opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to update source in new content: %v", err))) } } else { - // Override mode (default): replace local file with new content from source - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatVerboseMessage("Using override mode - local changes will be replaced")) - } - - // Update the source field in the new content with the new ref - newWithUpdatedSource, err := UpdateFieldInFrontmatter(string(newContent), "source", sourceSpecWithRef(sourceSpec, sourceFieldRef)) - if err != nil { - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to update source in new content: %v", err))) - } - // Continue with original new content - finalContent = string(newContent) - } else { - finalContent = newWithUpdatedSource - } - - // Process @include directives if present - workflow := &WorkflowSpec{ - RepoSpec: RepoSpec{ - RepoSlug: sourceSpec.Repo, - Version: latestRef, - }, - WorkflowPath: sourceSpec.Path, - } - - processedContent, err := processIncludesInContent(finalContent, workflow, latestRef, filepath.Dir(wf.Path), opts.Verbose) - if err != nil { - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to process includes: %v", err))) - } - // Continue with unprocessed content - } else { - finalContent = processedContent + finalContent = newWithUpdatedSource + } + workflow := &WorkflowSpec{RepoSpec: RepoSpec{RepoSlug: state.sourceSpec.Repo, Version: state.latestRef}, WorkflowPath: state.sourceSpec.Path} + processedContent, err := processIncludesInContent(finalContent, workflow, state.latestRef, filepath.Dir(state.wf.Path), state.opts.Verbose) + if err != nil { + if state.opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to process includes: %v", err))) } + } else { + finalContent = processedContent } + return finalContent, false, nil +} - // Handle stop-after field modifications +func updateWorkflowStopAfter(finalContent string, opts UpdateWorkflowsOptions) string { if opts.NoStopAfter { - // Remove stop-after field if requested cleanedContent, err := RemoveFieldFromOnTrigger(finalContent, "stop-after") if err != nil { if opts.Verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to remove stop-after field: %v", err))) } - } else { - finalContent = cleanedContent - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Removed stop-after field from workflow")) - } + return finalContent + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Removed stop-after field from workflow")) } - } else if opts.StopAfter != "" { - // Set custom stop-after value if provided + return cleanedContent + } + if opts.StopAfter != "" { updatedContent, err := SetFieldInOnTrigger(finalContent, "stop-after", opts.StopAfter) if err != nil { if opts.Verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to set stop-after field: %v", err))) } - } else { - finalContent = updatedContent - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Set stop-after field to: "+opts.StopAfter)) - } + return finalContent } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Set stop-after field to: "+opts.StopAfter)) + } + return updatedContent } + return finalContent +} - // Security scan: reject workflows containing malicious or dangerous content - if !opts.DisableSecurityScanner { - if findings := workflow.ScanMarkdownSecurity(finalContent); len(findings) > 0 { - fmt.Fprintln(os.Stderr, console.FormatErrorMessage("Security scan failed for workflow")) - fmt.Fprintln(os.Stderr, workflow.FormatSecurityFindings(findings, wf.Path)) - return fmt.Errorf("workflow '%s' failed security scan: %d issue(s) detected", wf.Name, len(findings)) - } +func updateWorkflowSecurityScan(wf *workflowWithSource, finalContent string, opts UpdateWorkflowsOptions) error { + if opts.DisableSecurityScanner { if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Security scan passed")) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Security scanning disabled")) } - } else if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Security scanning disabled")) + return nil + } + if findings := workflow.ScanMarkdownSecurity(finalContent); len(findings) > 0 { + fmt.Fprintln(os.Stderr, console.FormatErrorMessage("Security scan failed for workflow")) + fmt.Fprintln(os.Stderr, workflow.FormatSecurityFindings(findings, wf.Path)) + return fmt.Errorf("workflow '%s' failed security scan: %d issue(s) detected", wf.Name, len(findings)) + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Security scan passed")) } + return nil +} - // Write updated content - if err := os.WriteFile(wf.Path, []byte(finalContent), constants.FilePermPublic); err != nil { +func updateWorkflowWriteCompile(ctx context.Context, state *updateWorkflowState, finalContent string, hasConflicts bool) error { + if err := os.WriteFile(state.wf.Path, []byte(finalContent), constants.FilePermPublic); err != nil { return fmt.Errorf("failed to write updated workflow: %w", err) } - if hasConflicts { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Updated %s from %s to %s with CONFLICTS - please review and resolve manually", wf.Name, shortRef(currentRef), shortRef(latestRef)))) - return nil // Not an error, but user needs to resolve conflicts + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Updated %s from %s to %s with CONFLICTS - please review and resolve manually", state.wf.Name, shortRef(state.currentRef), shortRef(state.latestRef)))) + return nil } - - updateLog.Printf("Successfully updated workflow %s from %s to %s", wf.Name, currentRef, latestRef) - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Updated %s from %s to %s", wf.Name, shortRef(currentRef), shortRef(latestRef)))) - - // Compile the updated workflow with refreshStopTime enabled (unless --no-compile is set) - if !opts.NoCompile { - updateLog.Printf("Compiling updated workflow: %s", wf.Name) - if err := compileWorkflowWithRefresh(ctx, wf.Path, opts.Verbose, false, opts.EngineOverride, true, opts.Approve); err != nil { - updateLog.Printf("Compilation failed for workflow %s: %v", wf.Name, err) + updateLog.Printf("Successfully updated workflow %s from %s to %s", state.wf.Name, state.currentRef, state.latestRef) + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("Updated %s from %s to %s", state.wf.Name, shortRef(state.currentRef), shortRef(state.latestRef)))) + if !state.opts.NoCompile { + updateLog.Printf("Compiling updated workflow: %s", state.wf.Name) + if err := compileWorkflowWithRefresh(ctx, state.wf.Path, state.opts.Verbose, false, state.opts.EngineOverride, true, state.opts.Approve); err != nil { + updateLog.Printf("Compilation failed for workflow %s: %v", state.wf.Name, err) return fmt.Errorf("failed to compile updated workflow: %w", err) } } else { - updateLog.Printf("Skipping compilation of workflow %s (--no-compile specified)", wf.Name) + updateLog.Printf("Skipping compilation of workflow %s (--no-compile specified)", state.wf.Name) } - return nil } diff --git a/pkg/cli/upgrade_command.go b/pkg/cli/upgrade_command.go index 348b7d4801c..67a6c2dece4 100644 --- a/pkg/cli/upgrade_command.go +++ b/pkg/cli/upgrade_command.go @@ -32,6 +32,13 @@ type UpgradeConfig struct { // NewUpgradeCommand creates the upgrade command func NewUpgradeCommand(validateEngine func(string) error) *cobra.Command { + cmd := newUpgradeCommandBase(validateEngine) + newUpgradeCommandFlags(cmd) + newUpgradeCommandCompletions(cmd) + return cmd +} + +func newUpgradeCommandBase(validateEngine func(string) error) *cobra.Command { cmd := &cobra.Command{ Use: "upgrade", Short: "Upgrade local agent files and workflows (codemods, action updates, and compilation)", @@ -76,96 +83,13 @@ This command always upgrades all Markdown files in .github/workflows.`, ` + string(constants.CLIExtensionPrefix) + ` upgrade --pre-releases # Include pre-release versions when upgrading the extension (stable releases are the default)`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Flags().GetBool("verbose") - dir, _ := cmd.Flags().GetString("dir") - noFix, _ := cmd.Flags().GetBool("no-fix") - createPRFlag, _ := cmd.Flags().GetBool("create-pull-request") - prFlagAlias, _ := cmd.Flags().GetBool("pr") - createPR := createPRFlag || prFlagAlias - createIssue, _ := cmd.Flags().GetBool("create-issue") - yes, _ := cmd.Flags().GetBool("yes") - noActions, _ := cmd.Flags().GetBool("no-actions") - noCompile, _ := cmd.Flags().GetBool("no-compile") - auditFlag, _ := cmd.Flags().GetBool("audit") - jsonOutput, _ := cmd.Flags().GetBool("json") - disabledCodemods, _ := cmd.Flags().GetStringSlice("disable-codemod") - skipExtensionUpgrade, _ := cmd.Flags().GetBool("skip-extension-upgrade") - approveUpgrade, _ := cmd.Flags().GetBool("approve") - preReleases, _ := cmd.Flags().GetBool("pre-releases") - engineOverride, _ := cmd.Flags().GetString("engine") - targetRepo, _ := cmd.Flags().GetString("repo") - targetOrg, _ := cmd.Flags().GetString("org") - repoGlobs, _ := cmd.Flags().GetStringSlice("repos") - - if err := validateEngine(engineOverride); err != nil { - return err - } - - if targetRepo != "" && targetOrg != "" { - return errors.New("cannot specify both --repo and --org flags; use --repo for a single repository or --org for organization-wide upgrades") - } - - if len(repoGlobs) > 0 && targetOrg == "" { - return errors.New("--repos requires --org to be specified") - } - - if createIssue && targetOrg == "" { - return errors.New("--create-issue requires --org to be specified") - } - - if createPR && createIssue { - return errors.New("cannot specify both --create-pull-request and --create-issue") - } - - // Handle audit mode - if auditFlag { - return runDependencyAudit(cmd.Context(), verbose, jsonOutput) - } - - opts := upgradeOptions{ - ctx: cmd.Context(), - verbose: verbose, - workflowDir: dir, - noFix: noFix, - noCompile: noCompile, - noActions: noActions, - disabledCodemodIDs: disabledCodemods, - skipExtensionUpgrade: skipExtensionUpgrade, - approve: approveUpgrade, - preReleases: preReleases, - yes: yes, - engineOverride: engineOverride, - } - - if targetRepo != "" { - return runUpgradeForTargetRepoFn(cmd.Context(), targetRepo, opts, createPR, verbose) - } - - if targetOrg != "" { - return runUpgradeForOrg(cmd.Context(), targetOrg, repoGlobs, opts, createPR, createIssue, verbose) - } - - if createPR { - if err := PreflightCheckForCreatePR(verbose); err != nil { - return err - } - } - - if err := runUpgradeCommand(opts); err != nil { - return err - } - - if createPR { - prBody := "This PR upgrades agentic workflows by applying the latest codemods, " + - "updating GitHub Actions versions, and recompiling all workflows." - _, err := CreatePRWithChanges("upgrade-agentic-workflows", "chore: upgrade agentic workflows", - "Upgrade agentic workflows", prBody, verbose) - return err - } - return nil + return runUpgradeCommandFromCobra(cmd, validateEngine) }, } + return cmd +} +func newUpgradeCommandFlags(cmd *cobra.Command) { addEngineFlag(cmd) addRepoFlag(cmd) cmd.Flags().StringP("dir", "d", "", "Workflow directory (default: $GH_AW_WORKFLOWS_DIR or .github/workflows)") @@ -186,12 +110,131 @@ This command always upgrades all Markdown files in .github/workflows.`, cmd.Flags().String("org", "", "Preview or create upgrade pull requests across an organization") cmd.Flags().StringSlice("repos", nil, "Limit --org mode to repositories matching one or more glob patterns") addJSONFlag(cmd) +} +func newUpgradeCommandCompletions(cmd *cobra.Command) { // Register completions RegisterEngineFlagCompletion(cmd) RegisterDirFlagCompletion(cmd, "dir") +} - return cmd +type runUpgradeCommandFlags struct { + verbose bool + dir string + noFix bool + createPR bool + createIssue bool + yes bool + noActions bool + noCompile bool + auditFlag bool + jsonOutput bool + disabledCodemods []string + skipExtensionUpgrade bool + approveUpgrade bool + preReleases bool + engineOverride string + targetRepo string + targetOrg string + repoGlobs []string +} + +func runUpgradeCommandFromCobra(cmd *cobra.Command, validateEngine func(string) error) error { + flags := runUpgradeCommandReadFlags(cmd) + if err := runUpgradeCommandValidate(flags, validateEngine); err != nil { + return err + } + if flags.auditFlag { + return runDependencyAudit(cmd.Context(), flags.verbose, flags.jsonOutput) + } + opts := runUpgradeCommandOptions(cmd.Context(), flags) + if flags.targetRepo != "" { + return runUpgradeForTargetRepoFn(cmd.Context(), flags.targetRepo, opts, flags.createPR, flags.verbose) + } + if flags.targetOrg != "" { + return runUpgradeForOrg(cmd.Context(), flags.targetOrg, flags.repoGlobs, opts, flags.createPR, flags.createIssue, flags.verbose) + } + return runUpgradeCommandLocal(opts, flags.createPR, flags.verbose) +} + +func runUpgradeCommandReadFlags(cmd *cobra.Command) runUpgradeCommandFlags { + flags := runUpgradeCommandFlags{} + flags.verbose, _ = cmd.Flags().GetBool("verbose") + flags.dir, _ = cmd.Flags().GetString("dir") + flags.noFix, _ = cmd.Flags().GetBool("no-fix") + createPRFlag, _ := cmd.Flags().GetBool("create-pull-request") + prFlagAlias, _ := cmd.Flags().GetBool("pr") + flags.createPR = createPRFlag || prFlagAlias + flags.createIssue, _ = cmd.Flags().GetBool("create-issue") + flags.yes, _ = cmd.Flags().GetBool("yes") + flags.noActions, _ = cmd.Flags().GetBool("no-actions") + flags.noCompile, _ = cmd.Flags().GetBool("no-compile") + flags.auditFlag, _ = cmd.Flags().GetBool("audit") + flags.jsonOutput, _ = cmd.Flags().GetBool("json") + flags.disabledCodemods, _ = cmd.Flags().GetStringSlice("disable-codemod") + flags.skipExtensionUpgrade, _ = cmd.Flags().GetBool("skip-extension-upgrade") + flags.approveUpgrade, _ = cmd.Flags().GetBool("approve") + flags.preReleases, _ = cmd.Flags().GetBool("pre-releases") + flags.engineOverride, _ = cmd.Flags().GetString("engine") + flags.targetRepo, _ = cmd.Flags().GetString("repo") + flags.targetOrg, _ = cmd.Flags().GetString("org") + flags.repoGlobs, _ = cmd.Flags().GetStringSlice("repos") + return flags +} + +func runUpgradeCommandValidate(flags runUpgradeCommandFlags, validateEngine func(string) error) error { + if err := validateEngine(flags.engineOverride); err != nil { + return err + } + if flags.targetRepo != "" && flags.targetOrg != "" { + return errors.New("cannot specify both --repo and --org flags; use --repo for a single repository or --org for organization-wide upgrades") + } + if len(flags.repoGlobs) > 0 && flags.targetOrg == "" { + return errors.New("--repos requires --org to be specified") + } + if flags.createIssue && flags.targetOrg == "" { + return errors.New("--create-issue requires --org to be specified") + } + if flags.createPR && flags.createIssue { + return errors.New("cannot specify both --create-pull-request and --create-issue") + } + return nil +} + +func runUpgradeCommandOptions(ctx context.Context, flags runUpgradeCommandFlags) upgradeOptions { + return upgradeOptions{ + ctx: ctx, + verbose: flags.verbose, + workflowDir: flags.dir, + noFix: flags.noFix, + noCompile: flags.noCompile, + noActions: flags.noActions, + disabledCodemodIDs: flags.disabledCodemods, + skipExtensionUpgrade: flags.skipExtensionUpgrade, + approve: flags.approveUpgrade, + preReleases: flags.preReleases, + yes: flags.yes, + engineOverride: flags.engineOverride, + } +} + +func runUpgradeCommandLocal(opts upgradeOptions, createPR bool, verbose bool) error { + if createPR { + if err := PreflightCheckForCreatePR(verbose); err != nil { + return err + } + } + if err := runUpgradeCommand(opts); err != nil { + return err + } + if createPR { + prBody := "This PR upgrades agentic workflows by applying the latest codemods, " + + "updating GitHub Actions versions, and recompiling all workflows." + _, err := CreatePRWithChanges("upgrade-agentic-workflows", "chore: upgrade agentic workflows", + "Upgrade agentic workflows", prBody, verbose) + return err + } + return nil } // runDependencyAudit performs a dependency health audit @@ -234,188 +277,175 @@ func runUpgradeCommand(opts upgradeOptions) error { upgradeLog.Printf("Running upgrade command: verbose=%v, workflowDir=%s, noFix=%v, noCompile=%v, noActions=%v, disabledCodemodIDs=%v, skipExtensionUpgrade=%v", opts.verbose, opts.workflowDir, opts.noFix, opts.noCompile, opts.noActions, opts.disabledCodemodIDs, opts.skipExtensionUpgrade) - // Step 0b: Ensure gh-aw extension is on the latest version. - // If the extension was just upgraded, re-launch the freshly-installed binary - // with the same flags so that all subsequent steps (e.g. lock-file compilation) - // use the correct new version string. The hidden --skip-extension-upgrade flag - // prevents the re-launched process from entering this branch again. - if !opts.skipExtensionUpgrade { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Checking gh-aw extension version...")) - upgraded, installPath, err := upgradeExtensionIfOutdated(opts.verbose, opts.preReleases) - if err != nil { - upgradeLog.Printf("Extension upgrade failed: %v", err) - return err - } - if upgraded { - upgradeLog.Print("Extension was upgraded; re-launching with new binary") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Continuing upgrade with newly installed version...")) - // Pass installPath so relaunchWithSameArgs uses the pre-rename path; - // on Linux os.Executable() returns a "(deleted)" suffix after the rename. - if err := relaunchWithSameArgs("--skip-extension-upgrade", installPath); err != nil { - return err - } - // The child process completed all upgrade steps (including any PR creation). - // Signal the entry-point to exit cleanly without repeating those steps. - return &ExitCodeError{Code: 0} - } + if err := runUpgradeCommandMaybeRelaunch(opts); err != nil { + return err + } + if err := runUpgradeCommandUpdateArtifacts(opts); err != nil { + return err + } + runUpgradeCommandCodemods(opts) + runUpgradeCommandActions(opts) + runUpgradeCommandCompile(opts) + runUpgradeCommandContainerPins(opts) + + // Print success message + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Upgrade complete")) + + return nil +} + +func runUpgradeCommandMaybeRelaunch(opts upgradeOptions) error { + if opts.skipExtensionUpgrade { + return nil + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Checking gh-aw extension version...")) + upgraded, installPath, err := upgradeExtensionIfOutdated(opts.verbose, opts.preReleases) + if err != nil { + upgradeLog.Printf("Extension upgrade failed: %v", err) + return err + } + if !upgraded { + return nil } + upgradeLog.Print("Extension was upgraded; re-launching with new binary") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Continuing upgrade with newly installed version...")) + if err := relaunchWithSameArgs("--skip-extension-upgrade", installPath); err != nil { + return err + } + return &ExitCodeError{Code: 0} +} - // Step 1: Update dispatcher skill and related Copilot artifacts (like init command) +func runUpgradeCommandUpdateArtifacts(opts upgradeOptions) error { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updating dispatcher skill...")) upgradeLog.Print("Updating dispatcher skill") - if err := updateCopilotArtifacts(opts.ctx, opts.verbose); err != nil { upgradeLog.Printf("Failed to update dispatcher skill: %v", err) return fmt.Errorf("failed to update dispatcher skill: %w", err) } - if opts.verbose { fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Updated dispatcher skill")) } + return nil +} - // Step 2: Apply codemods to all workflows (unless --no-fix is specified) - if !opts.noFix { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Applying codemods to all workflows...")) - upgradeLog.Print("Applying codemods to all workflows") - - fixConfig := FixConfig{ - WorkflowIDs: nil, // nil means all workflows - Write: true, - Verbose: opts.verbose, - WorkflowDir: opts.workflowDir, - DisabledCodemodIDs: opts.disabledCodemodIDs, - } - - if err := RunFix(fixConfig); err != nil { - upgradeLog.Printf("Failed to apply codemods: %v", err) - // Don't fail the upgrade if fix fails - this is non-critical - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to apply codemods: %v", err))) - } - } else { +func runUpgradeCommandCodemods(opts upgradeOptions) { + if opts.noFix { upgradeLog.Print("Skipping codemods (--no-fix specified)") if opts.verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping codemods (--no-fix specified)")) } + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Applying codemods to all workflows...")) + upgradeLog.Print("Applying codemods to all workflows") + fixConfig := FixConfig{WorkflowIDs: nil, Write: true, Verbose: opts.verbose, WorkflowDir: opts.workflowDir, DisabledCodemodIDs: opts.disabledCodemodIDs} + if err := RunFix(fixConfig); err != nil { + upgradeLog.Printf("Failed to apply codemods: %v", err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to apply codemods: %v", err))) } +} - // Step 3: Update GitHub Actions versions (unless --no-fix or --no-actions is specified) - if !opts.noFix && !opts.noActions { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updating GitHub Actions versions...")) - upgradeLog.Print("Updating GitHub Actions versions") +func runUpgradeCommandActions(opts upgradeOptions) { + if opts.noFix || opts.noActions { + runUpgradeCommandActionsSkipped(opts) + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Updating GitHub Actions versions...")) + upgradeLog.Print("Updating GitHub Actions versions") + if err := UpdateActions(opts.ctx, false, opts.verbose, false, 0); err != nil { + upgradeLog.Printf("Failed to update actions: %v", err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to update actions: %v", err))) + return + } + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Updated GitHub Actions versions")) + } + upgradeLog.Print("Updating action references in workflow .md files") + if err := UpdateActionsInWorkflowFiles(opts.ctx, opts.workflowDir, opts.engineOverride, opts.verbose, false, true, 0, false); err != nil { + msg := fmt.Sprintf("Failed to update action references in workflow files: %v", err) + upgradeLog.Print(msg) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Warning: "+msg)) + } +} - if err := UpdateActions(opts.ctx, false, opts.verbose, false, 0); err != nil { - upgradeLog.Printf("Failed to update actions: %v", err) - // Don't fail the upgrade if action updates fail - this is non-critical - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to update actions: %v", err))) - } else { - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Updated GitHub Actions versions")) - } - - // Only update "uses:" references in source .md files when actions-lock.json - // was successfully updated, so both files stay in sync. Compilation is - // deferred to Step 4. - upgradeLog.Print("Updating action references in workflow .md files") - if err := UpdateActionsInWorkflowFiles(opts.ctx, opts.workflowDir, opts.engineOverride, opts.verbose, false, true, 0, false); err != nil { - msg := fmt.Sprintf("Failed to update action references in workflow files: %v", err) - upgradeLog.Print(msg) - // Non-critical: warn but don't fail the upgrade - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Warning: "+msg)) - } +func runUpgradeCommandActionsSkipped(opts upgradeOptions) { + if opts.noFix { + upgradeLog.Print("Skipping action updates (--no-fix specified)") + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping action updates (--no-fix specified)")) } - } else { - if opts.noFix { - upgradeLog.Print("Skipping action updates (--no-fix specified)") - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping action updates (--no-fix specified)")) - } - } else if opts.noActions { - upgradeLog.Print("Skipping action updates (--no-actions specified)") - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping action updates (--no-actions specified)")) - } + } else if opts.noActions { + upgradeLog.Print("Skipping action updates (--no-actions specified)") + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping action updates (--no-actions specified)")) } } +} - // Step 4: Compile all workflows (unless --no-fix or --no-compile is specified) - if !opts.noFix && !opts.noCompile { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Compiling all workflows...")) - upgradeLog.Print("Compiling all workflows") - - // Create and configure compiler - compiler := createAndConfigureCompiler(CompileConfig{ - Verbose: opts.verbose, - WorkflowDir: opts.workflowDir, - Approve: opts.approve, - EngineOverride: opts.engineOverride, - }) - - // Determine workflow directory - workflowsDir := opts.workflowDir - if workflowsDir == "" { - workflowsDir = constants.GetWorkflowDir() - } - - // Compile all workflow files - stats, compileErr := compileAllWorkflowFiles(opts.ctx, compiler, workflowsDir, opts.verbose) - if compileErr != nil { - upgradeLog.Printf("Failed to compile workflows: %v", compileErr) - // Don't fail the upgrade if compilation fails - this is non-critical - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to compile workflows: %v", compileErr))) - } else if stats != nil { - // Print compilation summary - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Compiled %d workflow(s)", stats.Total-stats.Errors))) - } - if stats.Errors > 0 { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: %d workflow(s) failed to compile", stats.Errors))) - } - } - } else { - if opts.noFix { - upgradeLog.Print("Skipping compilation (--no-fix specified)") - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping compilation (--no-fix specified)")) - } - } else if opts.noCompile { - upgradeLog.Print("Skipping compilation (--no-compile specified)") - if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping compilation (--no-compile specified)")) - } - } +func runUpgradeCommandCompile(opts upgradeOptions) { + if opts.noFix || opts.noCompile { + runUpgradeCommandCompileSkipped(opts) + return + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Compiling all workflows...")) + upgradeLog.Print("Compiling all workflows") + compiler := createAndConfigureCompiler(CompileConfig{Verbose: opts.verbose, WorkflowDir: opts.workflowDir, Approve: opts.approve, EngineOverride: opts.engineOverride}) + workflowsDir := opts.workflowDir + if workflowsDir == "" { + workflowsDir = constants.GetWorkflowDir() + } + stats, compileErr := compileAllWorkflowFiles(opts.ctx, compiler, workflowsDir, opts.verbose) + if compileErr != nil { + upgradeLog.Printf("Failed to compile workflows: %v", compileErr) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to compile workflows: %v", compileErr))) + } else if stats != nil { + runUpgradeCommandCompileSummary(stats, opts.verbose) } +} - // Step 4b: Update container image digest pins (unless --no-fix or --no-actions is specified) - // Container pins are stored alongside action pins in .github/aw/actions-lock.json. - // This runs AFTER compilation so that the compiled lock files already reflect the - // current AWF version; stale pins from superseded versions are pruned and new - // versions are resolved in a single pass. When --no-compile is set, the existing - // lock files are used as-is — pins are still pruned and refreshed against whatever - // lock files are currently on disk. - if !opts.noFix && !opts.noActions { - upgradeLog.Print("Updating container image digest pins") - newPins, err := UpdateContainerPins(opts.ctx, opts.workflowDir, opts.verbose) - if err != nil { - upgradeLog.Printf("Failed to update container pins: %v", err) - // Non-critical — Docker may not be available in all environments. - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to update container pins: %v", err))) - } else if opts.verbose { - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Updated container image pins")) +func runUpgradeCommandCompileSkipped(opts upgradeOptions) { + if opts.noFix { + upgradeLog.Print("Skipping compilation (--no-fix specified)") + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping compilation (--no-fix specified)")) } - if newPins && !opts.noCompile { - upgradeLog.Print("Recompiling workflows to embed new container digest pins") - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Recompiling workflows to embed container digest pins...")) - if recompileErr := recompileAllWorkflows(opts.ctx, opts.workflowDir, opts.engineOverride, opts.verbose, opts.approve); recompileErr != nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to recompile after container pin update: %v", recompileErr))) - } + } else if opts.noCompile { + upgradeLog.Print("Skipping compilation (--no-compile specified)") + if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Skipping compilation (--no-compile specified)")) } } +} - // Print success message - fmt.Fprintln(os.Stderr, "") - fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("Upgrade complete")) +func runUpgradeCommandCompileSummary(stats *CompilationStats, verbose bool) { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage(fmt.Sprintf("✓ Compiled %d workflow(s)", stats.Total-stats.Errors))) + } + if stats.Errors > 0 { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: %d workflow(s) failed to compile", stats.Errors))) + } +} - return nil +func runUpgradeCommandContainerPins(opts upgradeOptions) { + if opts.noFix || opts.noActions { + return + } + upgradeLog.Print("Updating container image digest pins") + newPins, err := UpdateContainerPins(opts.ctx, opts.workflowDir, opts.verbose) + if err != nil { + upgradeLog.Printf("Failed to update container pins: %v", err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to update container pins: %v", err))) + } else if opts.verbose { + fmt.Fprintln(os.Stderr, console.FormatSuccessMessage("✓ Updated container image pins")) + } + if newPins && !opts.noCompile { + upgradeLog.Print("Recompiling workflows to embed new container digest pins") + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Recompiling workflows to embed container digest pins...")) + if recompileErr := recompileAllWorkflows(opts.ctx, opts.workflowDir, opts.engineOverride, opts.verbose, opts.approve); recompileErr != nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Warning: Failed to recompile after container pin update: %v", recompileErr))) + } + } } // updateCopilotArtifacts updates the dispatcher skill and related Copilot setup artifacts. diff --git a/pkg/cli/upgrade_org.go b/pkg/cli/upgrade_org.go index 3281e951cd1..b135d5fb3ce 100644 --- a/pkg/cli/upgrade_org.go +++ b/pkg/cli/upgrade_org.go @@ -210,47 +210,16 @@ func normalizeDisplayVersion(version string) string { // runUpgradeForTargetRepo checks out repo to a temporary directory, runs the // upgrade command inside it, and opens a pull request with the resulting changes. func runUpgradeForTargetRepo(ctx context.Context, repo string, opts upgradeOptions, createPR bool, verbose bool) error { - gitRoot, err := gitutil.FindGitRoot() - if err != nil { - return fmt.Errorf("--repo/--org requires running inside a git repository: %w", err) - } - - updatesDir, err := ensureUpdateTargetRepoGitignore(gitRoot) + checkoutDir, err := runUpgradeForTargetRepoCheckout(ctx, repo, verbose) if err != nil { return err } - checkoutDir := filepath.Join(updatesDir, sanitizeRepoPath(repo)) - if err := shallowCloneTargetRepo(ctx, repo, checkoutDir); err != nil { - return err - } - - // Extend sparse checkout to include .github/skills; upgrade also updates - // the dispatcher skill (ensureAgenticWorkflowsDispatcher) and needs that path present. - sparseAddCmd := exec.CommandContext(ctx, "git", "-C", checkoutDir, "sparse-checkout", "add", orgUpgradeSkillsDir) - if output, err := sparseAddCmd.CombinedOutput(); err != nil { - trimmed := strings.TrimSpace(string(output)) - if trimmed == "" { - return fmt.Errorf("failed to extend sparse checkout for %s: %w", repo, err) - } - return fmt.Errorf("failed to extend sparse checkout for %s: %w: %s", repo, err, trimmed) - } - - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Checked out "+repo+" at "+checkoutDir)) - } - - originalDir, err := os.Getwd() + restoreDir, err := runUpgradeForTargetRepoChdir(checkoutDir) if err != nil { - return fmt.Errorf("failed to read current directory: %w", err) - } - defer func() { - _ = os.Chdir(originalDir) - }() - - if err := os.Chdir(checkoutDir); err != nil { - return fmt.Errorf("failed to change directory to checkout %s: %w", checkoutDir, err) + return err } + defer restoreDir() if createPR { if err := PreflightCheckForCreatePR(verbose); err != nil { @@ -273,8 +242,55 @@ func runUpgradeForTargetRepo(ctx context.Context, repo string, opts upgradeOptio if !createPR { return nil } + return runUpgradeForTargetRepoCreatePR(ctx, repo, verbose) +} + +func runUpgradeForTargetRepoCheckout(ctx context.Context, repo string, verbose bool) (string, error) { + gitRoot, err := gitutil.FindGitRoot() + if err != nil { + return "", fmt.Errorf("--repo/--org requires running inside a git repository: %w", err) + } + updatesDir, err := ensureUpdateTargetRepoGitignore(gitRoot) + if err != nil { + return "", err + } + checkoutDir := filepath.Join(updatesDir, sanitizeRepoPath(repo)) + if err := shallowCloneTargetRepo(ctx, repo, checkoutDir); err != nil { + return "", err + } + if err := runUpgradeForTargetRepoSparseSkills(ctx, repo, checkoutDir); err != nil { + return "", err + } + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Checked out "+repo+" at "+checkoutDir)) + } + return checkoutDir, nil +} + +func runUpgradeForTargetRepoSparseSkills(ctx context.Context, repo, checkoutDir string) error { + sparseAddCmd := exec.CommandContext(ctx, "git", "-C", checkoutDir, "sparse-checkout", "add", orgUpgradeSkillsDir) + if output, err := sparseAddCmd.CombinedOutput(); err != nil { + trimmed := strings.TrimSpace(string(output)) + if trimmed == "" { + return fmt.Errorf("failed to extend sparse checkout for %s: %w", repo, err) + } + return fmt.Errorf("failed to extend sparse checkout for %s: %w: %s", repo, err, trimmed) + } + return nil +} + +func runUpgradeForTargetRepoChdir(checkoutDir string) (func(), error) { + originalDir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("failed to read current directory: %w", err) + } + if err := os.Chdir(checkoutDir); err != nil { + return nil, fmt.Errorf("failed to change directory to checkout %s: %w", checkoutDir, err) + } + return func() { _ = os.Chdir(originalDir) }, nil +} - // Skip PR creation when the upgrade produced no changes (e.g. repo is already up to date). +func runUpgradeForTargetRepoCreatePR(ctx context.Context, repo string, verbose bool) error { changed, err := hasPendingChanges() if err != nil { return err diff --git a/pkg/cli/validate_command.go b/pkg/cli/validate_command.go index dc4a556a3e1..aa7126c7695 100644 --- a/pkg/cli/validate_command.go +++ b/pkg/cli/validate_command.go @@ -10,6 +10,13 @@ var validateLog = logger.New("cli:validate_command") // NewValidateCommand creates the validate command func NewValidateCommand(validateEngine func(string) error) *cobra.Command { + cmd := newValidateCommandBase(validateEngine) + newValidateCommandFlags(cmd) + newValidateCommandCompletions(cmd) + return cmd +} + +func newValidateCommandBase(validateEngine func(string) error) *cobra.Command { cmd := &cobra.Command{ Use: "validate [workflow]...", Short: "Validate agentic workflows without generating lock files", @@ -29,50 +36,13 @@ If no workflows are specified, all Markdown files in .github/workflows will be v ` + string(constants.CLIExtensionPrefix) + ` validate --strict # Enforce strict mode validation ` + string(constants.CLIExtensionPrefix) + ` validate --fail-fast # Stop at the first validation error`, RunE: func(cmd *cobra.Command, args []string) error { - engineOverride, _ := cmd.Flags().GetString("engine") - dir, _ := cmd.Flags().GetString("dir") - strict, _ := cmd.Flags().GetBool("strict") - jsonOutput, _ := cmd.Flags().GetBool("json") - failFast, _ := cmd.Flags().GetBool("fail-fast") - stats, _ := cmd.Flags().GetBool("stats") - allowActionRefs, _ := cmd.Flags().GetBool("allow-action-refs") - noCheckUpdate, _ := cmd.Flags().GetBool("no-check-update") - validateImages, _ := cmd.Flags().GetBool("validate-images") - verbose, _ := cmd.Flags().GetBool("verbose") - - if err := validateEngine(engineOverride); err != nil { - return err - } - - // Check for updates (non-blocking, runs once per day) - CheckForUpdatesAsync(cmd.Context(), noCheckUpdate, verbose) - - validateLog.Printf("Running validate command: workflows=%v, dir=%s", args, dir) - - config := CompileConfig{ - MarkdownFiles: args, - Verbose: verbose, - EngineOverride: engineOverride, - Validate: true, - NoEmit: true, - Zizmor: true, - Actionlint: true, - Poutine: true, - WorkflowDir: dir, - Strict: strict, - JSONOutput: jsonOutput, - FailFast: failFast, - Stats: stats, - AllowActionRefs: allowActionRefs, - ValidateImages: validateImages, - } - if _, err := CompileWorkflows(cmd.Context(), config); err != nil { - return err - } - return nil + return runValidateCommand(cmd, args, validateEngine) }, } + return cmd +} +func newValidateCommandFlags(cmd *cobra.Command) { addEngineFlag(cmd) cmd.Flags().StringP("dir", "d", "", "Workflow directory (default: $GH_AW_WORKFLOWS_DIR or .github/workflows)") cmd.Flags().Bool("strict", false, "Override frontmatter to enforce strict mode validation for all workflows (enforces action pinning, network config, safe-outputs, disallows write permissions and deprecated fields). Note: Workflows default to strict mode unless frontmatter sets strict: false") @@ -82,11 +52,52 @@ If no workflows are specified, all Markdown files in .github/workflows will be v cmd.Flags().Bool("stats", false, "Display statistics table sorted by workflow file size (shows jobs, steps, scripts, and shells)") cmd.Flags().Bool("allow-action-refs", false, "Allow unresolved action refs and emit warnings instead of failing validation") cmd.Flags().Bool("no-check-update", false, "Skip checking for gh-aw updates") +} +func newValidateCommandCompletions(cmd *cobra.Command) { // Register completions cmd.ValidArgsFunction = CompleteWorkflowNames RegisterEngineFlagCompletion(cmd) RegisterDirFlagCompletion(cmd, "dir") +} - return cmd +func runValidateCommand(cmd *cobra.Command, args []string, validateEngine func(string) error) error { + engineOverride, _ := cmd.Flags().GetString("engine") + dir, _ := cmd.Flags().GetString("dir") + strict, _ := cmd.Flags().GetBool("strict") + jsonOutput, _ := cmd.Flags().GetBool("json") + failFast, _ := cmd.Flags().GetBool("fail-fast") + stats, _ := cmd.Flags().GetBool("stats") + allowActionRefs, _ := cmd.Flags().GetBool("allow-action-refs") + noCheckUpdate, _ := cmd.Flags().GetBool("no-check-update") + validateImages, _ := cmd.Flags().GetBool("validate-images") + verbose, _ := cmd.Flags().GetBool("verbose") + + if err := validateEngine(engineOverride); err != nil { + return err + } + + CheckForUpdatesAsync(cmd.Context(), noCheckUpdate, verbose) + validateLog.Printf("Running validate command: workflows=%v, dir=%s", args, dir) + config := CompileConfig{ + MarkdownFiles: args, + Verbose: verbose, + EngineOverride: engineOverride, + Validate: true, + NoEmit: true, + Zizmor: true, + Actionlint: true, + Poutine: true, + WorkflowDir: dir, + Strict: strict, + JSONOutput: jsonOutput, + FailFast: failFast, + Stats: stats, + AllowActionRefs: allowActionRefs, + ValidateImages: validateImages, + } + if _, err := CompileWorkflows(cmd.Context(), config); err != nil { + return err + } + return nil } diff --git a/pkg/cli/view_command.go b/pkg/cli/view_command.go index 7317b87a47e..f1f1cb29c5c 100644 --- a/pkg/cli/view_command.go +++ b/pkg/cli/view_command.go @@ -31,6 +31,12 @@ var viewLog = logger.New("cli:view") // NewViewCommand creates the view command. func NewViewCommand() *cobra.Command { + cmd := newViewCommandBase() + newViewCommandFlags(cmd) + return cmd +} + +func newViewCommandBase() *cobra.Command { cmd := &cobra.Command{ Use: "view ", Short: "Render unified timeline and safe outputs for a workflow run", @@ -63,46 +69,45 @@ re-downloading.`, Args: cobra.ExactArgs(1), Hidden: true, RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Flags().GetBool("verbose") - outputDir, _ := cmd.Flags().GetString("output") - repoFlag, _ := cmd.Flags().GetString("repo") - - runIDOrURL := args[0] - - components, err := parser.ParseRunURLExtended(runIDOrURL) - if err != nil { - return err - } - - // Apply --repo flag when owner/repo were not inferred from a URL. - if repoFlag != "" && components.Owner == "" { - parts := strings.SplitN(repoFlag, "/", 2) - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return fmt.Errorf("invalid repository format %q: expected 'owner/repo'", repoFlag) - } - components.Owner = parts[0] - components.Repo = parts[1] - } - - if outputDir == "" { - outputDir = defaultLogsOutputDir - } - - return ViewWorkflowRun(cmd.Context(), components.Number, ViewOptions{ - Owner: components.Owner, - Repo: components.Repo, - Hostname: components.Host, - OutputDir: outputDir, - Verbose: verbose, - }) + return runViewCommand(cmd, args) }, } + return cmd +} +func newViewCommandFlags(cmd *cobra.Command) { addOutputFlag(cmd, defaultLogsOutputDir) addRepoFlag(cmd) RegisterDirFlagCompletion(cmd, "output") +} - return cmd +func runViewCommand(cmd *cobra.Command, args []string) error { + verbose, _ := cmd.Flags().GetBool("verbose") + outputDir, _ := cmd.Flags().GetString("output") + repoFlag, _ := cmd.Flags().GetString("repo") + + components, err := parser.ParseRunURLExtended(args[0]) + if err != nil { + return err + } + if repoFlag != "" && components.Owner == "" { + parts := strings.SplitN(repoFlag, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return fmt.Errorf("invalid repository format %q: expected 'owner/repo'", repoFlag) + } + components.Owner = parts[0] + components.Repo = parts[1] + } + if outputDir == "" { + outputDir = defaultLogsOutputDir + } + return ViewWorkflowRun(cmd.Context(), components.Number, ViewOptions{ + Owner: components.Owner, + Repo: components.Repo, + Hostname: components.Host, + OutputDir: outputDir, + Verbose: verbose, + }) } // ViewOptions holds configuration for the view command. @@ -120,18 +125,8 @@ func ViewWorkflowRun(ctx context.Context, runID int64, opts ViewOptions) error { viewLog.Printf("Starting view for run %d (owner=%s, repo=%s, hostname=%s)", runID, opts.Owner, opts.Repo, opts.Hostname) // Auto-detect GHES host from git remote when not explicitly provided. - hostname := opts.Hostname - if hostname == "" { - hostname = getHostFromOriginRemote() - if hostname != "github.com" { - viewLog.Printf("Auto-detected GHES host from git remote: %s", hostname) - } - } - - runDir := filepath.Join(opts.OutputDir, fmt.Sprintf("run-%d", runID)) - if absDir, err := filepath.Abs(runDir); err == nil { - runDir = absDir - } + hostname := viewWorkflowRunHostname(opts.Hostname) + runDir := viewWorkflowRunDir(opts.OutputDir, runID) if opts.Verbose { fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Viewing run %d...", runID))) @@ -142,15 +137,8 @@ func ViewWorkflowRun(ctx context.Context, runID int64, opts ViewOptions) error { // log files we need. We deliberately pass a nil artifact filter so that all // artifacts are downloaded — the timeline relies on whichever JSONL files // happen to be present; no single one is strictly required. - if err := downloadRunArtifacts(ctx, downloadArtifactsOptions{runID: runID, outputDir: runDir, verbose: opts.Verbose, owner: opts.Owner, repo: opts.Repo, hostname: hostname}); err != nil { - if !errors.Is(err, ErrNoArtifacts) { - return fmt.Errorf("failed to download artifacts for run %d: %w", runID, err) - } - // No artifacts is non-fatal: the run may still have useful events in the - // workflow logs or the directory may have been populated by a previous run. - if opts.Verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("No artifacts attached to this run; timeline may be empty.")) - } + if err := viewWorkflowRunDownload(ctx, runID, runDir, hostname, opts); err != nil { + return err } // Collect and merge events from all available JSONL sources. @@ -181,6 +169,36 @@ func ViewWorkflowRun(ctx context.Context, runID int64, opts ViewOptions) error { return nil } +func viewWorkflowRunHostname(hostname string) string { + if hostname == "" { + hostname = getHostFromOriginRemote() + if hostname != "github.com" { + viewLog.Printf("Auto-detected GHES host from git remote: %s", hostname) + } + } + return hostname +} + +func viewWorkflowRunDir(outputDir string, runID int64) string { + runDir := filepath.Join(outputDir, fmt.Sprintf("run-%d", runID)) + if absDir, err := filepath.Abs(runDir); err == nil { + runDir = absDir + } + return runDir +} + +func viewWorkflowRunDownload(ctx context.Context, runID int64, runDir, hostname string, opts ViewOptions) error { + if err := downloadRunArtifacts(ctx, downloadArtifactsOptions{runID: runID, outputDir: runDir, verbose: opts.Verbose, owner: opts.Owner, repo: opts.Repo, hostname: hostname}); err != nil { + if !errors.Is(err, ErrNoArtifacts) { + return fmt.Errorf("failed to download artifacts for run %d: %w", runID, err) + } + if opts.Verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("No artifacts attached to this run; timeline may be empty.")) + } + } + return nil +} + // buildRunHTMLURL constructs the GitHub Actions HTML URL for a workflow run. // Returns an empty string when owner or repo are unknown. func buildRunHTMLURL(hostname, owner, repo string, runID int64) string { diff --git a/pkg/cli/workflows.go b/pkg/cli/workflows.go index 52c79f4cd2d..0d4a7cc4ea0 100644 --- a/pkg/cli/workflows.go +++ b/pkg/cli/workflows.go @@ -77,95 +77,103 @@ func fetchGitHubWorkflows(repoOverride string, verbose bool) (map[string]*GitHub spinner.Start() } - args := []string{"workflow", "list", "--all", "--json", "id,name,path,state"} - if repoOverride != "" { - args = append(args, "--repo", repoOverride) + output, err := fetchGitHubWorkflowsRun(repoOverride) + if err != nil { + if !verbose { + spinner.Stop() + } + return nil, err } - cmd := workflow.ExecGH(args...) - output, err := cmd.Output() + workflows, err := fetchGitHubWorkflowsParse(output) if err != nil { - // Stop spinner on error if !verbose { spinner.Stop() } + return nil, err + } + workflowMap := fetchGitHubWorkflowsMap(workflows) + userWorkflowCount := fetchGitHubWorkflowsUserCount(workflowMap) - // Extract detailed error information including exit code and stderr - var exitCode int - var stderr string - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - exitCode = exitErr.ExitCode() - stderr = string(exitErr.Stderr) - workflowsLog.Printf("gh workflow list command failed with exit code %d. Command: gh %v", exitCode, args) - workflowsLog.Printf("stderr output: %s", stderr) + if !verbose { + fetchGitHubWorkflowsStopSpinner(spinner, userWorkflowCount) + } - return nil, fmt.Errorf("failed to execute gh workflow list command (exit code %d): %w. stderr: %s", exitCode, err, stderr) - } + workflowsLog.Printf("Fetched %d GitHub workflows (%d with .md files)", len(workflowMap), userWorkflowCount) + return workflowMap, nil +} - // If not an ExitError, log what we can - workflowsLog.Printf("gh workflow list command failed with error (not ExitError): %v. Command: gh %v", err, args) - return nil, fmt.Errorf("failed to execute gh workflow list command: %w", err) +func fetchGitHubWorkflowsRun(repoOverride string) ([]byte, error) { + args := []string{"workflow", "list", "--all", "--json", "id,name,path,state"} + if repoOverride != "" { + args = append(args, "--repo", repoOverride) + } + cmd := workflow.ExecGH(args...) + output, err := cmd.Output() + if err != nil { + return nil, fetchGitHubWorkflowsCommandError(err, args) } - - // Check if output is empty if len(output) == 0 { - if !verbose { - spinner.Stop() - } return nil, errors.New("gh workflow list returned empty output - check if repository has workflows and gh CLI is authenticated") } - - // Validate JSON before unmarshaling if !json.Valid(output) { - if !verbose { - spinner.Stop() - } return nil, errors.New("gh workflow list returned invalid JSON - this may be due to network issues or authentication problems") } + return output, nil +} +func fetchGitHubWorkflowsCommandError(err error, args []string) error { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + exitCode := exitErr.ExitCode() + stderr := string(exitErr.Stderr) + workflowsLog.Printf("gh workflow list command failed with exit code %d. Command: gh %v", exitCode, args) + workflowsLog.Printf("stderr output: %s", stderr) + return fmt.Errorf("failed to execute gh workflow list command (exit code %d): %w. stderr: %s", exitCode, err, stderr) + } + workflowsLog.Printf("gh workflow list command failed with error (not ExitError): %v. Command: gh %v", err, args) + return fmt.Errorf("failed to execute gh workflow list command: %w", err) +} + +func fetchGitHubWorkflowsParse(output []byte) ([]GitHubWorkflow, error) { var workflows []GitHubWorkflow if err := json.Unmarshal(output, &workflows); err != nil { - if !verbose { - spinner.Stop() - } return nil, fmt.Errorf("failed to parse workflow data: %w", err) } + return workflows, nil +} +func fetchGitHubWorkflowsMap(workflows []GitHubWorkflow) map[string]*GitHubWorkflow { workflowMap := make(map[string]*GitHubWorkflow) for i, workflow := range workflows { name := extractWorkflowNameFromPath(workflow.Path) workflowMap[name] = &workflows[i] } + return workflowMap +} - // Count user workflows (those with .md files) +func fetchGitHubWorkflowsUserCount(workflowMap map[string]*GitHubWorkflow) int { mdFiles, _ := getMarkdownWorkflowFiles("") - mdWorkflowNames := make(map[string]struct { - }) + mdWorkflowNames := make(map[string]struct{}) for _, file := range mdFiles { name := extractWorkflowNameFromPath(file) - mdWorkflowNames[name] = struct { - }{} + mdWorkflowNames[name] = struct{}{} } - var userWorkflowCount int for name := range workflowMap { if setutil.Contains(mdWorkflowNames, name) { userWorkflowCount++ } } + return userWorkflowCount +} - // Stop spinner with success message showing only user workflow count - if !verbose { - if userWorkflowCount == 1 { - spinner.StopWithMessage("✓ Fetched 1 workflow") - } else { - spinner.StopWithMessage(fmt.Sprintf("✓ Fetched %d workflows", userWorkflowCount)) - } +func fetchGitHubWorkflowsStopSpinner(spinner *console.SpinnerWrapper, userWorkflowCount int) { + if userWorkflowCount == 1 { + spinner.StopWithMessage("✓ Fetched 1 workflow") + } else { + spinner.StopWithMessage(fmt.Sprintf("✓ Fetched %d workflows", userWorkflowCount)) } - - workflowsLog.Printf("Fetched %d GitHub workflows (%d with .md files)", len(workflowMap), userWorkflowCount) - return workflowMap, nil } // extractWorkflowNameFromPath extracts workflow name from path @@ -357,46 +365,16 @@ func fastParseTitleFromReader(r io.Reader) (string, error) { scanner := bufio.NewScanner(r) // Reuse the small initial scanner buffer across calls while still allowing // growth up to 1 MB for large frontmatter values or long base64-encoded lines. - pooled := workflowTitleScannerBufferPool.Get() - scannerBufferPtr, ok := pooled.(*[]byte) - if !ok || scannerBufferPtr == nil { - fallback := make([]byte, workflowTitleScannerBufferSize) - scannerBufferPtr = &fallback - } - scannerBuffer := *scannerBufferPtr - if cap(scannerBuffer) != workflowTitleScannerBufferSize { - scannerBuffer = make([]byte, workflowTitleScannerBufferSize) - } else { - scannerBuffer = scannerBuffer[:workflowTitleScannerBufferSize] - } - defer func() { - *scannerBufferPtr = scannerBuffer - workflowTitleScannerBufferPool.Put(scannerBufferPtr) - }() + scannerBufferPtr, scannerBuffer := fastParseTitleFromReaderBuffer() + defer fastParseTitleFromReaderPutBuffer(scannerBufferPtr, scannerBuffer) scanner.Buffer(scannerBuffer, 1024*1024) firstLine := true inFrontmatter := false for scanner.Scan() { line := scanner.Bytes() - if firstLine { - firstLine = false - if isFrontmatterDelimiter(line) { - inFrontmatter = true - continue - } - if trimmed := bytes.TrimSpace(line); isFrontmatterDelimiter(trimmed) { - inFrontmatter = true - continue - } - } else if inFrontmatter { - if isFrontmatterDelimiter(line) { - inFrontmatter = false - } else if trimmed := bytes.TrimSpace(line); isFrontmatterDelimiter(trimmed) { - inFrontmatter = false - } + if fastParseTitleFromReaderFrontmatter(line, &firstLine, &inFrontmatter) { continue } - if title, ok := extractHeadingTitle(line); ok { return title, nil } @@ -418,6 +396,45 @@ func fastParseTitleFromReader(r io.Reader) (string, error) { return "", nil } +func fastParseTitleFromReaderBuffer() (*[]byte, []byte) { + pooled := workflowTitleScannerBufferPool.Get() + scannerBufferPtr, ok := pooled.(*[]byte) + if !ok || scannerBufferPtr == nil { + fallback := make([]byte, workflowTitleScannerBufferSize) + scannerBufferPtr = &fallback + } + scannerBuffer := *scannerBufferPtr + if cap(scannerBuffer) != workflowTitleScannerBufferSize { + scannerBuffer = make([]byte, workflowTitleScannerBufferSize) + } else { + scannerBuffer = scannerBuffer[:workflowTitleScannerBufferSize] + } + return scannerBufferPtr, scannerBuffer +} + +func fastParseTitleFromReaderPutBuffer(scannerBufferPtr *[]byte, scannerBuffer []byte) { + *scannerBufferPtr = scannerBuffer + workflowTitleScannerBufferPool.Put(scannerBufferPtr) +} + +func fastParseTitleFromReaderFrontmatter(line []byte, firstLine *bool, inFrontmatter *bool) bool { + if *firstLine { + *firstLine = false + if isFrontmatterDelimiter(line) || isFrontmatterDelimiter(bytes.TrimSpace(line)) { + *inFrontmatter = true + return true + } + return false + } + if *inFrontmatter { + if isFrontmatterDelimiter(line) || isFrontmatterDelimiter(bytes.TrimSpace(line)) { + *inFrontmatter = false + } + return true + } + return false +} + func isFrontmatterDelimiter(line []byte) bool { return len(line) == 3 && line[0] == '-' && line[1] == '-' && line[2] == '-' } diff --git a/pkg/cli/yaml_frontmatter_utils.go b/pkg/cli/yaml_frontmatter_utils.go index c1ffb3bf70c..6cf7e9a5dbe 100644 --- a/pkg/cli/yaml_frontmatter_utils.go +++ b/pkg/cli/yaml_frontmatter_utils.go @@ -170,78 +170,74 @@ func removeParentBlockIfTrulyEmpty(lines []string, parentBlock string) []string // Returns the modified lines and whether any changes were made. func removeFieldFromBlock(lines []string, fieldName string, parentBlock string) ([]string, bool) { var result []string - var modified bool - var inParentBlock bool - var parentIndent string - var inFieldBlock bool - var fieldIndent string + state := &removeFieldFromBlockState{fieldName: fieldName, parentBlock: parentBlock} for i, line := range lines { - trimmedLine := strings.TrimSpace(line) - - // Track if we're in the parent block - if strings.HasPrefix(trimmedLine, parentBlock+":") { - inParentBlock = true - parentIndent = getIndentation(line) + keep := removeFieldFromBlockLine(state, line, i+1) + if keep { result = append(result, line) - continue } + } - // Check if we've left the parent block - if inParentBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - if hasExitedBlock(line, parentIndent) { - inParentBlock = false - } - } + if state.modified { + result = removeParentBlockIfTrulyEmpty(result, parentBlock) + } - // Remove field line if in parent block - if inParentBlock && strings.HasPrefix(trimmedLine, fieldName+":") { - modified = true - inFieldBlock = true - fieldIndent = getIndentation(line) - yamlUtilsLog.Printf("Removed %s.%s on line %d", parentBlock, fieldName, i+1) - continue - } + return result, state.modified +} - // Skip nested properties under the field (lines with greater indentation) - if inFieldBlock { - // Empty lines within the field block should be removed - if trimmedLine == "" { - continue - } +type removeFieldFromBlockState struct { + fieldName string + parentBlock string + modified bool + inParentBlock bool + parentIndent string + inFieldBlock bool + fieldIndent string +} - currentIndent := getIndentation(line) - - // Comments need to check indentation - if strings.HasPrefix(trimmedLine, "#") { - if len(currentIndent) > len(fieldIndent) { - // Comment is nested under field, remove it - yamlUtilsLog.Printf("Removed nested %s comment on line %d: %s", fieldName, i+1, trimmedLine) - continue - } - // Comment is at same or less indentation, exit field block and keep it - inFieldBlock = false - result = append(result, line) - continue - } +func removeFieldFromBlockLine(state *removeFieldFromBlockState, line string, lineNumber int) bool { + trimmedLine := strings.TrimSpace(line) + if strings.HasPrefix(trimmedLine, state.parentBlock+":") { + state.inParentBlock = true + state.parentIndent = getIndentation(line) + return true + } + if state.inParentBlock && trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") && hasExitedBlock(line, state.parentIndent) { + state.inParentBlock = false + } + if state.inParentBlock && strings.HasPrefix(trimmedLine, state.fieldName+":") { + state.modified = true + state.inFieldBlock = true + state.fieldIndent = getIndentation(line) + yamlUtilsLog.Printf("Removed %s.%s on line %d", state.parentBlock, state.fieldName, lineNumber) + return false + } + if state.inFieldBlock { + return removeFieldFromBlockNestedLine(state, line, trimmedLine, lineNumber) + } + return true +} - // If this line has more indentation than field, it's a nested property - if len(currentIndent) > len(fieldIndent) { - yamlUtilsLog.Printf("Removed nested %s property on line %d: %s", fieldName, i+1, trimmedLine) - continue - } - // We've exited the field block (found a line at same or less indentation) - inFieldBlock = false +func removeFieldFromBlockNestedLine(state *removeFieldFromBlockState, line, trimmedLine string, lineNumber int) bool { + if trimmedLine == "" { + return false + } + currentIndent := getIndentation(line) + if strings.HasPrefix(trimmedLine, "#") { + if len(currentIndent) > len(state.fieldIndent) { + yamlUtilsLog.Printf("Removed nested %s comment on line %d: %s", state.fieldName, lineNumber, trimmedLine) + return false } - - result = append(result, line) + state.inFieldBlock = false + return true } - - if modified { - result = removeParentBlockIfTrulyEmpty(result, parentBlock) + if len(currentIndent) > len(state.fieldIndent) { + yamlUtilsLog.Printf("Removed nested %s property on line %d: %s", state.fieldName, lineNumber, trimmedLine) + return false } - - return result, modified + state.inFieldBlock = false + return true } // isDescendant returns true if childIndent is deeper (more indented) than parentIndent. diff --git a/pkg/cli/zizmor.go b/pkg/cli/zizmor.go index 3a217a0b8a2..289461b12eb 100644 --- a/pkg/cli/zizmor.go +++ b/pkg/cli/zizmor.go @@ -55,66 +55,63 @@ func runZizmorOnFiles(lockFiles []string, verbose bool, strict bool) error { zizmorLog.Printf("Running zizmor security scanner on %d file(s): %v (verbose=%t, strict=%t)", len(lockFiles), lockFiles, verbose, strict) - // Find git root to get the absolute path for Docker volume mount - gitRoot, err := gitutil.FindGitRoot() + gitRoot, relPaths, err := runZizmorOnFilesPaths(lockFiles) if err != nil { - return fmt.Errorf("failed to find git root: %w", err) + return err } + cmd := runZizmorOnFilesCommand(gitRoot, relPaths) + runZizmorOnFilesAnnounce(lockFiles, relPaths, gitRoot, verbose) + + // Capture output + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + // Run the command + err = cmd.Run() + return runZizmorOnFilesResult(err, lockFiles, stdout, stderr, verbose, strict) +} - // Validate gitRoot is an absolute path before use in Docker volume mount +func runZizmorOnFilesPaths(lockFiles []string) (string, []string, error) { + gitRoot, err := gitutil.FindGitRoot() + if err != nil { + return "", nil, fmt.Errorf("failed to find git root: %w", err) + } if !filepath.IsAbs(gitRoot) { - return fmt.Errorf("git root must be an absolute path, got: %s", gitRoot) + return "", nil, fmt.Errorf("git root must be an absolute path, got: %s", gitRoot) } - - // Get relative paths from git root for all files var relPaths []string for _, lockFile := range lockFiles { relPath, err := filepath.Rel(gitRoot, lockFile) if err != nil { - return fmt.Errorf("failed to get relative path for %s: %w", lockFile, err) + return "", nil, fmt.Errorf("failed to get relative path for %s: %w", lockFile, err) } relPaths = append(relPaths, relPath) } + return gitRoot, relPaths, nil +} - // Build the Docker command with JSON output for easier parsing - // docker run --rm -v "$(pwd)":/workdir -w /workdir ghcr.io/zizmorcore/zizmor:latest --format json ... - dockerArgs := []string{ - "run", - "--rm", - "-v", gitRoot + ":/workdir", - "-w", "/workdir", - "ghcr.io/zizmorcore/zizmor:latest", - "--format", "json", - } +func runZizmorOnFilesCommand(gitRoot string, relPaths []string) *exec.Cmd { + dockerArgs := []string{"run", "--rm", "-v", gitRoot + ":/workdir", "-w", "/workdir", "ghcr.io/zizmorcore/zizmor:latest", "--format", "json"} dockerArgs = append(dockerArgs, relPaths...) - // #nosec G204 -- exec.Command is used with separate args (not shell execution) to prevent shell injection. - // The gitRoot path is validated to be absolute, and relPaths are validated through filepath.Rel to be - // relative to gitRoot, preventing path traversal. The Docker container provides additional isolation. - cmd := exec.Command("docker", dockerArgs...) + // The gitRoot path is validated to be absolute, and relPaths are validated through filepath.Rel. + return exec.Command("docker", dockerArgs...) +} - // Always show that zizmor is running (regular verbosity) +func runZizmorOnFilesAnnounce(lockFiles, relPaths []string, gitRoot string, verbose bool) { if len(lockFiles) == 1 { fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Running zizmor security scanner on "+relPaths[0])) } else { fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage(fmt.Sprintf("Running zizmor security scanner on %d files", len(lockFiles)))) } - - // In verbose mode, also show the command that users can run directly if verbose { - dockerCmd := fmt.Sprintf("docker run --rm -v \"%s:/workdir\" -w /workdir ghcr.io/zizmorcore/zizmor:latest --format json %s", - gitRoot, strings.Join(relPaths, " ")) + dockerCmd := fmt.Sprintf("docker run --rm -v \"%s:/workdir\" -w /workdir ghcr.io/zizmorcore/zizmor:latest --format json %s", gitRoot, strings.Join(relPaths, " ")) fmt.Fprintf(os.Stderr, "%s\n", console.FormatInfoMessage("Run zizmor directly: "+dockerCmd)) } +} - // Capture output - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - // Run the command - err = cmd.Run() - +func runZizmorOnFilesResult(err error, lockFiles []string, stdout, stderr bytes.Buffer, verbose bool, strict bool) error { // Parse and reformat the output, get total warning count totalWarnings, parseErr := parseAndDisplayZizmorOutput(stdout.String(), stderr.String(), verbose) if parseErr != nil { @@ -180,123 +177,114 @@ func parseAndDisplayZizmorOutput(stdout, stderr string, verbose bool) (int, erro fileFindings := make(map[string][]zizmorFinding) // Parse stderr for "completed" messages to get list of files + completedFiles := parseAndDisplayZizmorOutputCompletedFiles(stderr, fileFindings) + + // Parse JSON findings from stdout + totalWarnings, err := parseAndDisplayZizmorOutputFindings(stdout, fileFindings) + if err != nil { + return 0, err + } + + // Display reformatted output for each completed file + for _, filePath := range completedFiles { + parseAndDisplayZizmorOutputFile(filePath, fileFindings[filePath]) + } + + return totalWarnings, nil +} + +func parseAndDisplayZizmorOutputCompletedFiles(stderr string, fileFindings map[string][]zizmorFinding) []string { completedFiles := []string{} scanner := bufio.NewScanner(strings.NewReader(stderr)) for scanner.Scan() { line := scanner.Text() - // Look for lines like: " INFO audit: zizmor: 🌈 completed ./.github/workflows/pdf-summary.lock.yml" if strings.Contains(line, "INFO audit: zizmor: 🌈 completed") { parts := strings.Split(line, "completed ") if len(parts) == 2 { filePath := strings.TrimSpace(parts[1]) completedFiles = append(completedFiles, filePath) - // Initialize empty findings slice if _, exists := fileFindings[filePath]; !exists { fileFindings[filePath] = []zizmorFinding{} } } } } + return completedFiles +} - // Parse JSON findings from stdout +func parseAndDisplayZizmorOutputFindings(stdout string, fileFindings map[string][]zizmorFinding) (int, error) { var findings []zizmorFinding totalWarnings := 0 - if stdout != "" && strings.HasPrefix(strings.TrimSpace(stdout), "[") { - if err := json.Unmarshal([]byte(stdout), &findings); err != nil { - return 0, fmt.Errorf("failed to parse zizmor JSON output: %w", err) - } - - // Organize findings by file - for _, finding := range findings { - // Track which files this finding affects (avoid duplicates) - affectedFiles := make(map[string]struct { - }) - for _, location := range finding.Locations { - filePath := location.Symbolic.Key.Local.GivenPath - if filePath != "" && !setutil.Contains(affectedFiles, filePath) { - affectedFiles[filePath] = struct { - }{} - fileFindings[filePath] = append(fileFindings[filePath], finding) - totalWarnings++ - } + if stdout == "" || !strings.HasPrefix(strings.TrimSpace(stdout), "[") { + return totalWarnings, nil + } + if err := json.Unmarshal([]byte(stdout), &findings); err != nil { + return 0, fmt.Errorf("failed to parse zizmor JSON output: %w", err) + } + for _, finding := range findings { + affectedFiles := make(map[string]struct{}) + for _, location := range finding.Locations { + filePath := location.Symbolic.Key.Local.GivenPath + if filePath != "" && !setutil.Contains(affectedFiles, filePath) { + affectedFiles[filePath] = struct{}{} + fileFindings[filePath] = append(fileFindings[filePath], finding) + totalWarnings++ } } } + return totalWarnings, nil +} - // Display reformatted output for each completed file - for _, filePath := range completedFiles { - findings := fileFindings[filePath] - count := len(findings) - - // Skip files with 0 warnings - if count == 0 { - continue - } - - // Read file content for context display - fileContent, err := os.ReadFile(filePath) - var fileLines []string - if err == nil { - fileLines = strings.Split(string(fileContent), "\n") - } - - // Display detailed findings using CompilerError format - for _, finding := range findings { - severity := finding.Determinations.Severity - ident := finding.Ident - desc := finding.Desc - url := finding.URL - - // Find the primary location (first location in the list) - if len(finding.Locations) > 0 { - loc := finding.Locations[0] - row := loc.Concrete.Location.StartPoint.Row - col := loc.Concrete.Location.StartPoint.Column - // Zizmor uses 0-based indexing, convert to 1-based for user display - lineNum := row + 1 - colNum := col + 1 - - // Create context lines around the error - var context []string - if len(fileLines) > 0 && lineNum > 0 && lineNum <= len(fileLines) { - startLine := max(1, lineNum-2) - endLine := min(len(fileLines), lineNum+2) - - for i := startLine; i <= endLine; i++ { - if i-1 < len(fileLines) { - context = append(context, fileLines[i-1]) - } - } - } - - // Map severity to error type - errorType := "warning" - if severity == "High" || severity == "Critical" { - errorType = "error" - } - - // Build message with URL link if available - message := fmt.Sprintf("[%s] %s: %s", severity, ident, desc) - if url != "" { - message = fmt.Sprintf("%s (%s)", message, url) - } +func parseAndDisplayZizmorOutputFile(filePath string, findings []zizmorFinding) { + if len(findings) == 0 { + return + } + fileContent, err := os.ReadFile(filePath) + var fileLines []string + if err == nil { + fileLines = strings.Split(string(fileContent), "\n") + } + for _, finding := range findings { + parseAndDisplayZizmorOutputFinding(filePath, fileLines, finding) + } +} - // Create and format CompilerError - compilerErr := console.CompilerError{ - Position: console.ErrorPosition{ - File: filePath, - Line: lineNum, - Column: colNum, - }, - Type: errorType, - Message: message, - Context: context, - } +func parseAndDisplayZizmorOutputFinding(filePath string, fileLines []string, finding zizmorFinding) { + if len(finding.Locations) == 0 { + return + } + loc := finding.Locations[0] + lineNum := loc.Concrete.Location.StartPoint.Row + 1 + colNum := loc.Concrete.Location.StartPoint.Column + 1 + context := parseAndDisplayZizmorOutputContext(fileLines, lineNum) + errorType := "warning" + if finding.Determinations.Severity == "High" || finding.Determinations.Severity == "Critical" { + errorType = "error" + } + message := fmt.Sprintf("[%s] %s: %s", finding.Determinations.Severity, finding.Ident, finding.Desc) + if finding.URL != "" { + message = fmt.Sprintf("%s (%s)", message, finding.URL) + } + compilerErr := console.CompilerError{ + Position: console.ErrorPosition{File: filePath, Line: lineNum, Column: colNum}, + Type: errorType, + Message: message, + Context: context, + } + fmt.Fprint(os.Stderr, console.FormatError(compilerErr)) +} - fmt.Fprint(os.Stderr, console.FormatError(compilerErr)) - } +func parseAndDisplayZizmorOutputContext(fileLines []string, lineNum int) []string { + var context []string + if len(fileLines) == 0 || lineNum <= 0 || lineNum > len(fileLines) { + return context + } + startLine := max(1, lineNum-2) + endLine := min(len(fileLines), lineNum+2) + for i := startLine; i <= endLine; i++ { + if i-1 < len(fileLines) { + context = append(context, fileLines[i-1]) } } - - return totalWarnings, nil + return context } diff --git a/pkg/workflow/antigravity_engine.go b/pkg/workflow/antigravity_engine.go index 2efc4232708..24eeed0a297 100644 --- a/pkg/workflow/antigravity_engine.go +++ b/pkg/workflow/antigravity_engine.go @@ -156,97 +156,81 @@ func (e *AntigravityEngine) GetExecutionSteps(workflowData *WorkflowData, logFil antigravityLog.Printf("Generating execution steps for Antigravity engine: workflow=%s, firewall=%v", workflowData.Name, isFirewallEnabled(workflowData)) var steps []GitHubActionStep - - // Write .antigravity/settings.json with context.includeDirectories and tools.core. - // This step runs after the MCP gateway setup (which may have written mcpServers config) - // and merges the context/tools settings into any existing settings.json. settingsStep := e.generateAntigravitySettingsStep(workflowData) steps = append(steps, settingsStep) - // Build agy CLI arguments based on configuration - var agyArgs []string - - // Model is passed via the native ANTIGRAVITY_MODEL environment variable only when explicitly - // configured. When not configured, the Antigravity CLI uses its built-in default model. - // This avoids embedding the value directly in the shell command (which fails template injection - // validation for GitHub Actions expressions like ${{ inputs.model }}). modelConfigured := workflowData.EngineConfig != nil && workflowData.EngineConfig.Model != "" + firewallEnabled := isFirewallEnabled(workflowData) + agyCommand := buildAntigravityCLICommand(workflowData) + command := buildAntigravityWrappedCommand(workflowData, logFile, agyCommand, firewallEnabled) + env := buildAntigravityExecutionEnv(workflowData, modelConfigured, firewallEnabled) + stepLines := []string{ + " - name: Execute Antigravity CLI", + " id: agentic_execution", + } - // Antigravity CLI reads MCP config from .antigravity/settings.json (project-level) - // The conversion script (convert_gateway_config_antigravity.sh) writes settings.json - // during the MCP setup step, so no --mcp-config flag is needed here. - - // Auto-approve all tool executions so non-interactive CI runs don't block on permission prompts. - // agy does not support the Gemini-style --yolo/--skip-trust flags. - // This flag grants broad tool permission inside the workflow sandbox, so it is only used in AWF-managed runs. - agyArgs = append(agyArgs, "--dangerously-skip-permissions") + allowedSecrets := append([]string{"GEMINI_API_KEY"}, e.GetRequiredSecretNames(workflowData)...) + filteredEnv := FilterEnvForSecrets(env, allowedSecrets) + addCliProxyGHTokenToEnv(filteredEnv, workflowData) + stepLines = FormatStepWithCommandAndEnv(stepLines, command, filteredEnv) - // Note: the --prompt argument is appended raw after shellJoinArgs below because it contains - // a shell command substitution ("$(cat ...)") that must NOT go through shellEscapeArg — - // single-quoting it would prevent shell expansion at runtime. + steps = append(steps, GitHubActionStep(stepLines)) + return steps +} - // Build the command +func buildAntigravityCLICommand(workflowData *WorkflowData) string { + agyArgs := []string{"--dangerously-skip-permissions"} commandName := "agy" if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { commandName = workflowData.EngineConfig.Command } - - // Append the prompt arg raw (not through shellJoinArgs) to preserve shell expansion agyCommand := fmt.Sprintf(`%s %s --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"`, commandName, shellJoinArgs(agyArgs)) - agyCommand = getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + agyCommand + return getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + agyCommand +} - // Build the full command with AWF wrapping if enabled - var command string - firewallEnabled := isFirewallEnabled(workflowData) - if firewallEnabled { - // Get allowed domains: prefer the pre-warmed cache on WorkflowData to avoid - // re-running the expensive map+sort operation. - var allowedDomains string - if workflowData.CachedAllowedDomainsComputed { - allowedDomains = workflowData.CachedAllowedDomainsStr - } else { - allowedDomains = GetAllowedDomainsForEngine(constants.AntigravityEngine, - workflowData.NetworkPermissions, - workflowData.Tools, - workflowData.Runtimes, - ) - } - // Add GHES/custom API target domains to the firewall allow-list when engine.api-target is set - if workflowData.EngineConfig != nil && workflowData.EngineConfig.APITarget != "" { - allowedDomains = mergeAPITargetDomains(allowedDomains, workflowData.EngineConfig.APITarget) - } - - npmPathSetup := GetNpmBinPathSetup() - agyCommandWithPath := fmt.Sprintf("%s && %s", npmPathSetup, agyCommand) - // Add MCP CLI bin directory to PATH when cli-proxy is enabled - if mcpCLIPath := GetMCPCLIPathSetup(workflowData); mcpCLIPath != "" { - agyCommandWithPath = fmt.Sprintf("%s && %s", mcpCLIPath, agyCommandWithPath) - } - - command = BuildAWFCommand(AWFCommandConfig{ - EngineName: "antigravity", - EngineCommand: agyCommandWithPath, - LogFile: logFile, - WorkflowData: workflowData, - UsesTTY: false, - AllowedDomains: allowedDomains, - // Create the agent step summary file before AWF starts so it is accessible - // inside the sandbox. The agent writes its step summary content here, and the - // file is appended to $GITHUB_STEP_SUMMARY after secret redaction. - PathSetup: "touch " + AgentStepSummaryPath, - // Exclude every env var whose step-env value is a secret so the agent - // cannot read raw token values via bash tools (env / printenv). - ExcludeEnvVarNames: ComputeAWFExcludeEnvVarNames(workflowData, []string{"ANTIGRAVITY_API_KEY", "GEMINI_API_KEY"}), - }) - } else { - command = fmt.Sprintf(`set -o pipefail +func buildAntigravityWrappedCommand(workflowData *WorkflowData, logFile, agyCommand string, firewallEnabled bool) string { + if !firewallEnabled { + return fmt.Sprintf(`set -o pipefail printf '%%s' "$(date +%%s%%3N)" > %s touch %s (umask 177 && touch %s) %s 2>&1 | tee -a %s`, AgentCLIStartMsPath, AgentStepSummaryPath, logFile, agyCommand, logFile) } + allowedDomains := resolveAntigravityAllowedDomains(workflowData) + agyCommandWithPath := fmt.Sprintf("%s && %s", GetNpmBinPathSetup(), agyCommand) + if mcpCLIPath := GetMCPCLIPathSetup(workflowData); mcpCLIPath != "" { + agyCommandWithPath = fmt.Sprintf("%s && %s", mcpCLIPath, agyCommandWithPath) + } + return BuildAWFCommand(AWFCommandConfig{ + EngineName: "antigravity", + EngineCommand: agyCommandWithPath, + LogFile: logFile, + WorkflowData: workflowData, + UsesTTY: false, + AllowedDomains: allowedDomains, + PathSetup: "touch " + AgentStepSummaryPath, + ExcludeEnvVarNames: ComputeAWFExcludeEnvVarNames(workflowData, []string{"ANTIGRAVITY_API_KEY", "GEMINI_API_KEY"}), + }) +} + +func resolveAntigravityAllowedDomains(workflowData *WorkflowData) string { + var allowedDomains string + if workflowData.CachedAllowedDomainsComputed { + allowedDomains = workflowData.CachedAllowedDomainsStr + } else { + allowedDomains = GetAllowedDomainsForEngine(constants.AntigravityEngine, + workflowData.NetworkPermissions, + workflowData.Tools, + workflowData.Runtimes, + ) + } + if workflowData.EngineConfig != nil && workflowData.EngineConfig.APITarget != "" { + allowedDomains = mergeAPITargetDomains(allowedDomains, workflowData.EngineConfig.APITarget) + } + return allowedDomains +} - // Build environment variables +func buildAntigravityExecutionEnv(workflowData *WorkflowData, modelConfigured, firewallEnabled bool) map[string]string { env := map[string]string{ "ANTIGRAVITY_API_KEY": "${{ secrets.ANTIGRAVITY_API_KEY }}", "GH_AW_PROMPT": constants.AwPromptsFile, @@ -269,8 +253,13 @@ touch %s "ANTIGRAVITY_CLI_TRUST_WORKSPACE": "true", } injectWorkflowCallNetworkAllowedEnv(env, workflowData) - // Indicate the phase: "agent" for the main run, "detection" for threat detection - // Include the compiler version so agents can identify which gh-aw version generated the workflow + applyAntigravityPhaseEnv(env, workflowData) + applyAntigravityOptionalEnv(env, workflowData, modelConfigured, firewallEnabled) + applyAntigravityCustomEnv(env, workflowData) + return env +} + +func applyAntigravityPhaseEnv(env map[string]string, workflowData *WorkflowData) { if workflowData.IsDetectionRun { env["GH_AW_PHASE"] = "detection" } else { @@ -281,27 +270,19 @@ touch %s } else { env["GH_AW_VERSION"] = "dev" } +} - // Add MCP config env var if needed (points to .antigravity/settings.json for Antigravity) +func applyAntigravityOptionalEnv(env map[string]string, workflowData *WorkflowData, modelConfigured, firewallEnabled bool) { if HasMCPServers(workflowData) { env["GH_AW_MCP_CONFIG"] = "${{ github.workspace }}/.antigravity/settings.json" } - // When the firewall (AWF) is enabled with --enable-api-proxy, point Antigravity CLI at the - // LLM gateway sidecar instead of the real googleapis.com endpoint. if firewallEnabled { env["ANTIGRAVITY_API_BASE_URL"] = fmt.Sprintf("http://host.docker.internal:%d", constants.AntigravityLLMGatewayPort) - - // Set git identity environment variables so the first git commit succeeds inside the - // container. AWF's --env-all forwards these to the container, ensuring git does not - // rely on the host-side ~/.gitconfig which is not visible in the sandbox. maps.Copy(env, getGitIdentityEnvVars()) } - // Add safe outputs env applySafeOutputEnvToMap(env, workflowData) - - // Propagate W3C trace context so engine spans nest under the gh-aw.agent.setup span. applyTraceContextEnvToMap(env) if workflowData.EngineConfig != nil && workflowData.EngineConfig.MaxTurns != "" { @@ -319,45 +300,20 @@ touch %s antigravityLog.Printf("Setting %s env var for model: %s", constants.AntigravityCLIModelEnvVar, workflowData.EngineConfig.Model) env[constants.AntigravityCLIModelEnvVar] = workflowData.EngineConfig.Model } +} - // Add custom environment variables from engine config. - // This allows users to override the default engine token expression (e.g. - // ANTIGRAVITY_API_KEY: ${{ secrets.MY_ORG_ANTIGRAVITY_KEY }}) via engine.env. +func applyAntigravityCustomEnv(env map[string]string, workflowData *WorkflowData) { applyEngineCwdEnv(env, workflowData) if workflowData.EngineConfig != nil && len(workflowData.EngineConfig.Env) > 0 { maps.Copy(env, workflowData.EngineConfig.Env) } - // Add custom environment variables from agent config agentConfig := getAgentConfig(workflowData) if agentConfig != nil && len(agentConfig.Env) > 0 { maps.Copy(env, agentConfig.Env) antigravityLog.Printf("Added %d custom env vars from agent config", len(agentConfig.Env)) } - // The Antigravity CLI and AWF's Gemini API proxy both rely on a Gemini provider key. - // Keep GEMINI_API_KEY aligned with the effective ANTIGRAVITY_API_KEY by default so the - // workflow can authenticate non-interactively without requiring users to duplicate secrets. if _, hasGeminiKey := env["GEMINI_API_KEY"]; !hasGeminiKey { env["GEMINI_API_KEY"] = env["ANTIGRAVITY_API_KEY"] } - - // Generate the execution step - stepLines := []string{ - " - name: Execute Antigravity CLI", - " id: agentic_execution", - } - - // Filter environment variables for security - allowedSecrets := append([]string{"GEMINI_API_KEY"}, e.GetRequiredSecretNames(workflowData)...) - filteredEnv := FilterEnvForSecrets(env, allowedSecrets) - - // Inject GH_TOKEN for CLI proxy (added after filtering since it uses a special - // fallback expression that is always allowed when cli-proxy is enabled) - addCliProxyGHTokenToEnv(filteredEnv, workflowData) - - // Format step with command and env - stepLines = FormatStepWithCommandAndEnv(stepLines, command, filteredEnv) - - steps = append(steps, GitHubActionStep(stepLines)) - return steps } diff --git a/pkg/workflow/awf_config.go b/pkg/workflow/awf_config.go index 12db8625bfd..49609368e7b 100644 --- a/pkg/workflow/awf_config.go +++ b/pkg/workflow/awf_config.go @@ -400,101 +400,124 @@ func BuildAWFConfigJSON(config AWFCommandConfig) (string, error) { Schema: buildAWFConfigSchemaURL(firewallConfig), } - // ── Runner section ────────────────────────────────────────────────────── + applyAWFRunnerNetworkAndPlatform(&awfConfig, config) + awfConfig.APIProxy = buildAWFAPIProxyConfig(config, firewallConfig) + applyAWFContainerConfig(&awfConfig, config, firewallConfig) + applyAWFLoggingConfig(&awfConfig, config.WorkflowData) + + jsonStr, err := jsonutil.MarshalCompactNoHTMLEscape(awfConfig) + if err != nil { + return "", fmt.Errorf("failed to marshal AWF config to JSON: %w", err) + } + + awfConfigLog.Printf("AWF config JSON generated: %d bytes", len(jsonStr)) + + if config.WorkflowData != nil && config.WorkflowData.ValidateAWFConfig { + if err := validateAWFConfigJSON(jsonStr); err != nil { + return "", fmt.Errorf("generated AWF config failed schema validation: %w", err) + } + } + + return jsonStr, nil +} + +func applyAWFRunnerNetworkAndPlatform(awfConfig *AWFConfigFile, config AWFCommandConfig) { if topology := getRunnerTopology(config.WorkflowData); topology != "" { awfConfig.Runner = &AWFRunnerConfig{Topology: topology} awfConfigLog.Printf("Runner section: topology=%s", topology) } + applyAWFNetworkConfig(awfConfig, config) + if platformType := extractPlatformType(config.WorkflowData); platformType != "" { + awfConfig.Platform = &AWFPlatformConfig{Type: platformType} + awfConfigLog.Printf("Platform section: type=%s", platformType) + } +} - // ── Network section ────────────────────────────────────────────────────── +func applyAWFNetworkConfig(awfConfig *AWFConfigFile, config AWFCommandConfig) { if config.AllowedDomains != "" { allowList := splitDomainList(config.AllowedDomains) - awfConfig.Network = &AWFNetworkConfig{ - AllowDomains: allowList, - } + awfConfig.Network = &AWFNetworkConfig{AllowDomains: allowList} awfConfigLog.Printf("Network section: %d allowed domains", len(allowList)) - - // Blocked domains (if configured in the workflow) if config.WorkflowData != nil { - blockedDomainsStr := formatBlockedDomains(config.WorkflowData.NetworkPermissions) - if blockedDomainsStr != "" { + if blockedDomainsStr := formatBlockedDomains(config.WorkflowData.NetworkPermissions); blockedDomainsStr != "" { blockList := splitDomainList(blockedDomainsStr) awfConfig.Network.BlockDomains = blockList awfConfigLog.Printf("Network section: %d blocked domains", len(blockList)) } } } + applyAWFNetworkIsolation(awfConfig, config.WorkflowData) + applyDockerSbxNetworkAllowlist(awfConfig, config.WorkflowData) +} - if isAWFNetworkIsolationEnabled(config.WorkflowData) { - if awfConfig.Network == nil { - awfConfig.Network = &AWFNetworkConfig{} - } - awfConfig.Network.Isolation = true - awfConfig.Network.TopologyAttach = buildAWFTopologyAttachList(config.WorkflowData) - awfConfigLog.Printf("Network section: isolation enabled with %d topology attachments", len(awfConfig.Network.TopologyAttach)) +func applyAWFNetworkIsolation(awfConfig *AWFConfigFile, workflowData *WorkflowData) { + if !isAWFNetworkIsolationEnabled(workflowData) { + return + } + if awfConfig.Network == nil { + awfConfig.Network = &AWFNetworkConfig{} } + awfConfig.Network.Isolation = true + awfConfig.Network.TopologyAttach = buildAWFTopologyAttachList(workflowData) + awfConfigLog.Printf("Network section: isolation enabled with %d topology attachments", len(awfConfig.Network.TopologyAttach)) +} - // docker-sbx: the sbx microVM resolves host services via host.docker.internal - // (the Docker bridge gateway, 172.17.0.1). Allow this domain so AWF's network - // policy permits connections from the microVM to the api-proxy, MCP gateway, and - // Squid proxy that are all published on the host bridge. - if isDockerSbxRuntime(config.WorkflowData) { - if awfConfig.Network == nil { - awfConfig.Network = &AWFNetworkConfig{} - } - const hostDockerInternal = "host.docker.internal" - if !slices.Contains(awfConfig.Network.AllowDomains, hostDockerInternal) { - awfConfig.Network.AllowDomains = append(awfConfig.Network.AllowDomains, hostDockerInternal) - awfConfigLog.Printf("Network section: added %s for docker-sbx microVM routing", hostDockerInternal) - } +func applyDockerSbxNetworkAllowlist(awfConfig *AWFConfigFile, workflowData *WorkflowData) { + if !isDockerSbxRuntime(workflowData) { + return + } + if awfConfig.Network == nil { + awfConfig.Network = &AWFNetworkConfig{} + } + const hostDockerInternal = "host.docker.internal" + if !slices.Contains(awfConfig.Network.AllowDomains, hostDockerInternal) { + awfConfig.Network.AllowDomains = append(awfConfig.Network.AllowDomains, hostDockerInternal) + awfConfigLog.Printf("Network section: added %s for docker-sbx microVM routing", hostDockerInternal) } +} - if platformType := extractPlatformType(config.WorkflowData); platformType != "" { - awfConfig.Platform = &AWFPlatformConfig{Type: platformType} - awfConfigLog.Printf("Platform section: type=%s", platformType) +func buildAWFAPIProxyConfig(config AWFCommandConfig, firewallConfig *FirewallConfig) *AWFAPIProxyConfig { + maxAICredits, maxRuns, maxTurnCacheMisses, enableTokenSteering := resolveAWFAPIProxyLimits(config.WorkflowData) + apiProxy := &AWFAPIProxyConfig{ + Enabled: true, + MaxRuns: maxRuns, + MaxTurnCacheMisses: maxTurnCacheMisses, + MaxAICredits: maxAICredits, + EnableTokenSteering: enableTokenSteering && awfSupportsTokenSteering(firewallConfig), } + logAWFTokenSteeringDecision(enableTokenSteering, firewallConfig) + applyAWFAPIProxyFallbackAndTargets(apiProxy, config) + applyAWFAPIProxyModels(apiProxy, config.WorkflowData) + return apiProxy +} - // ── API proxy section ───────────────────────────────────────────────────── - // maxAICredits is taken from frontmatter/imports only; when unset (0) the - // runtime value is resolved from vars.GH_AW_DEFAULT_MAX_AI_CREDITS via a - // GitHub Actions expression injected directly into the JSON string in - // BuildAWFCommand (see injectMaxAICreditsExpression in awf_helpers.go). +func resolveAWFAPIProxyLimits(workflowData *WorkflowData) (int64, int, int, bool) { maxAICredits := int64(0) maxRuns := constants.DefaultMaxRuns - // GetMaxTurnCacheMisses handles nil receiver and env-var fallback, so pre-init - // via the nil receiver avoids a redundant os.Getenv when EngineConfig is set. maxTurnCacheMisses := (*EngineConfig)(nil).GetMaxTurnCacheMisses() - if config.WorkflowData != nil && config.WorkflowData.EngineConfig != nil { - if config.WorkflowData.EngineConfig.MaxAICredits != 0 { - maxAICredits = config.WorkflowData.EngineConfig.MaxAICredits + if workflowData != nil && workflowData.EngineConfig != nil { + if workflowData.EngineConfig.MaxAICredits != 0 { + maxAICredits = workflowData.EngineConfig.MaxAICredits } - maxRuns = config.WorkflowData.EngineConfig.GetMaxRuns() - maxTurnCacheMisses = config.WorkflowData.EngineConfig.GetMaxTurnCacheMisses() + maxRuns = workflowData.EngineConfig.GetMaxRuns() + maxTurnCacheMisses = workflowData.EngineConfig.GetMaxTurnCacheMisses() } - - // Token steering is enabled by default. Setting max-ai-credits to a negative - // value (-1) omits that budget from the AWF config and disables token steering. - // When maxAICredits is 0 (runtime default), token steering stays enabled here. enableTokenSteering := maxAICredits >= 0 if maxAICredits < 0 { - // Negative signals "disabled" — omit the budget from the AWF config. maxAICredits = 0 } + return maxAICredits, maxRuns, maxTurnCacheMisses, enableTokenSteering +} - apiProxy := &AWFAPIProxyConfig{ - Enabled: true, - MaxRuns: maxRuns, - MaxTurnCacheMisses: maxTurnCacheMisses, - MaxAICredits: maxAICredits, - EnableTokenSteering: enableTokenSteering && awfSupportsTokenSteering(firewallConfig), - } - +func logAWFTokenSteeringDecision(enableTokenSteering bool, firewallConfig *FirewallConfig) { if !enableTokenSteering { awfConfigLog.Printf("Skipping apiProxy.enableTokenSteering: max-ai-credits is negative (disabled)") } else if !awfSupportsTokenSteering(firewallConfig) { awfConfigLog.Printf("Skipping apiProxy.enableTokenSteering: AWF version %q requires at least %s", getAWFImageTag(firewallConfig), constants.AWFTokenSteeringMinVersion) } +} +func applyAWFAPIProxyFallbackAndTargets(apiProxy *AWFAPIProxyConfig, config AWFCommandConfig) { if mf := extractModelFallback(config.WorkflowData); mf != nil { apiProxy.ModelFallback = mf enabledDisplay := "" @@ -503,23 +526,39 @@ func BuildAWFConfigJSON(config AWFCommandConfig) (string, error) { } awfConfigLog.Printf("API proxy: modelFallback configured: enabled=%s", enabledDisplay) } + targets := buildAWFAPITargets(config.WorkflowData, config.EngineName) + if len(targets) > 0 { + apiProxy.Targets = targets + awfConfigLog.Printf("API proxy: %d custom targets configured", len(targets)) + } +} +func buildAWFAPITargets(workflowData *WorkflowData, engineName string) map[string]*AWFAPITargetConfig { targets := map[string]*AWFAPITargetConfig{} - - if openaiTarget := extractAPITargetHost(config.WorkflowData, "OPENAI_BASE_URL"); openaiTarget != "" { - targets["openai"] = &AWFAPITargetConfig{Host: openaiTarget} - awfConfigLog.Printf("API proxy: custom openai target=%s", openaiTarget) + addAWFHostAPITarget(targets, "openai", extractAPITargetHost(workflowData, "OPENAI_BASE_URL")) + addAWFHostAPITarget(targets, "anthropic", extractAPITargetHost(workflowData, "ANTHROPIC_BASE_URL")) + applyAWFAPITargetAuthHeaders(targets, workflowData) + addAWFHostAPITarget(targets, "copilot", GetCopilotAPITarget(workflowData)) + if antigravityTarget := GetAntigravityAPITarget(workflowData, engineName); antigravityTarget != "" { + awfConfigLog.Printf("API proxy: mapped antigravity target to gemini provider target=%s", antigravityTarget) + targets["gemini"] = &AWFAPITargetConfig{Host: antigravityTarget} + } else { + addAWFHostAPITarget(targets, "gemini", GetGeminiAPITarget(workflowData, engineName)) } - if anthropicTarget := extractAPITargetHost(config.WorkflowData, "ANTHROPIC_BASE_URL"); anthropicTarget != "" { - targets["anthropic"] = &AWFAPITargetConfig{Host: anthropicTarget} - awfConfigLog.Printf("API proxy: custom anthropic target=%s", anthropicTarget) + return targets +} + +func addAWFHostAPITarget(targets map[string]*AWFAPITargetConfig, provider, host string) { + if host == "" { + return } + targets[provider] = &AWFAPITargetConfig{Host: host} + awfConfigLog.Printf("API proxy: custom %s target=%s", provider, host) +} - // Apply authHeader overrides from sandbox.agent.targets frontmatter. - // These are independent of the host/env-var settings: authHeader can be set - // even when no custom host is configured. +func applyAWFAPITargetAuthHeaders(targets map[string]*AWFAPITargetConfig, workflowData *WorkflowData) { for _, provider := range []string{"openai", "anthropic"} { - authHeader := extractAPITargetAuthHeader(config.WorkflowData, provider) + authHeader := extractAPITargetAuthHeader(workflowData, provider) if authHeader == "" { continue } @@ -530,31 +569,14 @@ func BuildAWFConfigJSON(config AWFCommandConfig) (string, error) { } awfConfigLog.Printf("API proxy: custom %s authHeader=%s", provider, authHeader) } - if copilotTarget := GetCopilotAPITarget(config.WorkflowData); copilotTarget != "" { - targets["copilot"] = &AWFAPITargetConfig{Host: copilotTarget} - awfConfigLog.Printf("API proxy: custom copilot target=%s", copilotTarget) - } - if antigravityTarget := GetAntigravityAPITarget(config.WorkflowData, config.EngineName); antigravityTarget != "" { - // Route the Antigravity-resolved API target through the "gemini" provider key - // to match AWF's supported target providers. - awfConfigLog.Printf("API proxy: mapped antigravity target to gemini provider target=%s", antigravityTarget) - targets["gemini"] = &AWFAPITargetConfig{Host: antigravityTarget} - } else if geminiTarget := GetGeminiAPITarget(config.WorkflowData, config.EngineName); geminiTarget != "" { - awfConfigLog.Printf("API proxy: custom gemini target=%s", geminiTarget) - targets["gemini"] = &AWFAPITargetConfig{Host: geminiTarget} - } - - if len(targets) > 0 { - apiProxy.Targets = targets - awfConfigLog.Printf("API proxy: %d custom targets configured", len(targets)) - } +} - // ── Models section (nested under apiProxy per AWF config schema) ────────── - if config.WorkflowData != nil && len(config.WorkflowData.ModelMappings) > 0 { - apiProxy.Models = config.WorkflowData.ModelMappings - awfConfigLog.Printf("Models section: %d alias entries", len(config.WorkflowData.ModelMappings)) +func applyAWFAPIProxyModels(apiProxy *AWFAPIProxyConfig, workflowData *WorkflowData) { + if workflowData != nil && len(workflowData.ModelMappings) > 0 { + apiProxy.Models = workflowData.ModelMappings + awfConfigLog.Printf("Models section: %d alias entries", len(workflowData.ModelMappings)) } - allowedModels, disallowedModels := resolveModelPolicyForAWFConfig(config.WorkflowData) + allowedModels, disallowedModels := resolveModelPolicyForAWFConfig(workflowData) if len(allowedModels) > 0 { apiProxy.AllowedModels = allowedModels awfConfigLog.Printf("Models policy: %d allowed model pattern(s)", len(allowedModels)) @@ -563,77 +585,50 @@ func BuildAWFConfigJSON(config AWFCommandConfig) (string, error) { apiProxy.DisallowedModels = disallowedModels awfConfigLog.Printf("Models policy: %d disallowed model pattern(s)", len(disallowedModels)) } +} - awfConfig.APIProxy = apiProxy - - // ── Container section ───────────────────────────────────────────────────── +func applyAWFContainerConfig(awfConfig *AWFConfigFile, config AWFCommandConfig, firewallConfig *FirewallConfig) { awfImageTag := buildAWFImageTagWithDigests(getAWFImageTag(firewallConfig), config.WorkflowData) agentRuntime := getAgentContainerRuntime(config.WorkflowData) agentTimeout := 0 if isDockerSbxRuntime(config.WorkflowData) { agentTimeout = resolveAWFContainerAgentTimeoutMinutes(config.WorkflowData) } - // containerRuntime is only emitted when the effective AWF version supports it. - // Gate here to avoid sending an unrecognised field to older AWF binaries. if !awfSupportsContainerRuntime(firewallConfig) { if agentRuntime != "" { awfConfigLog.Printf("Skipping containerRuntime: AWF version %q requires at least %s (gh-aw-firewall#6093)", getAWFImageTag(firewallConfig), constants.AWFContainerRuntimeMinVersion) } agentRuntime = "" } - if awfImageTag != "" || isArcDindTopology(config.WorkflowData) || agentRuntime != "" || agentTimeout > 0 { - container := &AWFContainerConfig{ - ImageTag: awfImageTag, - AgentTimeout: agentTimeout, - ContainerRuntime: agentRuntime, - } - // NOTE: dockerHostPathPrefix is intentionally NOT set for arc-dind topology. - // With sysroot-stage active, the Docker daemon can access all needed paths: - // - Workspace & RUNNER_TEMP: on the shared work volume (/home/runner/_work/) - // - System binaries: provided by the sysroot named volume (not bind mounts) - // - Kernel VFS (/dev, /sys): daemon's own kernel - // Setting a prefix would incorrectly translate the workspace mount source to - // a non-existent path (e.g. /prefix/home/runner/_work/repo → empty dir), - // causing the agent to see an empty workspace. See gh-aw#34896. - awfConfig.Container = container - if awfImageTag != "" { - awfConfigLog.Printf("Container section: image_tag=%s", awfImageTag) - } - if agentRuntime != "" { - awfConfigLog.Printf("Container section: containerRuntime=%s", agentRuntime) - } - if agentTimeout > 0 { - awfConfigLog.Printf("Container section: agentTimeout=%d", agentTimeout) - } + if awfImageTag == "" && !isArcDindTopology(config.WorkflowData) && agentRuntime == "" && agentTimeout <= 0 { + return + } + awfConfig.Container = &AWFContainerConfig{ImageTag: awfImageTag, AgentTimeout: agentTimeout, ContainerRuntime: agentRuntime} + logAWFContainerConfig(awfImageTag, agentRuntime, agentTimeout) +} + +func logAWFContainerConfig(awfImageTag, agentRuntime string, agentTimeout int) { + if awfImageTag != "" { + awfConfigLog.Printf("Container section: image_tag=%s", awfImageTag) + } + if agentRuntime != "" { + awfConfigLog.Printf("Container section: containerRuntime=%s", agentRuntime) + } + if agentTimeout > 0 { + awfConfigLog.Printf("Container section: agentTimeout=%d", agentTimeout) } +} - // ── Logging section ────────────────────────────────────────────────────── - // Logging paths are set in config. For ARC/DinD, the config file is written at runtime, - // so ${RUNNER_TEMP} can be preserved for shell expansion before AWF reads the JSON. +func applyAWFLoggingConfig(awfConfig *AWFConfigFile, workflowData *WorkflowData) { awfConfig.Logging = &AWFLoggingConfig{ ProxyLogsDir: string(constants.AWFProxyLogsDir), AuditDir: string(constants.AWFAuditDir), } - if isArcDindTopology(config.WorkflowData) { + if isArcDindTopology(workflowData) { awfConfig.Logging.ProxyLogsDir = awfArcDindProxyLogsDirExpr awfConfig.Logging.AuditDir = awfArcDindAuditDirExpr } awfConfigLog.Printf("Logging section: proxyLogsDir=%s, auditDir=%s", awfConfig.Logging.ProxyLogsDir, awfConfig.Logging.AuditDir) - - jsonStr, err := jsonutil.MarshalCompactNoHTMLEscape(awfConfig) - if err != nil { - return "", fmt.Errorf("failed to marshal AWF config to JSON: %w", err) - } - - awfConfigLog.Printf("AWF config JSON generated: %d bytes", len(jsonStr)) - - if config.WorkflowData != nil && config.WorkflowData.ValidateAWFConfig { - if err := validateAWFConfigJSON(jsonStr); err != nil { - return "", fmt.Errorf("generated AWF config failed schema validation: %w", err) - } - } - - return jsonStr, nil } func resolveAWFContainerAgentTimeoutMinutes(workflowData *WorkflowData) int { diff --git a/pkg/workflow/awf_helpers.go b/pkg/workflow/awf_helpers.go index fcb0b5db3d8..c38c5ddba56 100644 --- a/pkg/workflow/awf_helpers.go +++ b/pkg/workflow/awf_helpers.go @@ -213,34 +213,7 @@ func buildWorkflowCallNetworkAllowedUpdateScript() (string, error) { shellEscapeArg(string(ecosystemJSON))), nil } -// BuildAWFCommand builds a complete AWF command with all arguments. -// This consolidates the AWF command building logic that was duplicated across -// Copilot, Claude, and Codex engines. -// -// Parameters: -// - config: AWF command configuration -// -// Returns: -// - string: Complete AWF command with arguments and wrapped engine command -func BuildAWFCommand(config AWFCommandConfig) string { - awfHelpersLog.Printf("Building AWF command for engine: %s", config.EngineName) - isArcDind := isArcDindTopology(config.WorkflowData) - - // Get AWF command prefix (custom or standard) - awfCommand := GetAWFCommandPrefix(config.WorkflowData) - - // Build AWF arguments. The returned list contains only args that are safe to pass - // through shellJoinArgs. Expandable-var args (--container-workdir "${GITHUB_WORKSPACE}" - // and --mount "${RUNNER_TEMP}/...") are appended raw below so that shell variable - // expansion is not suppressed by single-quoting. - awfArgs := BuildAWFArgs(config) - firewallConfig := getFirewallConfig(config.WorkflowData) - - // Auto-detect ARC/DinD split daemon topology at runtime: probe DOCKER_HOST for a - // tcp:// scheme and pass it through to AWF via --docker-host. - // All behaviors avoid requiring workflow-authored sandbox.agent.args for standard ARC DinD setups. - // When AWF also supports chroot config (v0.27.1+), the Python patch body is embedded inside - // the same if-block so the script only contains one DOCKER_HOST condition check. +func buildArcDindProbes(config AWFCommandConfig, firewallConfig *FirewallConfig) (string, string, string) { arcDindPrefixProbe := "" arcDindDockerHostProbe := fmt.Sprintf(`%s="" if [[ "${DOCKER_HOST:-}" =~ %s ]]; then @@ -251,227 +224,153 @@ fi`, awfDockerHostVarName, ) arcDindDockerHostRef := fmt.Sprintf("${%s:+--docker-host \"$%s\"}", awfDockerHostVarName, awfDockerHostVarName) - if awfSupportsDockerHostPathPrefix(firewallConfig) { - chrootPatchBody := "" - if awfSupportsChrootConfig(firewallConfig) { - if config.WorkflowData != nil && config.WorkflowData.IsDetectionRun { - chrootPatchBody = "\n" + buildArcDindChrootConfigPatchBodyBash() - } else { - chrootPatchBody = "\n" + buildArcDindChrootConfigPatchBody() - } + if !awfSupportsDockerHostPathPrefix(firewallConfig) { + return arcDindPrefixProbe, arcDindDockerHostProbe, arcDindDockerHostRef + } + chrootPatchBody := "" + if awfSupportsChrootConfig(firewallConfig) { + if config.WorkflowData != nil && config.WorkflowData.IsDetectionRun { + chrootPatchBody = "\n" + buildArcDindChrootConfigPatchBodyBash() + } else { + chrootPatchBody = "\n" + buildArcDindChrootConfigPatchBody() } - // NOTE: --docker-host-path-prefix is intentionally NOT passed. With sysroot-stage - // active, all bind-mount source paths are on the shared work volume and visible to - // the Docker daemon without translation. The prefix caused AWF to translate - // GITHUB_WORKSPACE to a non-existent path, resulting in an empty workspace (gh-aw#34896). - // The probe block is preserved for the chroot config patch which still requires the - // DOCKER_HOST guard. - if chrootPatchBody != "" { - arcDindPrefixProbe = fmt.Sprintf(`if [[ "${DOCKER_HOST:-}" =~ %s ]]; then%s + } + if chrootPatchBody != "" { + arcDindPrefixProbe = fmt.Sprintf(`if [[ "${DOCKER_HOST:-}" =~ %s ]]; then%s fi`, - awfArcDindDockerHostRegex, - chrootPatchBody) - } + awfArcDindDockerHostRegex, + chrootPatchBody) } - toolCacheMountProbe := fmt.Sprintf(`%s="" + return arcDindPrefixProbe, arcDindDockerHostProbe, arcDindDockerHostRef +} + +func buildToolCacheMountProbe() (string, string) { + return fmt.Sprintf(`%s="" GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" if [ -d "$GH_AW_TOOL_CACHE" ]; then if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then %s="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" fi fi`, - awfToolCacheMountVarName, - awfToolCacheMountVarName, - ) - toolCacheMountRef := fmt.Sprintf("${%s:+--mount \"$%s\"}", awfToolCacheMountVarName, awfToolCacheMountVarName) + awfToolCacheMountVarName, + awfToolCacheMountVarName, + ), + fmt.Sprintf("${%s:+--mount \"$%s\"}", awfToolCacheMountVarName, awfToolCacheMountVarName) +} - // Build the expandable args string for args that need shell variable expansion. - // These MUST be appended as raw (unescaped) strings because single-quoting would - // prevent the runner's shell from expanding ${GITHUB_WORKSPACE} and ${RUNNER_TEMP}. +func buildAWFExpandableArgs(isArcDind bool, arcDindDockerHostProbe string) (string, string) { ghAwDir := constants.GhAwRootDirShell expandableArgs := fmt.Sprintf( `--container-workdir "${GITHUB_WORKSPACE}" --mount "%s:%s:ro" --mount "%s:/host%s:ro"`, ghAwDir, ghAwDir, ghAwDir, ghAwDir, ) - if isArcDind { - expandableArgs += fmt.Sprintf( - ` --mount "%s:%s:rw" --mount "%s:%s:rw"`, - awfArcDindHomePathExpr, awfArcDindHomePathExpr, - awfArcDindRootPathExpr+"/sandbox/agent", awfArcDindRootPathExpr+"/sandbox/agent", - ) - // Explicitly mount the workspace so AWF can see it without path-prefix translation. - // GITHUB_WORKSPACE is on the shared work volume, so the Docker daemon can access it. - expandableArgs += ` --mount "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}:rw"` - // Pre-create the rw mount source directories. AWF validates that mount source - // paths exist before starting containers, so these must be created on the host - // before the AWF invocation. The parent ${RUNNER_TEMP}/gh-aw/ already exists - // (created by actions/setup), but the subdirectories may not. - arcDindDockerHostProbe += fmt.Sprintf("\nmkdir -p \"%s\" \"%s\"", - awfArcDindHomePathExpr, - awfArcDindRootPathExpr+"/sandbox/agent", - ) - // Copy prompt files to daemon-visible path. On ARC/DinD, /tmp/gh-aw/ is NOT - // accessible to the Docker daemon. The activation job writes prompts to - // /tmp/gh-aw/aw-prompts/, so we copy them to ${RUNNER_TEMP}/gh-aw/aw-prompts/. - arcDindDockerHostProbe += fmt.Sprintf("\nif [ -d /tmp/gh-aw/aw-prompts ]; then cp -a /tmp/gh-aw/aw-prompts \"%s/aw-prompts\"; fi", - awfArcDindRootPathExpr, - ) - } - - // Generate a JSON config file and reference it via --config "${RUNNER_TEMP}/gh-aw/awf-config.json". - // This replaces several verbose CLI flags (--allow-domains, --enable-api-proxy, --image-tag, - // API targets) with a structured JSON file that is easier to audit and extend. - // - // The config file is written at runtime (inside the run: step) immediately before the AWF - // invocation, using printf to a fixed path inside the pre-existing ${RUNNER_TEMP}/gh-aw/ - // directory that is already set up by actions/setup. - var configFileSetup string + if !isArcDind { + return expandableArgs, arcDindDockerHostProbe + } + expandableArgs += fmt.Sprintf( + ` --mount "%s:%s:rw" --mount "%s:%s:rw"`, + awfArcDindHomePathExpr, awfArcDindHomePathExpr, + awfArcDindRootPathExpr+"/sandbox/agent", awfArcDindRootPathExpr+"/sandbox/agent", + ) + expandableArgs += ` --mount "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}:rw"` + arcDindDockerHostProbe += fmt.Sprintf("\nmkdir -p \"%s\" \"%s\"", + awfArcDindHomePathExpr, + awfArcDindRootPathExpr+"/sandbox/agent", + ) + arcDindDockerHostProbe += fmt.Sprintf("\nif [ -d /tmp/gh-aw/aw-prompts ]; then cp -a /tmp/gh-aw/aw-prompts \"%s/aw-prompts\"; fi", + awfArcDindRootPathExpr, + ) + return expandableArgs, arcDindDockerHostProbe +} + +func buildMaxAICreditsExportLine(config AWFCommandConfig) string { + if config.WorkflowData != nil && config.WorkflowData.EngineConfig != nil && config.WorkflowData.EngineConfig.MaxAICredits != 0 { + return "" + } + defaultMaxAICredits := strconv.FormatInt(constants.DefaultMaxAICredits, 10) + if config.WorkflowData != nil && config.WorkflowData.IsDetectionRun { + defaultMaxAICredits = strconv.FormatInt(constants.DefaultDetectionMaxAICredits, 10) + } + if config.ResolveMaxAICreditsFromEnv { + return fmt.Sprintf(`%s="${%s:-%s}"`, awfMaxAICreditsVarName, awfMaxAICreditsVarName, defaultMaxAICredits) + } + expr := compilerenv.BuildDefaultMaxAICreditsExpression(defaultMaxAICredits) + if config.WorkflowData != nil && config.WorkflowData.IsDetectionRun { + expr = compilerenv.BuildDefaultDetectionMaxAICreditsExpression(defaultMaxAICredits) + } + return fmt.Sprintf(`%s="%s"`, awfMaxAICreditsVarName, expr) +} + +func buildAWFConfigPrintfArg(awfConfigJSON, maxAICreditsExportLine string) string { + preservedVars := make([]string, 0, 2) + if maxAICreditsExportLine != "" { + preservedVars = append(preservedVars, awfMaxAICreditsVarName) + } + if strings.Contains(awfConfigJSON, awfArcDindRootPathExpr) { + preservedVars = append(preservedVars, "RUNNER_TEMP") + } + if len(preservedVars) > 0 { + return shellEscapeArgWithVarsPreserved(awfConfigJSON, preservedVars...) + } + return shellEscapeArg(awfConfigJSON) +} + +func buildAWFConfigFileSetup(config AWFCommandConfig, expandableArgs string) (string, string) { awfConfigJSON, err := BuildAWFConfigJSON(config) if err != nil { awfHelpersLog.Printf("Warning: failed to build AWF config JSON: %v", err) - } else { - // When max-ai-credits is not set by frontmatter/imports, export a local shell - // variable (GH_AW_MAX_AI_CREDITS) holding a GitHub Actions runtime expression, - // then inject a reference to that variable (${GH_AW_MAX_AI_CREDITS}) into the - // "maxAiCredits" field of the apiProxy JSON object. GitHub Actions evaluates - // the ${{ }} expression before the shell runs, so the variable is set to the - // resolved integer by the time printf writes the config file. - // - // Standard agent runs use vars.GH_AW_DEFAULT_MAX_AI_CREDITS with built-in - // fallback 1000. Threat-detection runs use - // vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS with built-in fallback 400. - // EngineConfig.MaxAICredits is 0 when no compile-time value was set - // (neither frontmatter nor detection-engine config provided one). - // In that case, emit a runtime expression that lets the org variable - // or the built-in default resolve the budget at action run time. - // For detection runs, use the detection-specific variable/fallback; - // for standard agent runs, use the main-agent variable/fallback. - var maxAICreditsExportLine string - if config.WorkflowData == nil || config.WorkflowData.EngineConfig == nil || config.WorkflowData.EngineConfig.MaxAICredits == 0 { - defaultMaxAICredits := strconv.FormatInt(constants.DefaultMaxAICredits, 10) - if config.WorkflowData != nil && config.WorkflowData.IsDetectionRun { - defaultMaxAICredits = strconv.FormatInt(constants.DefaultDetectionMaxAICredits, 10) - } - awfConfigJSON = injectMaxAICreditsExpression(awfConfigJSON, fmt.Sprintf("${%s}", awfMaxAICreditsVarName)) - if config.ResolveMaxAICreditsFromEnv { - maxAICreditsExportLine = fmt.Sprintf(`%s="${%s:-%s}"`, awfMaxAICreditsVarName, awfMaxAICreditsVarName, defaultMaxAICredits) - } else { - expr := compilerenv.BuildDefaultMaxAICreditsExpression(defaultMaxAICredits) - if config.WorkflowData != nil && config.WorkflowData.IsDetectionRun { - expr = compilerenv.BuildDefaultDetectionMaxAICreditsExpression(defaultMaxAICredits) - } - maxAICreditsExportLine = fmt.Sprintf(`%s="%s"`, awfMaxAICreditsVarName, expr) - } - awfHelpersLog.Printf("Injected maxAiCredits local var reference into AWF config JSON") - } - // Write the config JSON to ${RUNNER_TEMP}/gh-aw/awf-config.json before AWF runs. - // When the generated JSON contains compiler-owned runtime variables such as - // ${GH_AW_MAX_AI_CREDITS} or ${RUNNER_TEMP}, use shellEscapeArgWithVarsPreserved - // which always uses double-quote wrapping: it escapes bare $ signs (e.g. - // "$schema" → "\$schema") while preserving both ${{ }} GitHub Actions expressions - // (e.g. in AllowedDomains) and approved shell variable references so bash expands - // them to runtime-resolved values. When no such variables are injected, - // shellEscapeArg handles escaping normally. - // Also copy it to /tmp/gh-aw/awf-config.json for the unified agent artifact upload. - var printfArg string - preservedVars := make([]string, 0, 2) - if maxAICreditsExportLine != "" { - preservedVars = append(preservedVars, awfMaxAICreditsVarName) - } - if strings.Contains(awfConfigJSON, awfArcDindRootPathExpr) { - preservedVars = append(preservedVars, "RUNNER_TEMP") - } - if len(preservedVars) > 0 { - printfArg = shellEscapeArgWithVarsPreserved(awfConfigJSON, preservedVars...) + return "", expandableArgs + } + maxAICreditsExportLine := buildMaxAICreditsExportLine(config) + if maxAICreditsExportLine != "" { + awfConfigJSON = injectMaxAICreditsExpression(awfConfigJSON, fmt.Sprintf("${%s}", awfMaxAICreditsVarName)) + awfHelpersLog.Printf("Injected maxAiCredits local var reference into AWF config JSON") + } + configFileSetup := fmt.Sprintf( + "printf '%%s\\n' %s > %q", + buildAWFConfigPrintfArg(awfConfigJSON, maxAICreditsExportLine), + awfConfigRuntimePathExpr, + ) + if maxAICreditsExportLine != "" { + configFileSetup = maxAICreditsExportLine + "\n" + configFileSetup + } + if shouldUseWorkflowCallNetworkAllowedInput(config.WorkflowData) { + updateScript, updateErr := buildWorkflowCallNetworkAllowedUpdateScript() + if updateErr != nil { + awfHelpersLog.Printf("Warning: failed to build workflow_call network_allowed updater: %v", updateErr) } else { - printfArg = shellEscapeArg(awfConfigJSON) - } - configFileSetup = fmt.Sprintf( - "printf '%%s\\n' %s > %q", - printfArg, - awfConfigRuntimePathExpr, - ) - if maxAICreditsExportLine != "" { - configFileSetup = maxAICreditsExportLine + "\n" + configFileSetup - } - if shouldUseWorkflowCallNetworkAllowedInput(config.WorkflowData) { - updateScript, updateErr := buildWorkflowCallNetworkAllowedUpdateScript() - if updateErr != nil { - awfHelpersLog.Printf("Warning: failed to build workflow_call network_allowed updater: %v", updateErr) - } else { - configFileSetup += "\n" + updateScript - } + configFileSetup += "\n" + updateScript } - configFileSetup += fmt.Sprintf("\ncp %q %s", awfConfigRuntimePathExpr, constants.AWFConfigFilePath) - // Add --config as the first expandable arg so it appears before --container-workdir. - expandableArgs = fmt.Sprintf("--config %q ", awfConfigRuntimePathExpr) + expandableArgs - awfHelpersLog.Print("Using AWF config file (--config flag)") } - modelsJSONPathExport := buildModelsJSONPathExportScript(isArcDind) + configFileSetup += fmt.Sprintf("\ncp %q %s", awfConfigRuntimePathExpr, constants.AWFConfigFilePath) + return configFileSetup, fmt.Sprintf("--config %q ", awfConfigRuntimePathExpr) + expandableArgs +} - // When upload_artifact is configured, add a read-write mount for the staging directory - // so the model can copy files there from inside the container. The parent ${RUNNER_TEMP}/gh-aw - // is mounted :ro above; this child mount overrides access for the staging subdirectory only. - // The staging directory must already exist on the host (created in Generate Safe Outputs Config step). - if config.WorkflowData != nil && config.WorkflowData.SafeOutputs != nil && config.WorkflowData.SafeOutputs.UploadArtifact != nil { +func appendAWFUploadArtifactMount(expandableArgs string, workflowData *WorkflowData) string { + if workflowData != nil && workflowData.SafeOutputs != nil && workflowData.SafeOutputs.UploadArtifact != nil { stagingDir := SafeOutputsUploadArtifactsDir expandableArgs += fmt.Sprintf(` --mount "%s:%s:rw"`, stagingDir, stagingDir) awfHelpersLog.Print("Added read-write mount for upload_artifact staging directory") } + return expandableArgs +} - // Add --allow-host-service-ports for services with port mappings. - // This flag requires --legacy-security since it grants host network access. - // This is appended as a raw (expandable) arg because the value contains - // ${{ job.services..ports[''] }} expressions that include single quotes. - // These expressions are resolved by the GitHub Actions runner before shell execution, - // so they must not be shell-escaped. - agentCfg := getAgentConfig(config.WorkflowData) +func appendAWFServicePortArgs(expandableArgs string, workflowData *WorkflowData) string { + agentCfg := getAgentConfig(workflowData) isLegacyMode := agentCfg != nil && agentCfg.LegacySecurity - if config.WorkflowData != nil && config.WorkflowData.ServicePortExpressions != "" && isLegacyMode { - expandableArgs += fmt.Sprintf(` --allow-host-service-ports "%s"`, config.WorkflowData.ServicePortExpressions) - awfHelpersLog.Printf("Added --allow-host-service-ports with %s", config.WorkflowData.ServicePortExpressions) - } else if config.WorkflowData != nil && config.WorkflowData.ServicePortExpressions != "" { + if workflowData != nil && workflowData.ServicePortExpressions != "" && isLegacyMode { + expandableArgs += fmt.Sprintf(` --allow-host-service-ports "%s"`, workflowData.ServicePortExpressions) + awfHelpersLog.Printf("Added --allow-host-service-ports with %s", workflowData.ServicePortExpressions) + } else if workflowData != nil && workflowData.ServicePortExpressions != "" { awfHelpersLog.Print("Skipping --allow-host-service-ports: requires legacy-security mode") } + return expandableArgs +} - engineCommand := config.EngineCommand - if isArcDind { - engineCommand = rewriteArcDindEngineCommand(engineCommand) - } - - // Wrap engine command in shell (command already includes any internal setup like npm PATH) - shellWrappedCommand := WrapCommandInShell(engineCommand) - - // Pre-create the agent stdio log file with restrictive permissions (0600) before - // starting the AWF container. tee would otherwise create it with the default - // umask (0644), leaving secrets (e.g. MCP gateway tokens) world-readable on the - // runner host until the secret-redaction step runs. - preCreateLog := fmt.Sprintf("(umask 177 && touch %s)", shellEscapeArg(config.LogFile)) - - // Capture the epoch-millisecond timestamp at the very start of the Execute Agent CLI - // step on the host, before the AWF container launches. sendJobConclusionSpan reads - // this file to set the dedicated gh-aw..agent span start time, which excludes - // pre-agent overhead such as workspace audit and CLI proxy startup. - writeAgentCLIStartMs := "printf '%s' \"$(date +%s%3N)\" > " + shellEscapeArg(AgentCLIStartMsPath) - - // Build the complete command with proper formatting. - // configFileSetup (if non-empty) writes the AWF config JSON immediately before the - // AWF invocation so the file is present when AWF parses --config. - // - // shellcheck directive rationale: - // - SC1003 is expected because this generated block intentionally contains GitHub - // expression literals (for example ${{ job.services..ports[''] }}) - // that include single quotes and must survive into runtime unchanged. - // - SC2086 is expected because a subset of AWF arguments are intentionally emitted - // as expandable shell fragments (for example ${GH_AW_TOOL_CACHE_MOUNT:+...} and - // ${GH_AW_DOCKER_HOST:+...}). These fragments are produced by trusted - // compiler-owned probes above and are not user-provided free-form shell input. - // - // We keep normal quoting for all user-controlled values via shellEscapeArg/shellJoinArgs - // and scope this suppression to the generated AWF invocation line only. - var command string - if config.PathSetup != "" && configFileSetup != "" { - command = fmt.Sprintf(`set -o pipefail +func buildAWFInvocationCommand(config AWFCommandConfig, commandParts awfInvocationParts) string { + commonArgs := buildAWFInvocationArgs(config, commandParts) + if config.PathSetup != "" && commandParts.configFileSetup != "" { + return fmt.Sprintf(`set -o pipefail %s %s %s @@ -482,26 +381,11 @@ fi`, %s %s %s %s %s %s %s \ - -- %s 2>&1 | tee -a %s`, - writeAgentCLIStartMs, - config.PathSetup, - preCreateLog, - configFileSetup, - modelsJSONPathExport, - arcDindDockerHostProbe, - arcDindPrefixProbe, - toolCacheMountProbe, - awfShellcheckDirective, - awfCommand, - expandableArgs, - toolCacheMountRef, - arcDindDockerHostRef, - shellJoinArgs(awfArgs), - shellWrappedCommand, - shellEscapeArg(config.LogFile)) - } else if config.PathSetup != "" { - // Include path setup before AWF command (runs on host before AWF) - command = fmt.Sprintf(`set -o pipefail + -- %s 2>&1 | tee -a %s`, append([]any{commonArgs[0], config.PathSetup}, commonArgs[1:]...)...) + } + if config.PathSetup != "" { + args := append([]any{commonArgs[0], config.PathSetup, commonArgs[1]}, commonArgs[3:]...) + return fmt.Sprintf(`set -o pipefail %s %s %s @@ -511,24 +395,10 @@ fi`, %s %s %s %s %s %s %s \ - -- %s 2>&1 | tee -a %s`, - writeAgentCLIStartMs, - config.PathSetup, - preCreateLog, - modelsJSONPathExport, - arcDindDockerHostProbe, - arcDindPrefixProbe, - toolCacheMountProbe, - awfShellcheckDirective, - awfCommand, - expandableArgs, - toolCacheMountRef, - arcDindDockerHostRef, - shellJoinArgs(awfArgs), - shellWrappedCommand, - shellEscapeArg(config.LogFile)) - } else if configFileSetup != "" { - command = fmt.Sprintf(`set -o pipefail + -- %s 2>&1 | tee -a %s`, args...) + } + if commandParts.configFileSetup != "" { + return fmt.Sprintf(`set -o pipefail %s %s %s @@ -538,24 +408,9 @@ fi`, %s %s %s %s %s %s %s \ - -- %s 2>&1 | tee -a %s`, - writeAgentCLIStartMs, - preCreateLog, - configFileSetup, - modelsJSONPathExport, - arcDindDockerHostProbe, - arcDindPrefixProbe, - toolCacheMountProbe, - awfShellcheckDirective, - awfCommand, - expandableArgs, - toolCacheMountRef, - arcDindDockerHostRef, - shellJoinArgs(awfArgs), - shellWrappedCommand, - shellEscapeArg(config.LogFile)) - } else { - command = fmt.Sprintf(`set -o pipefail + -- %s 2>&1 | tee -a %s`, commonArgs...) + } + return fmt.Sprintf(`set -o pipefail %s %s %s @@ -564,25 +419,107 @@ fi`, %s %s %s %s %s %s %s \ - -- %s 2>&1 | tee -a %s`, - writeAgentCLIStartMs, - preCreateLog, - modelsJSONPathExport, - arcDindDockerHostProbe, - arcDindPrefixProbe, - toolCacheMountProbe, - awfShellcheckDirective, - awfCommand, - expandableArgs, - toolCacheMountRef, - arcDindDockerHostRef, - shellJoinArgs(awfArgs), - shellWrappedCommand, - shellEscapeArg(config.LogFile)) + -- %s 2>&1 | tee -a %s`, append(commonArgs[:2], commonArgs[3:]...)...) +} + +func buildAWFInvocationArgs(config AWFCommandConfig, commandParts awfInvocationParts) []any { + return []any{ + commandParts.writeAgentCLIStartMs, + commandParts.preCreateLog, + commandParts.configFileSetup, + commandParts.modelsJSONPathExport, + commandParts.arcDindDockerHostProbe, + commandParts.arcDindPrefixProbe, + commandParts.toolCacheMountProbe, + awfShellcheckDirective, + commandParts.awfCommand, + commandParts.expandableArgs, + commandParts.toolCacheMountRef, + commandParts.arcDindDockerHostRef, + shellJoinArgs(commandParts.awfArgs), + commandParts.shellWrappedCommand, + shellEscapeArg(config.LogFile), + } +} + +type awfInvocationParts struct { + awfCommand string + awfArgs []string + expandableArgs string + arcDindPrefixProbe string + arcDindDockerHostProbe string + arcDindDockerHostRef string + toolCacheMountProbe string + toolCacheMountRef string + configFileSetup string + modelsJSONPathExport string + shellWrappedCommand string + preCreateLog string + writeAgentCLIStartMs string +} + +// BuildAWFCommand builds a complete AWF command with all arguments. +// This consolidates the AWF command building logic that was duplicated across +// Copilot, Claude, and Codex engines. +// +// Parameters: +// - config: AWF command configuration +// +// Returns: +// - string: Complete AWF command with arguments and wrapped engine command +func BuildAWFCommand(config AWFCommandConfig) string { + awfHelpersLog.Printf("Building AWF command for engine: %s", config.EngineName) + isArcDind := isArcDindTopology(config.WorkflowData) + awfCommand := GetAWFCommandPrefix(config.WorkflowData) + awfArgs := BuildAWFArgs(config) + firewallConfig := getFirewallConfig(config.WorkflowData) + arcDindPrefixProbe, arcDindDockerHostProbe, arcDindDockerHostRef := buildArcDindProbes(config, firewallConfig) + toolCacheMountProbe, toolCacheMountRef := buildToolCacheMountProbe() + expandableArgs, arcDindDockerHostProbe := buildAWFExpandableArgs(isArcDind, arcDindDockerHostProbe) + configFileSetup, expandableArgs := buildAWFConfigFileSetup(config, expandableArgs) + if configFileSetup != "" { + awfHelpersLog.Print("Using AWF config file (--config flag)") + } + modelsJSONPathExport := buildModelsJSONPathExportScript(isArcDind) + expandableArgs = appendAWFUploadArtifactMount(expandableArgs, config.WorkflowData) + expandableArgs = appendAWFServicePortArgs(expandableArgs, config.WorkflowData) + + engineCommand := config.EngineCommand + if isArcDind { + engineCommand = rewriteArcDindEngineCommand(engineCommand) } + // Wrap engine command in shell (command already includes any internal setup like npm PATH) + shellWrappedCommand := WrapCommandInShell(engineCommand) + + // Pre-create the agent stdio log file with restrictive permissions (0600) before + // starting the AWF container. tee would otherwise create it with the default + // umask (0644), leaving secrets (e.g. MCP gateway tokens) world-readable on the + // runner host until the secret-redaction step runs. + preCreateLog := fmt.Sprintf("(umask 177 && touch %s)", shellEscapeArg(config.LogFile)) + + // Capture the epoch-millisecond timestamp at the very start of the Execute Agent CLI + // step on the host, before the AWF container launches. sendJobConclusionSpan reads + // this file to set the dedicated gh-aw..agent span start time, which excludes + // pre-agent overhead such as workspace audit and CLI proxy startup. + writeAgentCLIStartMs := "printf '%s' \"$(date +%s%3N)\" > " + shellEscapeArg(AgentCLIStartMsPath) + awfHelpersLog.Print("Successfully built AWF command") - return command + return buildAWFInvocationCommand(config, awfInvocationParts{ + awfCommand: awfCommand, + awfArgs: awfArgs, + expandableArgs: expandableArgs, + arcDindPrefixProbe: arcDindPrefixProbe, + arcDindDockerHostProbe: arcDindDockerHostProbe, + arcDindDockerHostRef: arcDindDockerHostRef, + toolCacheMountProbe: toolCacheMountProbe, + toolCacheMountRef: toolCacheMountRef, + configFileSetup: configFileSetup, + modelsJSONPathExport: modelsJSONPathExport, + shellWrappedCommand: shellWrappedCommand, + preCreateLog: preCreateLog, + writeAgentCLIStartMs: writeAgentCLIStartMs, + }) } // BuildAWFArgs constructs common AWF arguments from configuration. @@ -611,36 +548,39 @@ func BuildAWFArgs(config AWFCommandConfig) []string { firewallConfig := getFirewallConfig(config.WorkflowData) agentConfig := getAgentConfig(config.WorkflowData) - var awfArgs []string + awfArgs = appendAWFRuntimeArgs(awfArgs, config, firewallConfig) + awfArgs = appendAWFEnvArgs(awfArgs, config, firewallConfig) + awfArgs = appendAWFMountArgs(awfArgs, agentConfig) + awfArgs = appendAWFLogArgs(awfArgs, config, firewallConfig) + awfArgs = appendAWFLegacySecurityArgs(awfArgs, config, firewallConfig, agentConfig) + awfArgs = append(awfArgs, "--skip-pull") + awfHelpersLog.Print("Using --skip-pull since images are pre-downloaded") + awfArgs = appendAWFCLIProxyArgs(awfArgs, config, firewallConfig) + awfArgs = appendAWFBasePathArgs(awfArgs, config.WorkflowData) + awfArgs = append(awfArgs, getSSLBumpArgs(firewallConfig)...) + awfArgs = appendAWFCustomArgs(awfArgs, firewallConfig, agentConfig) + + awfHelpersLog.Printf("Built %d AWF arguments", len(awfArgs)) + return awfArgs +} - // Add TTY flag if needed (Claude requires this), except for docker-sbx where - // sbx exec --tty can terminate long-running Claude sessions prematurely. +func appendAWFRuntimeArgs(awfArgs []string, config AWFCommandConfig, firewallConfig *FirewallConfig) []string { if config.UsesTTY && !isDockerSbxRuntime(config.WorkflowData) { awfArgs = append(awfArgs, "--tty") } - - // docker-sbx: tell AWF to launch the agent inside a Docker sbx microVM instead - // of as a standard Docker Compose service. Guard on the effective AWF version so - // older binaries do not receive an unknown flag. if isDockerSbxRuntime(config.WorkflowData) && awfSupportsContainerRuntime(firewallConfig) { awfArgs = append(awfArgs, "--container-runtime", "sbx") awfHelpersLog.Print("Added --container-runtime sbx for docker-sbx microVM runtime") } else if isDockerSbxRuntime(config.WorkflowData) { awfHelpersLog.Printf("Skipping --container-runtime sbx: AWF version %q is older than required minimum %s", getAWFImageTag(firewallConfig), constants.AWFContainerRuntimeMinVersion) } + return awfArgs +} - // Pass all environment variables to the container, but exclude every variable whose - // step-env value comes from a GitHub Actions secret. AWF's API proxy (--enable-api-proxy) - // handles authentication for these tokens transparently, so the container does not need - // the raw values. Excluding them via --exclude-env prevents a prompt-injected agent from - // exfiltrating tokens through bash tools such as `env` or `printenv`. - // The caller computes ExcludeEnvVarNames from ComputeAWFExcludeEnvVarNames() so that every - // secret-bearing variable is covered — not just a hardcoded subset. - // --exclude-env requires AWF v0.25.3+; skip the flags for workflows that pin an older version. +func appendAWFEnvArgs(awfArgs []string, config AWFCommandConfig, firewallConfig *FirewallConfig) []string { awfArgs = append(awfArgs, "--env-all") if awfSupportsExcludeEnv(firewallConfig) { - // Sort for deterministic output in compiled lock files. sortedExclude := make([]string, len(config.ExcludeEnvVarNames)) copy(sortedExclude, config.ExcludeEnvVarNames) sort.Strings(sortedExclude) @@ -650,26 +590,23 @@ func BuildAWFArgs(config AWFCommandConfig) []string { } else { awfHelpersLog.Printf("Skipping --exclude-env: AWF version %q is older than minimum %s", getAWFImageTag(firewallConfig), constants.AWFExcludeEnvMinVersion) } + return awfArgs +} - // Note: --container-workdir "${GITHUB_WORKSPACE}" and --mount "${RUNNER_TEMP}/gh-aw:..." - // are intentionally NOT added here. They contain shell variable references that require - // double-quote expansion. These args are appended raw in BuildAWFCommand to ensure - // ${GITHUB_WORKSPACE} and ${RUNNER_TEMP} are expanded by the runner's shell. - - // Add custom mounts from agent config if specified +func appendAWFMountArgs(awfArgs []string, agentConfig *AgentSandboxConfig) []string { if agentConfig != nil && len(agentConfig.Mounts) > 0 { - // Sort mounts for consistent output sortedMounts := make([]string, len(agentConfig.Mounts)) copy(sortedMounts, agentConfig.Mounts) sort.Strings(sortedMounts) - for _, mount := range sortedMounts { awfArgs = append(awfArgs, "--mount", mount) } awfHelpersLog.Printf("Added %d custom mounts from agent config", len(sortedMounts)) } + return awfArgs +} - // Set log level +func appendAWFLogArgs(awfArgs []string, config AWFCommandConfig, firewallConfig *FirewallConfig) []string { awfLogLevel := string(constants.AWFDefaultLogLevel) if firewallConfig != nil && firewallConfig.LogLevel != "" { awfLogLevel = firewallConfig.LogLevel @@ -679,8 +616,10 @@ func BuildAWFArgs(config AWFCommandConfig) []string { awfArgs = append(awfArgs, "--diagnostic-logs") awfHelpersLog.Print("Added --diagnostic-logs because awf-diagnostic-logs feature flag is enabled") } + return awfArgs +} - // Legacy security mode: emit --legacy-security, --enable-host-access, and --allow-host-ports +func appendAWFLegacySecurityArgs(awfArgs []string, config AWFCommandConfig, firewallConfig *FirewallConfig, agentConfig *AgentSandboxConfig) []string { isLegacy := agentConfig != nil && agentConfig.LegacySecurity if isLegacy { if awfSupportsLegacySecurity(firewallConfig) { @@ -708,14 +647,10 @@ func BuildAWFArgs(config AWFCommandConfig) []string { } else { awfHelpersLog.Print("Strict security: skipping host-access flags (default)") } + return awfArgs +} - // Skip pulling images since they are pre-downloaded - awfArgs = append(awfArgs, "--skip-pull") - awfHelpersLog.Print("Using --skip-pull since images are pre-downloaded") - - // Enable CLI proxy sidecar when GitHub mode is gh-proxy. - // Start the difc-proxy on the host and tell AWF where to connect - // (firewall v0.25.17+). +func appendAWFCLIProxyArgs(awfArgs []string, config AWFCommandConfig, firewallConfig *FirewallConfig) []string { if isGitHubCLIModeEnabled(config.WorkflowData) { if awfSupportsCliProxy(firewallConfig) { difcProxyHost := "host.docker.internal:18443" @@ -729,51 +664,42 @@ func BuildAWFArgs(config AWFCommandConfig) []string { awfHelpersLog.Printf("Skipping CLI proxy flags: AWF version %q is older than minimum %s", getAWFImageTag(firewallConfig), constants.AWFCliProxyMinVersion) } } + return awfArgs +} - // Pass base path if URL contains a path component - // This is required for endpoints with path prefixes (e.g., Databricks /serving-endpoints, - // Azure OpenAI /openai/deployments/, corporate LLM routers with path-based routing) - // Base paths remain as CLI flags — they are not yet represented in the config file schema. - openaiBasePath := extractAPIBasePath(config.WorkflowData, "OPENAI_BASE_URL") +func appendAWFBasePathArgs(awfArgs []string, workflowData *WorkflowData) []string { + openaiBasePath := extractAPIBasePath(workflowData, "OPENAI_BASE_URL") if openaiBasePath != "" { awfArgs = append(awfArgs, "--openai-api-base-path", openaiBasePath) awfHelpersLog.Printf("Added --openai-api-base-path=%s", openaiBasePath) } - anthropicBasePath := extractAPIBasePath(config.WorkflowData, "ANTHROPIC_BASE_URL") + anthropicBasePath := extractAPIBasePath(workflowData, "ANTHROPIC_BASE_URL") if anthropicBasePath != "" { awfArgs = append(awfArgs, "--anthropic-api-base-path", anthropicBasePath) awfHelpersLog.Printf("Added --anthropic-api-base-path=%s", anthropicBasePath) } - geminiBasePath := extractAPIBasePath(config.WorkflowData, "GEMINI_API_BASE_URL") + geminiBasePath := extractAPIBasePath(workflowData, "GEMINI_API_BASE_URL") if geminiBasePath != "" { awfArgs = append(awfArgs, "--gemini-api-base-path", geminiBasePath) awfHelpersLog.Printf("Added --gemini-api-base-path=%s", geminiBasePath) } + return awfArgs +} - // Add SSL Bump support for HTTPS content inspection (v0.9.0+) - sslBumpArgs := getSSLBumpArgs(firewallConfig) - awfArgs = append(awfArgs, sslBumpArgs...) - - // Add custom args if specified in firewall config +func appendAWFCustomArgs(awfArgs []string, firewallConfig *FirewallConfig, agentConfig *AgentSandboxConfig) []string { if firewallConfig != nil && len(firewallConfig.Args) > 0 { awfArgs = append(awfArgs, firewallConfig.Args...) } - - // Add custom args from agent config if specified if agentConfig != nil && len(agentConfig.Args) > 0 { awfArgs = append(awfArgs, agentConfig.Args...) awfHelpersLog.Printf("Added %d custom args from agent config", len(agentConfig.Args)) } - - // Pass memory limit to AWF container if specified in agent config if agentConfig != nil && agentConfig.Memory != "" { awfArgs = append(awfArgs, "--memory-limit", agentConfig.Memory) awfHelpersLog.Printf("Set AWF memory limit to %s", agentConfig.Memory) } - - awfHelpersLog.Printf("Built %d AWF arguments", len(awfArgs)) return awfArgs } @@ -910,7 +836,6 @@ func ComputeAWFExcludeEnvVarNames(workflowData *WorkflowData, coreSecretVarNames seen := make(map[string]struct { }) var names []string - addUnique := func(name string) { if !setutil.Contains(seen, name) { seen[name] = struct { @@ -918,30 +843,32 @@ func ComputeAWFExcludeEnvVarNames(workflowData *WorkflowData, coreSecretVarNames names = append(names, name) } } - - // Core secret vars for this engine (always contain secret references). for _, name := range coreSecretVarNames { addUnique(name) } - - // MCP gateway API key is always a secret when MCP servers are present. if HasMCPServers(workflowData) { addUnique("MCP_GATEWAY_API_KEY") } - - // GitHub MCP server token is always a secret when the GitHub tool is present. if hasGitHubTool(workflowData.ParsedTools) { addUnique("GITHUB_MCP_SERVER_TOKEN") } - - // HTTP MCP header secrets: values are always ${{ secrets.* }} references. for varName := range collectHTTPMCPHeaderSecrets(workflowData.Tools) { addUnique(varName) } + addAWFMCPScriptSecretEnvVars(workflowData, addUnique) + addAWFConfigSecretEnvVars(workflowData, addUnique) + if isGitHubCLIModeEnabled(workflowData) { + addUnique("GH_TOKEN") + } + for _, name := range workflowData.ExcludedEnv { + addUnique(name) + } + + awfHelpersLog.Printf("Computed %d AWF env vars to exclude", len(names)) + return names +} - // mcp-scripts env vars: only add those whose configured values contain a secret reference - // or a job-output expression (e.g. ${{ needs.fetch_token.outputs.token }}). - // (Non-secret vars like GH_DEBUG: "1" must NOT be excluded.) +func addAWFMCPScriptSecretEnvVars(workflowData *WorkflowData, addUnique func(string)) { if workflowData.MCPScripts != nil { for _, toolConfig := range workflowData.MCPScripts.Tools { for envName, envValue := range toolConfig.Env { @@ -951,8 +878,9 @@ func ComputeAWFExcludeEnvVarNames(workflowData *WorkflowData, coreSecretVarNames } } } +} - // engine.env vars that contain a secret reference or a job-output expression. +func addAWFConfigSecretEnvVars(workflowData *WorkflowData, addUnique func(string)) { if workflowData.EngineConfig != nil { for varName, varValue := range workflowData.EngineConfig.Env { if strings.Contains(varValue, "${{ secrets.") || ContainsJobOutputExpr(varValue) { @@ -960,8 +888,6 @@ func ComputeAWFExcludeEnvVarNames(workflowData *WorkflowData, coreSecretVarNames } } } - - // agent.env vars that contain a secret reference or a job-output expression. agentConfig := getAgentConfig(workflowData) if agentConfig != nil { for varName, varValue := range agentConfig.Env { @@ -970,21 +896,6 @@ func ComputeAWFExcludeEnvVarNames(workflowData *WorkflowData, coreSecretVarNames } } } - - // GH_TOKEN when GitHub mode is gh-proxy: the token is passed in the AWF step env for the - // host difc-proxy but must be excluded from the agent container. - if isGitHubCLIModeEnabled(workflowData) { - addUnique("GH_TOKEN") - } - - // Explicitly excluded env vars from the frontmatter excluded-env field. - // These are always excluded regardless of their value content. - for _, name := range workflowData.ExcludedEnv { - addUnique(name) - } - - awfHelpersLog.Printf("Computed %d AWF env vars to exclude", len(names)) - return names } // addCliProxyGHTokenToEnv adds GH_TOKEN to the AWF step environment when GitHub diff --git a/pkg/workflow/behavior_defined_engine.go b/pkg/workflow/behavior_defined_engine.go index 5dd16e5fef9..e53609717eb 100644 --- a/pkg/workflow/behavior_defined_engine.go +++ b/pkg/workflow/behavior_defined_engine.go @@ -272,101 +272,114 @@ func (e *BehaviorDefinedEngine) GetExecutionSteps(workflowData *WorkflowData, lo } exec := behavior.Execution + engineCommand := e.buildExecutionEngineCommand(exec, behavior, workflowData) + firewallEnabled := e.executionFirewallEnabled(behavior, workflowData) + command := e.buildExecutionRunCommand(exec, workflowData, logFile, engineCommand, firewallEnabled) + env := e.buildExecutionEnv(exec, behavior, workflowData, firewallEnabled) + + stepLines := buildExecutionStepLines(exec, workflowData) + filteredEnv := FilterEnvForSecrets(env, e.GetRequiredSecretNames(workflowData)) + addCliProxyGHTokenToEnv(filteredEnv, workflowData) + stepLines = FormatStepWithCommandAndEnv(stepLines, command, filteredEnv) + steps = append(steps, GitHubActionStep(stepLines)) + return steps +} + +func (e *BehaviorDefinedEngine) buildExecutionEngineCommand(exec *EngineExecutionDefinition, behavior *EngineBehaviorDefinition, workflowData *WorkflowData) string { commandName := exec.CommandName if workflowData != nil && workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { commandName = workflowData.EngineConfig.Command } - - var engineCommand string if behavior.HarnessScript != "" { - // Harness execution: the harness is responsible for reading GH_AW_PROMPT and - // spawning the engine CLI. Pass the shell-escaped command name and configured args - // so the harness can forward them or use them to build the full command. harnessArgs := []string{shellEscapeArg(commandName)} if len(exec.Args) > 0 { harnessArgs = append(harnessArgs, shellJoinArgs(exec.Args)) } harnessPath := SetupActionDestinationShell + "/" + e.harnessScriptFilename() - engineCommand = fmt.Sprintf("%s %s %s", nodeRuntimeResolutionCommand, harnessPath, strings.Join(harnessArgs, " ")) - } else { - commandParts := []string{commandName} - if len(exec.Args) > 0 { - commandParts = append(commandParts, shellJoinArgs(exec.Args)) - } - if modelFragment := e.modelFlagFragment(exec, workflowData); modelFragment != "" { - commandParts = append(commandParts, modelFragment) - } - if mcpFragment := e.mcpFlagFragment(exec, workflowData); mcpFragment != "" { - commandParts = append(commandParts, mcpFragment) - } - commandParts = append(commandParts, fmt.Sprintf(`"$(cat %s)"`, constants.AwPromptsFile)) - engineCommand = strings.Join(commandParts, " ") - engineCommand = getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + engineCommand + return fmt.Sprintf("%s %s %s", nodeRuntimeResolutionCommand, harnessPath, strings.Join(harnessArgs, " ")) + } + commandParts := []string{commandName} + if len(exec.Args) > 0 { + commandParts = append(commandParts, shellJoinArgs(exec.Args)) } + if modelFragment := e.modelFlagFragment(exec, workflowData); modelFragment != "" { + commandParts = append(commandParts, modelFragment) + } + if mcpFragment := e.mcpFlagFragment(exec, workflowData); mcpFragment != "" { + commandParts = append(commandParts, mcpFragment) + } + commandParts = append(commandParts, fmt.Sprintf(`"$(cat %s)"`, constants.AwPromptsFile)) + return getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + strings.Join(commandParts, " ") +} +func (e *BehaviorDefinedEngine) executionFirewallEnabled(behavior *EngineBehaviorDefinition, workflowData *WorkflowData) bool { firewallEnabled := isFirewallEnabled(workflowData) - // harness-script requires the AWF API proxy sidecar (/reflect) to dynamically - // configure the agentic engine at runtime. Force AWF execution when a harness is - // present unless the sandbox has been explicitly disabled via sandbox.agent: false, - // which also prevents the /reflect endpoint from being available. if behavior.HarnessScript != "" && !isFirewallDisabledBySandboxAgent(workflowData) { firewallEnabled = true } - var command string + return firewallEnabled +} + +func (e *BehaviorDefinedEngine) buildExecutionRunCommand(exec *EngineExecutionDefinition, workflowData *WorkflowData, logFile, engineCommand string, firewallEnabled bool) string { if firewallEnabled { - command = e.buildFirewallCommand(exec, workflowData, logFile, engineCommand) - } else if exec.WriteTimestamp { - command = fmt.Sprintf("set -o pipefail\nexport no_proxy=\"${NO_PROXY:-}\"\nprintf '%%s' \"$(date +%%s%%3N)\" > %s\n%s 2>&1 | tee -a %s", - AgentCLIStartMsPath, engineCommand, logFile) - } else { - command = fmt.Sprintf("set -o pipefail\nexport no_proxy=\"${NO_PROXY:-}\"\n%s 2>&1 | tee -a %s", engineCommand, logFile) + return e.buildFirewallCommand(exec, workflowData, logFile, engineCommand) } - - env := map[string]string{ - "GH_AW_PROMPT": constants.AwPromptsFile, - "GITHUB_WORKSPACE": "${{ github.workspace }}", - "RUNNER_TEMP": "${{ runner.temp }}", - // Set NO_PROXY so that the AWF agent's HTTP client skips the squid proxy - // for local endpoints. The lowercase no_proxy variant is exported inside - // the run script rather than as a YAML env key because GitHub's workflow - // parser rejects case-insensitive duplicate env keys (NO_PROXY/no_proxy), - // which causes workflow_dispatch to fail with "failed to parse workflow". - "NO_PROXY": constants.AWFNoProxyHosts, + if exec.WriteTimestamp { + return fmt.Sprintf("set -o pipefail\nexport no_proxy=\"${NO_PROXY:-}\"\nprintf '%%s' \"$(date +%%s%%3N)\" > %s\n%s 2>&1 | tee -a %s", AgentCLIStartMsPath, engineCommand, logFile) } - injectWorkflowCallNetworkAllowedEnv(env, workflowData) + return fmt.Sprintf("set -o pipefail\nexport no_proxy=\"${NO_PROXY:-}\"\n%s 2>&1 | tee -a %s", engineCommand, logFile) +} - // Apply static env vars declared in the engine definition first so that - // the dynamic AWF vars below can still override them if needed. +func (e *BehaviorDefinedEngine) buildExecutionEnv(exec *EngineExecutionDefinition, behavior *EngineBehaviorDefinition, workflowData *WorkflowData, firewallEnabled bool) map[string]string { + env := baseExecutionEnv() + injectWorkflowCallNetworkAllowedEnv(env, workflowData) maps.Copy(env, exec.Env) - if exec.ProviderEnvMode == behaviorProviderEnvModeUniversalLLMConsumer { e.ApplyUniversalProviderEnv(env, workflowData, firewallEnabled) } + e.addExecutionMCPEnv(env, exec, behavior, workflowData) + e.addExecutionAuthEnv(env) + if behavior.HarnessScript != "" && firewallEnabled { + env["AWF_REFLECT_ENABLED"] = "1" + } + applyExecutionCommonEnv(env, workflowData) + e.addExecutionModelEnv(env, exec, workflowData) + return env +} - if exec.MCPConfigEnvVar != "" && HasMCPServers(workflowData) { - if behavior.ConfigFile != nil { - env[exec.MCPConfigEnvVar] = "${{ github.workspace }}/" + behavior.ConfigFile.Path - } else { - mcpPath := constants.McpServersJsonPathExpr - if behavior.MCP != nil && behavior.MCP.ConfigPath != "" { - mcpPath = behavior.MCP.ConfigPath - } - env[exec.MCPConfigEnvVar] = mcpPath - } +func baseExecutionEnv() map[string]string { + return map[string]string{ + "GH_AW_PROMPT": constants.AwPromptsFile, + "GITHUB_WORKSPACE": "${{ github.workspace }}", + "RUNNER_TEMP": "${{ runner.temp }}", + "NO_PROXY": constants.AWFNoProxyHosts, + } +} + +func (e *BehaviorDefinedEngine) addExecutionMCPEnv(env map[string]string, exec *EngineExecutionDefinition, behavior *EngineBehaviorDefinition, workflowData *WorkflowData) { + if exec.MCPConfigEnvVar == "" || !HasMCPServers(workflowData) { + return } + if behavior.ConfigFile != nil { + env[exec.MCPConfigEnvVar] = "${{ github.workspace }}/" + behavior.ConfigFile.Path + return + } + mcpPath := constants.McpServersJsonPathExpr + if behavior.MCP != nil && behavior.MCP.ConfigPath != "" { + mcpPath = behavior.MCP.ConfigPath + } + env[exec.MCPConfigEnvVar] = mcpPath +} +func (e *BehaviorDefinedEngine) addExecutionAuthEnv(env map[string]string) { for _, binding := range e.definition.Auth { if binding.Secret != "" { env[binding.Secret] = "${{ secrets." + binding.Secret + " }}" } } +} - // When a harness script is present and the AWF firewall is running, signal to the - // harness that the AWF API proxy sidecar is available so it can read /reflect data. - if behavior.HarnessScript != "" && firewallEnabled { - env["AWF_REFLECT_ENABLED"] = "1" - } - +func applyExecutionCommonEnv(env map[string]string, workflowData *WorkflowData) { applySafeOutputEnvToMap(env, workflowData) applyTraceContextEnvToMap(env) applyOptionalEngineToolTimeouts(env, workflowData) @@ -374,34 +387,31 @@ func (e *BehaviorDefinedEngine) GetExecutionSteps(workflowData *WorkflowData, lo applyEngineCwdEnv(env, workflowData) applyEngineAndAgentEnv(env, workflowData, behaviorDefinedEngineLog) applyMCPScriptsSecretEnv(env, workflowData) +} - if exec.ModelEnvVarName != "" { - if workflowData != nil && workflowData.EngineConfig != nil && workflowData.EngineConfig.Model != "" { - modelVal := workflowData.EngineConfig.Model - if exec.ModelEnvProviderPrefix != "" { - if parts := strings.SplitN(modelVal, "/", 2); len(parts) == 2 { - modelVal = exec.ModelEnvProviderPrefix + "/" + parts[1] - } - } - env[exec.ModelEnvVarName] = modelVal +func (e *BehaviorDefinedEngine) addExecutionModelEnv(env map[string]string, exec *EngineExecutionDefinition, workflowData *WorkflowData) { + if exec.ModelEnvVarName == "" || workflowData == nil || workflowData.EngineConfig == nil || workflowData.EngineConfig.Model == "" { + return + } + modelVal := workflowData.EngineConfig.Model + if exec.ModelEnvProviderPrefix != "" { + if parts := strings.SplitN(modelVal, "/", 2); len(parts) == 2 { + modelVal = exec.ModelEnvProviderPrefix + "/" + parts[1] } } + env[exec.ModelEnvVarName] = modelVal +} +func buildExecutionStepLines(exec *EngineExecutionDefinition, workflowData *WorkflowData) []string { stepLines := []string{ " - name: " + exec.StepName, " id: agentic_execution", } if workflowData != nil && workflowData.TimeoutMinutes != "" { timeoutValue := strings.TrimPrefix(workflowData.TimeoutMinutes, "timeout-minutes: ") - stepLines = append(stepLines, " timeout-minutes: "+timeoutValue) - } else { - stepLines = append(stepLines, fmt.Sprintf(" timeout-minutes: %d", int(constants.DefaultAgenticWorkflowTimeout/time.Minute))) + return append(stepLines, " timeout-minutes: "+timeoutValue) } - filteredEnv := FilterEnvForSecrets(env, e.GetRequiredSecretNames(workflowData)) - addCliProxyGHTokenToEnv(filteredEnv, workflowData) - stepLines = FormatStepWithCommandAndEnv(stepLines, command, filteredEnv) - steps = append(steps, GitHubActionStep(stepLines)) - return steps + return append(stepLines, fmt.Sprintf(" timeout-minutes: %d", int(constants.DefaultAgenticWorkflowTimeout/time.Minute))) } func (e *BehaviorDefinedEngine) modelFlagFragment(exec *EngineExecutionDefinition, workflowData *WorkflowData) string { diff --git a/pkg/workflow/cache.go b/pkg/workflow/cache.go index 18689f0ec67..0402bad54d7 100644 --- a/pkg/workflow/cache.go +++ b/pkg/workflow/cache.go @@ -516,91 +516,72 @@ func generateCacheMemorySteps(builder *strings.Builder, data *WorkflowData) { if data.CacheMemoryConfig == nil || len(data.CacheMemoryConfig.Caches) == 0 { return } - cacheLog.Printf("Generating cache-memory setup steps for %d caches", len(data.CacheMemoryConfig.Caches)) - builder.WriteString(" # Cache memory file share configuration from frontmatter processed below\n") - - // Use backward-compatible paths only when there's a single cache with ID "default" - // This maintains compatibility with existing workflows useBackwardCompatiblePaths := len(data.CacheMemoryConfig.Caches) == 1 && data.CacheMemoryConfig.Caches[0].ID == "default" - - // Extract GitHub guard policy for integrity-aware cache key generation. - var githubConfig *GitHubToolConfig - if data.ParsedTools != nil { - githubConfig = data.ParsedTools.GitHub - } + githubConfig := cacheMemoryGitHubConfig(data) integrityLevel := cacheIntegrityLevel(githubConfig) for i, cache := range data.CacheMemoryConfig.Caches { - cacheDir := cacheMemoryDirFor(cache.ID) - restoreStepID := fmt.Sprintf("restore_cache_memory_%d", i) - - // Add step to create cache-memory directory for this cache - if useBackwardCompatiblePaths { - // For single default cache, use the original directory for backward compatibility - builder.WriteString(" - name: Create cache-memory directory\n") - builder.WriteString(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh\"\n") - } else { - fmt.Fprintf(builder, " - name: Create cache-memory directory (%s)\n", cache.ID) - builder.WriteString(" run: |\n") - fmt.Fprintf(builder, " mkdir -p %s\n", cacheDir) - } - - // Use integrity-aware cache key (includes integrity level + policy hash prefix). - cacheKey := computeIntegrityCacheKey(cache, githubConfig) - - // Ensure run_id suffix is present (computeIntegrityCacheKey guarantees this, - // but we check again for clarity and safety). - runIdSuffix := "-${{ github.run_id }}" - if !strings.HasSuffix(cacheKey, runIdSuffix) { - cacheKey = cacheKey + runIdSuffix - } - - // Generate restore keys based on scope - // - "workflow" (default): Single restore key with workflow ID (secure) - // - "repo": Two restore keys - with and without workflow ID (allows cross-workflow sharing) - restoreKeys := buildCacheRestoreKeys(cacheKey, cache.Scope) - - // Step name and action - // Use actions/cache/restore for restore-only caches or when threat detection is enabled - // When threat detection is enabled, we only restore the cache and defer saving to a separate job after detection - // Use actions/cache for normal caches (which auto-saves via post-action) - threatDetectionEnabled := IsDetectionJobEnabled(data.SafeOutputs) - useRestoreOnly := cache.RestoreOnly || threatDetectionEnabled - - actionName := "Restore cache-memory file share data" - - if useBackwardCompatiblePaths { - fmt.Fprintf(builder, " - name: %s\n", actionName) - } else { - fmt.Fprintf(builder, " - name: %s (%s)\n", actionName, cache.ID) - } - fmt.Fprintf(builder, " id: %s\n", restoreStepID) + generateCacheMemorySetupForCache(builder, data, cache, i, githubConfig, integrityLevel, useBackwardCompatiblePaths) + } +} - // Use actions/cache/restore@v4 when restore-only or threat detection enabled - // Use actions/cache@v4 for normal caches - if useRestoreOnly { - fmt.Fprintf(builder, " uses: %s\n", getActionPin("actions/cache/restore")) - } else { - fmt.Fprintf(builder, " uses: %s\n", getActionPin("actions/cache")) - } - builder.WriteString(" with:\n") - fmt.Fprintf(builder, " key: %s\n", cacheKey) +func cacheMemoryGitHubConfig(data *WorkflowData) *GitHubToolConfig { + if data.ParsedTools != nil { + return data.ParsedTools.GitHub + } + return nil +} - // Path - always use the new cache directory format - fmt.Fprintf(builder, " path: %s\n", cacheDir) +func generateCacheMemorySetupForCache(builder *strings.Builder, data *WorkflowData, cache CacheMemoryEntry, index int, githubConfig *GitHubToolConfig, integrityLevel string, useBackwardCompatiblePaths bool) { + cacheDir := cacheMemoryDirFor(cache.ID) + generateCacheMemoryDirectoryStep(builder, cache, cacheDir, useBackwardCompatiblePaths) + cacheKey := ensureCacheKeyRunID(computeIntegrityCacheKey(cache, githubConfig)) + restoreKeys := buildCacheRestoreKeys(cacheKey, cache.Scope) + restoreStepID := fmt.Sprintf("restore_cache_memory_%d", index) + generateCacheMemoryRestoreStep(builder, data, cache, cacheDir, restoreStepID, cacheKey, restoreKeys, useBackwardCompatiblePaths) + generateCacheMemoryGitSetupStep(builder, cache, cacheDir, integrityLevel, useBackwardCompatiblePaths) +} - builder.WriteString(" restore-keys: |\n") - for _, key := range restoreKeys { - fmt.Fprintf(builder, " %s\n", key) - } +func generateCacheMemoryDirectoryStep(builder *strings.Builder, cache CacheMemoryEntry, cacheDir string, useBackwardCompatiblePaths bool) { + if useBackwardCompatiblePaths { + builder.WriteString(" - name: Create cache-memory directory\n") + builder.WriteString(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/create_cache_memory_dir.sh\"\n") + return + } + fmt.Fprintf(builder, " - name: Create cache-memory directory (%s)\n", cache.ID) + builder.WriteString(" run: |\n") + fmt.Fprintf(builder, " mkdir -p %s\n", cacheDir) +} - // Add git setup step after cache restore. - // This initialises (or migrates) the git repository used for integrity branching, - // checks out the current integrity branch, and merges down from higher-integrity branches. - generateCacheMemoryGitSetupStep(builder, cache, cacheDir, integrityLevel, useBackwardCompatiblePaths) +func generateCacheMemoryRestoreStep(builder *strings.Builder, data *WorkflowData, cache CacheMemoryEntry, cacheDir, restoreStepID, cacheKey string, restoreKeys []string, useBackwardCompatiblePaths bool) { + useRestoreOnly := cache.RestoreOnly || IsDetectionJobEnabled(data.SafeOutputs) + if useBackwardCompatiblePaths { + builder.WriteString(" - name: Restore cache-memory file share data\n") + } else { + fmt.Fprintf(builder, " - name: Restore cache-memory file share data (%s)\n", cache.ID) + } + fmt.Fprintf(builder, " id: %s\n", restoreStepID) + if useRestoreOnly { + fmt.Fprintf(builder, " uses: %s\n", getActionPin("actions/cache/restore")) + } else { + fmt.Fprintf(builder, " uses: %s\n", getActionPin("actions/cache")) + } + builder.WriteString(" with:\n") + fmt.Fprintf(builder, " key: %s\n", cacheKey) + fmt.Fprintf(builder, " path: %s\n", cacheDir) + builder.WriteString(" restore-keys: |\n") + for _, key := range restoreKeys { + fmt.Fprintf(builder, " %s\n", key) } +} +func ensureCacheKeyRunID(cacheKey string) string { + runIdSuffix := "-${{ github.run_id }}" + if !strings.HasSuffix(cacheKey, runIdSuffix) { + return cacheKey + runIdSuffix + } + return cacheKey } // generateCacheMemoryGitSetupStep emits a pre-agent step that sets up the git-backed integrity @@ -718,66 +699,55 @@ func generateCacheMemoryArtifactUpload(builder *strings.Builder, data *WorkflowD if data.CacheMemoryConfig == nil || len(data.CacheMemoryConfig.Caches) == 0 { return } - - // Only upload artifacts when threat detection is enabled (needed for update_cache_memory job) - // When threat detection is disabled, cache is saved automatically by actions/cache post-action - threatDetectionEnabled := IsDetectionJobEnabled(data.SafeOutputs) - if !threatDetectionEnabled { + if !IsDetectionJobEnabled(data.SafeOutputs) { cacheLog.Print("Skipping cache-memory artifact upload (threat detection disabled)") return } - cacheLog.Printf("Generating cache-memory artifact upload steps for %d caches", len(data.CacheMemoryConfig.Caches)) - - // Use backward-compatible paths only when there's a single cache with ID "default" useBackwardCompatiblePaths := len(data.CacheMemoryConfig.Caches) == 1 && data.CacheMemoryConfig.Caches[0].ID == "default" - - // In workflow_call context, apply the per-invocation prefix to avoid artifact name clashes. prefix := artifactPrefixExprForDownstreamJob(data) - for _, cache := range data.CacheMemoryConfig.Caches { - // Skip restore-only caches if cache.RestoreOnly { continue } + generateCacheMemoryArtifactUploadForCache(builder, cache, prefix, useBackwardCompatiblePaths, pinAction) + } +} - cacheDir := cacheMemoryDirFor(cache.ID) - - // Add a best-effort git integrity check and reseed step before upload. - // This prevents upload-artifact from failing on torn/corrupt .git object stores. - if useBackwardCompatiblePaths { - builder.WriteString(" - name: Check cache-memory git integrity\n") - } else { - fmt.Fprintf(builder, " - name: Check cache-memory git integrity (%s)\n", cache.ID) - } - builder.WriteString(" if: always()\n") - builder.WriteString(" continue-on-error: true\n") - builder.WriteString(" env:\n") - fmt.Fprintf(builder, " GH_AW_CACHE_DIR: %s\n", cacheDir) - builder.WriteString(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/check_cache_memory_git_integrity.sh\"\n") +func generateCacheMemoryArtifactUploadForCache(builder *strings.Builder, cache CacheMemoryEntry, prefix string, useBackwardCompatiblePaths bool, pinAction func(string) string) { + cacheDir := cacheMemoryDirFor(cache.ID) + generateCacheMemoryIntegrityCheckStep(builder, cache, cacheDir, useBackwardCompatiblePaths) + if useBackwardCompatiblePaths { + builder.WriteString(" - name: Upload cache-memory data as artifact\n") + } else { + fmt.Fprintf(builder, " - name: Upload cache-memory data as artifact (%s)\n", cache.ID) + } + fmt.Fprintf(builder, " uses: %s\n", pinAction("actions/upload-artifact")) + builder.WriteString(" if: always()\n") + builder.WriteString(" with:\n") + if useBackwardCompatiblePaths { + fmt.Fprintf(builder, " name: %scache-memory\n", prefix) + } else { + fmt.Fprintf(builder, " name: %scache-memory-%s\n", prefix, cache.ID) + } + builder.WriteString(" include-hidden-files: true\n") + fmt.Fprintf(builder, " path: %s\n", cacheDir) + if cache.RetentionDays != nil { + fmt.Fprintf(builder, " retention-days: %d\n", *cache.RetentionDays) + } +} - // Add upload-artifact step for each cache (runs always) - if useBackwardCompatiblePaths { - builder.WriteString(" - name: Upload cache-memory data as artifact\n") - } else { - fmt.Fprintf(builder, " - name: Upload cache-memory data as artifact (%s)\n", cache.ID) - } - fmt.Fprintf(builder, " uses: %s\n", pinAction("actions/upload-artifact")) - builder.WriteString(" if: always()\n") - builder.WriteString(" with:\n") - // Always use the new artifact name and path format, with prefix in workflow_call context - if useBackwardCompatiblePaths { - fmt.Fprintf(builder, " name: %scache-memory\n", prefix) - } else { - fmt.Fprintf(builder, " name: %scache-memory-%s\n", prefix, cache.ID) - } - builder.WriteString(" include-hidden-files: true\n") - fmt.Fprintf(builder, " path: %s\n", cacheDir) - // Add retention-days if configured - if cache.RetentionDays != nil { - fmt.Fprintf(builder, " retention-days: %d\n", *cache.RetentionDays) - } +func generateCacheMemoryIntegrityCheckStep(builder *strings.Builder, cache CacheMemoryEntry, cacheDir string, useBackwardCompatiblePaths bool) { + if useBackwardCompatiblePaths { + builder.WriteString(" - name: Check cache-memory git integrity\n") + } else { + fmt.Fprintf(builder, " - name: Check cache-memory git integrity (%s)\n", cache.ID) } + builder.WriteString(" if: always()\n") + builder.WriteString(" continue-on-error: true\n") + builder.WriteString(" env:\n") + fmt.Fprintf(builder, " GH_AW_CACHE_DIR: %s\n", cacheDir) + builder.WriteString(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/check_cache_memory_git_integrity.sh\"\n") } // buildCacheMemoryPromptSection builds a PromptSection for cache memory instructions @@ -786,48 +756,40 @@ func buildCacheMemoryPromptSection(config *CacheMemoryConfig) *PromptSection { if config == nil || len(config.Caches) == 0 { return nil } - - // Check if there's only one cache with ID "default" to use singular template if len(config.Caches) == 1 && config.Caches[0].ID == "default" { - cache := config.Caches[0] - // Trailing slash makes the path look like a directory in prompt context. - cacheDir := cacheMemoryDirFor(cache.ID) + "/" - - // Build description text - descriptionText := "" - if cache.Description != "" { - descriptionText = cache.Description - } - - // Build allowed extensions text. - // When non-empty, add a compact plain-text restriction line. - // When empty (all extensions allowed), the placeholder is replaced with nothing. - var allowedExtsText string - if len(cache.AllowedExtensions) > 0 { - allowedExtsText = "\nAllowed file extensions: " + strings.Join(cache.AllowedExtensions, ", ") + "." - } - - cacheLog.Printf("Building cache memory prompt section with env vars: cache_dir=%s, description=%s, allowed_extensions=%v", cacheDir, descriptionText, cache.AllowedExtensions) - - // Return prompt section with template file and environment variables for substitution - return &PromptSection{ - Content: cacheMemoryPromptFile, - IsFile: true, - EnvVars: map[string]string{ - "GH_AW_CACHE_DIR": cacheDir, - "GH_AW_CACHE_DESCRIPTION": descriptionText, - "GH_AW_ALLOWED_EXTENSIONS": allowedExtsText, - }, - } + return buildDefaultCacheMemoryPromptSection(config.Caches[0]) } - - // Multiple caches or non-default single cache - use template file with substitutions cacheLog.Print("Building cache memory prompt section for multiple caches using template") + return &PromptSection{ + Content: cacheMemoryPromptMultiFile, + IsFile: true, + EnvVars: map[string]string{ + "GH_AW_CACHE_LIST": buildCacheMemoryPromptCacheList(config.Caches), + "GH_AW_ALLOWED_EXTENSIONS": buildCacheMemoryPromptAllowedExtensions(config.Caches), + "GH_AW_CACHE_EXAMPLES": buildCacheMemoryPromptExamples(config.Caches), + }, + } +} - // Build cache list +func buildDefaultCacheMemoryPromptSection(cache CacheMemoryEntry) *PromptSection { + cacheDir := cacheMemoryDirFor(cache.ID) + "/" + descriptionText := cache.Description + allowedExtsText := formatCacheMemoryAllowedExtensions(cache.AllowedExtensions) + cacheLog.Printf("Building cache memory prompt section with env vars: cache_dir=%s, description=%s, allowed_extensions=%v", cacheDir, descriptionText, cache.AllowedExtensions) + return &PromptSection{ + Content: cacheMemoryPromptFile, + IsFile: true, + EnvVars: map[string]string{ + "GH_AW_CACHE_DIR": cacheDir, + "GH_AW_CACHE_DESCRIPTION": descriptionText, + "GH_AW_ALLOWED_EXTENSIONS": allowedExtsText, + }, + } +} + +func buildCacheMemoryPromptCacheList(caches []CacheMemoryEntry) string { var cacheList strings.Builder - for _, cache := range config.Caches { - // Trailing slash makes the path look like a directory in prompt context. + for _, cache := range caches { cacheDir := cacheMemoryDirFor(cache.ID) + "/" if cache.Description != "" { fmt.Fprintf(&cacheList, "- **%s**: `%s` - %s\n", cache.ID, cacheDir, cache.Description) @@ -835,19 +797,22 @@ func buildCacheMemoryPromptSection(config *CacheMemoryConfig) *PromptSection { fmt.Fprintf(&cacheList, "- **%s**: `%s`\n", cache.ID, cacheDir) } } + return cacheList.String() +} + +func buildCacheMemoryPromptAllowedExtensions(caches []CacheMemoryEntry) string { + return formatCacheMemoryAllowedExtensions(cacheMemoryAllowedExtensionsUnion(caches)) +} - // Build allowed extensions text. - // Compute the union of all allowed extensions across all caches. - // When non-empty, add a compact plain-text restriction line. - // When empty (all extensions allowed for all caches), the placeholder is replaced with nothing. +func cacheMemoryAllowedExtensionsUnion(caches []CacheMemoryEntry) []string { allSame := true - for i := 1; i < len(config.Caches); i++ { - if len(config.Caches[i].AllowedExtensions) != len(config.Caches[0].AllowedExtensions) { + for i := 1; i < len(caches); i++ { + if len(caches[i].AllowedExtensions) != len(caches[0].AllowedExtensions) { allSame = false break } - for j, ext := range config.Caches[i].AllowedExtensions { - if ext != config.Caches[0].AllowedExtensions[j] { + for j, ext := range caches[i].AllowedExtensions { + if ext != caches[0].AllowedExtensions[j] { allSame = false break } @@ -856,33 +821,35 @@ func buildCacheMemoryPromptSection(config *CacheMemoryConfig) *PromptSection { break } } - - var extsUnion []string if allSame { - extsUnion = config.Caches[0].AllowedExtensions - } else { - extensionSet := make(map[string]struct { - }) - for _, cache := range config.Caches { - for _, ext := range cache.AllowedExtensions { - extensionSet[ext] = struct { - }{} - } - } - for ext := range extensionSet { - extsUnion = append(extsUnion, ext) + return caches[0].AllowedExtensions + } + extensionSet := make(map[string]struct { + }) + for _, cache := range caches { + for _, ext := range cache.AllowedExtensions { + extensionSet[ext] = struct { + }{} } - sort.Strings(extsUnion) } + extsUnion := make([]string, 0, len(extensionSet)) + for ext := range extensionSet { + extsUnion = append(extsUnion, ext) + } + sort.Strings(extsUnion) + return extsUnion +} - var allowedExtsText string - if len(extsUnion) > 0 { - allowedExtsText = "\nAllowed file extensions: " + strings.Join(extsUnion, ", ") + "." +func formatCacheMemoryAllowedExtensions(extensions []string) string { + if len(extensions) == 0 { + return "" } + return "\nAllowed file extensions: " + strings.Join(extensions, ", ") + "." +} - // Build cache examples +func buildCacheMemoryPromptExamples(caches []CacheMemoryEntry) string { var cacheExamples strings.Builder - for _, cache := range config.Caches { + for _, cache := range caches { cacheDir := cacheMemoryDirFor(cache.ID) fmt.Fprintf(&cacheExamples, "- `%s/notes.txt` - general notes and observations\n", cacheDir) fmt.Fprintf(&cacheExamples, "- `%s/notes.md` - markdown formatted notes\n", cacheDir) @@ -891,16 +858,7 @@ func buildCacheMemoryPromptSection(config *CacheMemoryConfig) *PromptSection { fmt.Fprintf(&cacheExamples, "- `%s/data.csv` - tabular data\n", cacheDir) fmt.Fprintf(&cacheExamples, "- `%s/state/` - organized state files in subdirectories (with allowed file types)\n", cacheDir) } - - return &PromptSection{ - Content: cacheMemoryPromptMultiFile, - IsFile: true, - EnvVars: map[string]string{ - "GH_AW_CACHE_LIST": cacheList.String(), - "GH_AW_ALLOWED_EXTENSIONS": allowedExtsText, - "GH_AW_CACHE_EXAMPLES": cacheExamples.String(), - }, - } + return cacheExamples.String() } // buildUpdateCacheMemoryJob builds a job that updates cache-memory after detection passes @@ -919,162 +877,135 @@ func (c *Compiler) buildUpdateCacheMemoryJob(data *WorkflowData, threatDetection cacheLog.Printf("Building update_cache_memory job for %d caches (threatDetectionEnabled=%v)", len(data.CacheMemoryConfig.Caches), threatDetectionEnabled) var steps []string - - // Build steps for each cache - // In workflow_call context, use the per-invocation prefix from the agent job. cacheArtifactPrefix := artifactPrefixExprForAgentDownstreamJob(data) - for _, cache := range data.CacheMemoryConfig.Caches { - // Skip restore-only caches if cache.RestoreOnly { continue } - - // Determine artifact name and cache directory. - // Apply the workflow_call prefix to ensure we download the correct invocation's artifact. - cacheDir := cacheMemoryDirFor(cache.ID) - var artifactName string - if cache.ID == "default" { - artifactName = cacheArtifactPrefix + "cache-memory" - } else { - artifactName = cacheArtifactPrefix + "cache-memory-" + cache.ID - } - - // Download artifact step - var downloadStep strings.Builder - // Generate a safe step ID from cache ID (replace hyphens with underscores) - downloadStepID := strings.ReplaceAll("download_cache_"+cache.ID, "-", "_") - fmt.Fprintf(&downloadStep, " - name: Download cache-memory artifact (%s)\n", cache.ID) - fmt.Fprintf(&downloadStep, " id: %s\n", downloadStepID) - fmt.Fprintf(&downloadStep, " uses: %s\n", c.getActionPin("actions/download-artifact")) - downloadStep.WriteString(" continue-on-error: true\n") - downloadStep.WriteString(" with:\n") - fmt.Fprintf(&downloadStep, " name: %s\n", artifactName) - fmt.Fprintf(&downloadStep, " path: %s\n", cacheDir) - steps = append(steps, downloadStep.String()) - - // Check if cache folder exists and is not empty - var checkStep strings.Builder - checkStepID := strings.ReplaceAll("check_cache_"+cache.ID, "-", "_") - fmt.Fprintf(&checkStep, " - name: Check if cache-memory folder has content (%s)\n", cache.ID) - fmt.Fprintf(&checkStep, " id: %s\n", checkStepID) - checkStep.WriteString(" shell: bash\n") - checkStep.WriteString(" run: |\n") - fmt.Fprintf(&checkStep, " if [ -d \"%s\" ] && [ \"$(ls -A %s 2>/dev/null)\" ]; then\n", cacheDir, cacheDir) - checkStep.WriteString(" echo \"has_content=true\" >> \"$GITHUB_OUTPUT\"\n") - checkStep.WriteString(" else\n") - checkStep.WriteString(" echo \"has_content=false\" >> \"$GITHUB_OUTPUT\"\n") - checkStep.WriteString(" fi\n") - steps = append(steps, checkStep.String()) - - // Skip validation step if allowed extensions is empty (means all files are allowed) - if len(cache.AllowedExtensions) == 0 { - cacheLog.Printf("Skipping validation step for cache %s in update job (empty allowed-extensions means all files are allowed)", cache.ID) - } else { - // Prepare allowed extensions array for JavaScript - allowedExtsJSON, _ := json.Marshal(cache.AllowedExtensions) //nolint:jsonmarshalignoredeerror // marshaling a string slice cannot fail - - // Build validation script - var validationScript strings.Builder - validationScript.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") - validationScript.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") - validationScript.WriteString(" const { validateMemoryFiles } = require('${{ runner.temp }}/gh-aw/actions/validate_memory_files.cjs');\n") - fmt.Fprintf(&validationScript, " const allowedExtensions = %s;\n", allowedExtsJSON) - fmt.Fprintf(&validationScript, " const result = validateMemoryFiles('%s', 'cache', allowedExtensions);\n", cacheDir) - validationScript.WriteString(" if (!result.valid) {\n") - fmt.Fprintf(&validationScript, " core.setFailed(`File type validation failed: Found ${result.invalidFiles.length} file(s) with invalid extensions. Only %s are allowed.`);\n", strings.Join(cache.AllowedExtensions, ", ")) - validationScript.WriteString(" }\n") - - // Generate validation step using helper with condition to only run if cache has content - stepName := fmt.Sprintf("Validate cache-memory file types (%s)", cache.ID) - condition := fmt.Sprintf("steps.%s.outputs.has_content == 'true'", checkStepID) - steps = append(steps, generateInlineGitHubScriptStep(stepName, validationScript.String(), condition, data)) - } - - // Generate cache key using integrity-aware format (matches generateCacheMemorySteps) - var githubConfig *GitHubToolConfig - if data.ParsedTools != nil { - githubConfig = data.ParsedTools.GitHub - } - cacheKey := computeIntegrityCacheKey(cache, githubConfig) - - // Ensure run_id suffix is present - runIdSuffix := "-${{ github.run_id }}" - if !strings.HasSuffix(cacheKey, runIdSuffix) { - cacheKey = cacheKey + runIdSuffix - } - - // Save to cache step - only run if cache has content - var saveStep strings.Builder - fmt.Fprintf(&saveStep, " - name: Save cache-memory to cache (%s)\n", cache.ID) - fmt.Fprintf(&saveStep, " if: steps.%s.outputs.has_content == 'true'\n", checkStepID) - fmt.Fprintf(&saveStep, " uses: %s\n", getActionPin("actions/cache/save")) - saveStep.WriteString(" with:\n") - fmt.Fprintf(&saveStep, " key: %s\n", cacheKey) - fmt.Fprintf(&saveStep, " path: %s\n", cacheDir) - steps = append(steps, saveStep.String()) + steps = c.appendUpdateCacheMemorySteps(steps, data, cache, cacheArtifactPrefix) } - - // If no writable caches, return nil if len(steps) == 0 { return nil, nil } - - // Add setup step to copy scripts at the beginning + setupActionRef, setupSteps := c.buildUpdateCacheMemorySetupSteps(data) + steps = append(setupSteps, steps...) + return &Job{ + Name: updateCacheMemoryJobName, + DisplayName: "", // No display name - job ID is sufficient + RunsOn: c.formatFrameworkJobRunsOn(data), + If: buildUpdateCacheMemoryJobCondition(), + Permissions: c.updateCacheMemoryPermissions(data, setupActionRef), + Needs: []string{string(constants.AgentJobName), string(constants.DetectionJobName), string(constants.ActivationJobName)}, + Env: updateCacheMemoryJobEnv(data), + Steps: steps, + }, nil +} + +func (c *Compiler) appendUpdateCacheMemorySteps(steps []string, data *WorkflowData, cache CacheMemoryEntry, artifactPrefix string) []string { + cacheDir := cacheMemoryDirFor(cache.ID) + checkStepID := strings.ReplaceAll("check_cache_"+cache.ID, "-", "_") + steps = append(steps, c.buildUpdateCacheMemoryDownloadStep(cache, cacheDir, artifactPrefix)) + steps = append(steps, buildUpdateCacheMemoryCheckStep(cache, cacheDir, checkStepID)) + steps = appendUpdateCacheMemoryValidationStep(steps, data, cache, cacheDir, checkStepID) + githubConfig := cacheMemoryGitHubConfig(data) + cacheKey := ensureCacheKeyRunID(computeIntegrityCacheKey(cache, githubConfig)) + return append(steps, buildUpdateCacheMemorySaveStep(cache, cacheDir, checkStepID, cacheKey)) +} + +func (c *Compiler) buildUpdateCacheMemoryDownloadStep(cache CacheMemoryEntry, cacheDir, artifactPrefix string) string { + artifactName := artifactPrefix + "cache-memory" + if cache.ID != "default" { + artifactName = artifactPrefix + "cache-memory-" + cache.ID + } + downloadStepID := strings.ReplaceAll("download_cache_"+cache.ID, "-", "_") + var downloadStep strings.Builder + fmt.Fprintf(&downloadStep, " - name: Download cache-memory artifact (%s)\n", cache.ID) + fmt.Fprintf(&downloadStep, " id: %s\n", downloadStepID) + fmt.Fprintf(&downloadStep, " uses: %s\n", c.getActionPin("actions/download-artifact")) + downloadStep.WriteString(" continue-on-error: true\n") + downloadStep.WriteString(" with:\n") + fmt.Fprintf(&downloadStep, " name: %s\n", artifactName) + fmt.Fprintf(&downloadStep, " path: %s\n", cacheDir) + return downloadStep.String() +} + +func buildUpdateCacheMemoryCheckStep(cache CacheMemoryEntry, cacheDir, checkStepID string) string { + var checkStep strings.Builder + fmt.Fprintf(&checkStep, " - name: Check if cache-memory folder has content (%s)\n", cache.ID) + fmt.Fprintf(&checkStep, " id: %s\n", checkStepID) + checkStep.WriteString(" shell: bash\n") + checkStep.WriteString(" run: |\n") + fmt.Fprintf(&checkStep, " if [ -d \"%s\" ] && [ \"$(ls -A %s 2>/dev/null)\" ]; then\n", cacheDir, cacheDir) + checkStep.WriteString(" echo \"has_content=true\" >> \"$GITHUB_OUTPUT\"\n") + checkStep.WriteString(" else\n") + checkStep.WriteString(" echo \"has_content=false\" >> \"$GITHUB_OUTPUT\"\n") + checkStep.WriteString(" fi\n") + return checkStep.String() +} + +func appendUpdateCacheMemoryValidationStep(steps []string, data *WorkflowData, cache CacheMemoryEntry, cacheDir, checkStepID string) []string { + if len(cache.AllowedExtensions) == 0 { + cacheLog.Printf("Skipping validation step for cache %s in update job (empty allowed-extensions means all files are allowed)", cache.ID) + return steps + } + allowedExtsJSON, _ := json.Marshal(cache.AllowedExtensions) //nolint:jsonmarshalignoredeerror // marshaling a string slice cannot fail + var validationScript strings.Builder + validationScript.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") + validationScript.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") + validationScript.WriteString(" const { validateMemoryFiles } = require('${{ runner.temp }}/gh-aw/actions/validate_memory_files.cjs');\n") + fmt.Fprintf(&validationScript, " const allowedExtensions = %s;\n", allowedExtsJSON) + fmt.Fprintf(&validationScript, " const result = validateMemoryFiles('%s', 'cache', allowedExtensions);\n", cacheDir) + validationScript.WriteString(" if (!result.valid) {\n") + fmt.Fprintf(&validationScript, " core.setFailed(`File type validation failed: Found ${result.invalidFiles.length} file(s) with invalid extensions. Only %s are allowed.`);\n", strings.Join(cache.AllowedExtensions, ", ")) + validationScript.WriteString(" }\n") + stepName := fmt.Sprintf("Validate cache-memory file types (%s)", cache.ID) + condition := fmt.Sprintf("steps.%s.outputs.has_content == 'true'", checkStepID) + return append(steps, generateInlineGitHubScriptStep(stepName, validationScript.String(), condition, data)) +} + +func buildUpdateCacheMemorySaveStep(cache CacheMemoryEntry, cacheDir, checkStepID, cacheKey string) string { + var saveStep strings.Builder + fmt.Fprintf(&saveStep, " - name: Save cache-memory to cache (%s)\n", cache.ID) + fmt.Fprintf(&saveStep, " if: steps.%s.outputs.has_content == 'true'\n", checkStepID) + fmt.Fprintf(&saveStep, " uses: %s\n", getActionPin("actions/cache/save")) + saveStep.WriteString(" with:\n") + fmt.Fprintf(&saveStep, " key: %s\n", cacheKey) + fmt.Fprintf(&saveStep, " path: %s\n", cacheDir) + return saveStep.String() +} + +func (c *Compiler) buildUpdateCacheMemorySetupSteps(data *WorkflowData) (string, []string) { var setupSteps []string setupActionRef := c.resolveActionReference("./actions/setup", data) if setupActionRef != "" || c.actionMode.IsScript() { - // For dev mode (local action path), checkout the actions folder first setupSteps = append(setupSteps, c.generateCheckoutActionsFolder(data)...) - - // Cache restore job doesn't need project support - // Cache job depends on agent job; reuse the agent's trace ID so all jobs share one OTLP trace cacheTraceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) cacheParentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) setupSteps = append(setupSteps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, cacheTraceID, cacheParentSpanID)...) } + return setupActionRef, setupSteps +} - // Prepend setup steps to all cache steps - steps = append(setupSteps, steps...) - - // Job condition: run only if detection job succeeded (no threats found), - // AND the agent job succeeded (do not persist cache when agent failed or was skipped). - // Using always() so this condition is evaluated even if an upstream job is skipped/failed. - // Detection always runs when the agent ran (even for noop), so detection.result == 'success' - // is sufficient — detection short-circuits with success=true when there is nothing to analyze. +func buildUpdateCacheMemoryJobCondition() string { agentSucceeded := BuildEquals( BuildPropertyAccess(fmt.Sprintf("needs.%s.result", constants.AgentJobName)), BuildStringLiteral("success"), ) - jobCondition := RenderCondition(BuildAnd(BuildAnd(BuildFunctionCall("always"), buildDetectionSuccessCondition()), agentSucceeded)) + return RenderCondition(BuildAnd(BuildAnd(BuildFunctionCall("always"), buildDetectionSuccessCondition()), agentSucceeded)) +} - // Set up permissions for the cache update job - // If using local actions (dev mode without action-tag), we need contents: read to checkout the actions folder - permissions := NewPermissionsEmpty().RenderToYAML() // Default: no special permissions needed +func (c *Compiler) updateCacheMemoryPermissions(data *WorkflowData, setupActionRef string) string { if setupActionRef != "" && len(c.generateCheckoutActionsFolder(data)) > 0 { - // Need contents: read to checkout the actions folder - perms := NewPermissionsContentsRead() - permissions = perms.RenderToYAML() + return NewPermissionsContentsRead().RenderToYAML() } + return NewPermissionsEmpty().RenderToYAML() +} - // Set GH_AW_WORKFLOW_ID_SANITIZED so cache keys match those used in the agent job - var jobEnv map[string]string - if data.WorkflowID != "" { - jobEnv = map[string]string{ - "GH_AW_WORKFLOW_ID_SANITIZED": SanitizeWorkflowIDForCacheKey(data.WorkflowID), - } +func updateCacheMemoryJobEnv(data *WorkflowData) map[string]string { + if data.WorkflowID == "" { + return nil } - - job := &Job{ - Name: updateCacheMemoryJobName, - DisplayName: "", // No display name - job ID is sufficient - RunsOn: c.formatFrameworkJobRunsOn(data), - If: jobCondition, - Permissions: permissions, - Needs: []string{string(constants.AgentJobName), string(constants.DetectionJobName), string(constants.ActivationJobName)}, - Env: jobEnv, - Steps: steps, + return map[string]string{ + "GH_AW_WORKFLOW_ID_SANITIZED": SanitizeWorkflowIDForCacheKey(data.WorkflowID), } - - return job, nil } diff --git a/pkg/workflow/central_slash_command_workflow.go b/pkg/workflow/central_slash_command_workflow.go index 57f05a836c5..529a4ec2fa3 100644 --- a/pkg/workflow/central_slash_command_workflow.go +++ b/pkg/workflow/central_slash_command_workflow.go @@ -47,6 +47,15 @@ type helpCommandEntry struct { SourceFile string `json:"source_file,omitempty"` } +type helpCommandAggregate struct { + Description string + DescriptionBy string + SourceFile string + Centralized bool + Decentralized bool + Label bool +} + // GenerateCentralSlashCommandWorkflow generates a single centralized slash-command trigger // workflow for workflows that opt into on.slash_command.strategy: centralized. // When no centralized slash-command workflows are found, any existing generated file is deleted. @@ -167,34 +176,7 @@ func collectCentralSlashCommandRoutes(workflowDataList []*WorkflowData) (map[str // Stable ordering for deterministic output. for commandName := range routesByCommand { - slices.SortFunc(routesByCommand[commandName], func(left, right slashCommandRoute) int { - if left.Workflow != right.Workflow { - if left.Workflow < right.Workflow { - return -1 - } - return 1 - } - leftEvents := strings.Join(left.Events, ",") - rightEvents := strings.Join(right.Events, ",") - if leftEvents != rightEvents { - if leftEvents < rightEvents { - return -1 - } - return 1 - } - switch { - case left.AIReaction < right.AIReaction: - return -1 - case left.AIReaction > right.AIReaction: - return 1 - case !left.StatusComment && right.StatusComment: - return -1 - case left.StatusComment && !right.StatusComment: - return 1 - default: - return 0 - } - }) + slices.SortFunc(routesByCommand[commandName], compareSlashCommandRoutes) } return routesByCommand, mergedEvents @@ -234,39 +216,41 @@ func collectCentralLabelCommandRoutes(workflowDataList []*WorkflowData, mergedEv } for labelName := range routesByLabel { - slices.SortFunc(routesByLabel[labelName], func(left, right slashCommandRoute) int { - if left.Workflow != right.Workflow { - if left.Workflow < right.Workflow { - return -1 - } - return 1 - } - leftEvents := strings.Join(left.Events, ",") - rightEvents := strings.Join(right.Events, ",") - if leftEvents != rightEvents { - if leftEvents < rightEvents { - return -1 - } - return 1 - } - switch { - case left.AIReaction < right.AIReaction: - return -1 - case left.AIReaction > right.AIReaction: - return 1 - case !left.StatusComment && right.StatusComment: - return -1 - case left.StatusComment && !right.StatusComment: - return 1 - default: - return 0 - } - }) + slices.SortFunc(routesByLabel[labelName], compareSlashCommandRoutes) } return routesByLabel } +func compareSlashCommandRoutes(left, right slashCommandRoute) int { + if left.Workflow != right.Workflow { + if left.Workflow < right.Workflow { + return -1 + } + return 1 + } + leftEvents := strings.Join(left.Events, ",") + rightEvents := strings.Join(right.Events, ",") + if leftEvents != rightEvents { + if leftEvents < rightEvents { + return -1 + } + return 1 + } + switch { + case left.AIReaction < right.AIReaction: + return -1 + case left.AIReaction > right.AIReaction: + return 1 + case !left.StatusComment && right.StatusComment: + return -1 + case left.StatusComment && !right.StatusComment: + return 1 + default: + return 0 + } +} + func buildCentralizedRoutes(wd *WorkflowData, routeEvents []string, includeStatusComment bool) []slashCommandRoute { if wd == nil { return nil @@ -348,37 +332,46 @@ func buildCentralSlashCommandWorkflowYAML( helpCommands []helpCommandEntry, helpCommandEnabled bool, ) (string, error) { - slashRoutesJSON, err := json.Marshal(slashRoutesByCommand) - if err != nil { - return "", fmt.Errorf("failed to marshal centralized slash-command routes: %w", err) - } - labelRoutesJSON, err := json.Marshal(labelRoutesByCommand) - if err != nil { - return "", fmt.Errorf("failed to marshal decentralized label-command routes: %w", err) - } - helpCommandsJSON, err := json.Marshal(helpCommands) - if err != nil { - return "", fmt.Errorf("failed to marshal help commands metadata: %w", err) - } - - commandsMetadata, err := json.Marshal(buildCommandsHeaderMetadata(slashRoutesByCommand, labelRoutesByCommand)) + payload, err := buildCentralSlashCommandJSONPayloads(slashRoutesByCommand, labelRoutesByCommand, helpCommands) if err != nil { - return "", fmt.Errorf("failed to marshal centralized slash-command metadata: %w", err) + return "", err } header := GenerateWorkflowHeader("", "gh-aw", "") var b strings.Builder + writeCentralSlashCommandWorkflowHeader(&b, payload.commandsMetadata, slashRoutesByCommand, labelRoutesByCommand, header) + writeCentralSlashCommandWorkflowBody(&b, mergedEvents, runsOn, setupActionRef, payload, helpCommandEnabled) + return b.String(), nil +} + +func writeCentralSlashCommandWorkflowHeader( + b *strings.Builder, + commandsMetadata []byte, + slashRoutesByCommand map[string][]slashCommandRoute, + labelRoutesByCommand map[string][]slashCommandRoute, + header string, +) { b.WriteString("# gh-aw-commands: ") b.Write(commandsMetadata) b.WriteString("\n") - writeCentralRouteSummaryComments(&b, slashRoutesByCommand, labelRoutesByCommand) + writeCentralRouteSummaryComments(b, slashRoutesByCommand, labelRoutesByCommand) b.WriteString(header) +} + +func writeCentralSlashCommandWorkflowBody( + b *strings.Builder, + mergedEvents map[string]map[string]bool, + runsOn string, + setupActionRef string, + payload centralSlashCommandJSONPayloads, + helpCommandEnabled bool, +) { b.WriteString(`name: "Agentic Commands" on: `) - writeCentralSlashEventsYAML(&b, mergedEvents) + writeCentralSlashEventsYAML(b, mergedEvents) b.WriteString(` permissions: {} @@ -387,7 +380,7 @@ jobs: runs-on: ` + runsOn + ` timeout-minutes: 15 `) - writeCentralSlashRoutePermissions(&b, mergedEvents) + writeCentralSlashRoutePermissions(b, mergedEvents) b.WriteString(` steps: - name: Checkout repository @@ -401,9 +394,9 @@ jobs: - name: Route slash command uses: ` + getActionPin("actions/github-script") + ` env: - GH_AW_SLASH_ROUTING: '` + escapeYAMLSingleQuoted(string(slashRoutesJSON)) + `' - GH_AW_LABEL_ROUTING: '` + escapeYAMLSingleQuoted(string(labelRoutesJSON)) + `' - GH_AW_HELP_COMMANDS: '` + escapeYAMLSingleQuoted(string(helpCommandsJSON)) + `' + GH_AW_SLASH_ROUTING: '` + escapeYAMLSingleQuoted(string(payload.slashRoutes)) + `' + GH_AW_LABEL_ROUTING: '` + escapeYAMLSingleQuoted(string(payload.labelRoutes)) + `' + GH_AW_HELP_COMMANDS: '` + escapeYAMLSingleQuoted(string(payload.helpCommands)) + `' GH_AW_HELP_COMMAND_ENABLED: '` + strconv.FormatBool(helpCommandEnabled) + `' GH_AW_SLASH_COMMAND_DOCS_URL: 'https://github.github.com/gh-aw/reference/command-triggers/' with: @@ -413,101 +406,117 @@ jobs: const { main } = require('` + SetupActionDestination + `/route_slash_command.cjs'); await main(); `) - return b.String(), nil } -func buildHelpCommandEntries(workflowDataList []*WorkflowData) []helpCommandEntry { - type aggregate struct { - Description string - DescriptionBy string - SourceFile string - Centralized bool - Decentralized bool - Label bool +type centralSlashCommandJSONPayloads struct { + slashRoutes []byte + labelRoutes []byte + helpCommands []byte + commandsMetadata []byte +} + +func buildCentralSlashCommandJSONPayloads( + slashRoutesByCommand map[string][]slashCommandRoute, + labelRoutesByCommand map[string][]slashCommandRoute, + helpCommands []helpCommandEntry, +) (centralSlashCommandJSONPayloads, error) { + slashRoutesJSON, err := json.Marshal(slashRoutesByCommand) + if err != nil { + return centralSlashCommandJSONPayloads{}, fmt.Errorf("failed to marshal centralized slash-command routes: %w", err) + } + labelRoutesJSON, err := json.Marshal(labelRoutesByCommand) + if err != nil { + return centralSlashCommandJSONPayloads{}, fmt.Errorf("failed to marshal decentralized label-command routes: %w", err) + } + helpCommandsJSON, err := json.Marshal(helpCommands) + if err != nil { + return centralSlashCommandJSONPayloads{}, fmt.Errorf("failed to marshal help commands metadata: %w", err) } - byCommand := make(map[string]aggregate) - byLabel := make(map[string]aggregate) + commandsMetadata, err := json.Marshal(buildCommandsHeaderMetadata(slashRoutesByCommand, labelRoutesByCommand)) + if err != nil { + return centralSlashCommandJSONPayloads{}, fmt.Errorf("failed to marshal centralized slash-command metadata: %w", err) + } + return centralSlashCommandJSONPayloads{slashRoutesJSON, labelRoutesJSON, helpCommandsJSON, commandsMetadata}, nil +} + +func buildHelpCommandEntries(workflowDataList []*WorkflowData) []helpCommandEntry { + byCommand := make(map[string]helpCommandAggregate) + byLabel := make(map[string]helpCommandAggregate) for _, wd := range workflowDataList { if wd == nil || (len(wd.Command) == 0 && len(wd.LabelCommand) == 0) { continue } description := strings.TrimSpace(wd.Description) + addHelpSlashCommandAggregates(byCommand, wd, description) + addHelpLabelCommandAggregates(byLabel, wd, description) + } - for _, commandName := range wd.Command { - trimmed := strings.TrimSpace(commandName) - if trimmed == "" { - continue - } - if trimmed == "help" { - centralSlashCommandWorkflowLog.Printf( - "Warning: 'help' is reserved for the builtin /help handler in workflow %s; "+ - "this command will not be dispatched unless help_command is disabled via aw.json", - wd.WorkflowID, - ) - } - existing := byCommand[trimmed] - if existing.Description != "" && description != "" && existing.Description != description { - centralSlashCommandWorkflowLog.Printf( - "Conflicting descriptions for /%s: keeping %q from workflow %s, ignoring %q from workflow %s", - trimmed, - existing.Description, - existing.DescriptionBy, - description, - wd.WorkflowID, - ) - } - // Conflict resolution keeps the first non-empty description encountered - // while iterating workflowDataList, which is deterministic for compilation. - if existing.Description == "" && description != "" { - existing.Description = description - existing.DescriptionBy = wd.WorkflowID - } - if existing.SourceFile == "" { - existing.SourceFile = wd.WorkflowID - } - if wd.CommandCentralized { - existing.Centralized = true - } else { - // Slash commands are either centralized or decentralized in current workflow metadata: - // CommandCentralized=false indicates the command is handled in its own workflow. - existing.Decentralized = true - } - byCommand[trimmed] = existing + return buildHelpCommandEntriesFromAggregates(byCommand, byLabel) +} + +func addHelpSlashCommandAggregates(byCommand map[string]helpCommandAggregate, wd *WorkflowData, description string) { + for _, commandName := range wd.Command { + trimmed := strings.TrimSpace(commandName) + if trimmed == "" { + continue } + if trimmed == "help" { + centralSlashCommandWorkflowLog.Printf( + "Warning: 'help' is reserved for the builtin /help handler in workflow %s; "+ + "this command will not be dispatched unless help_command is disabled via aw.json", + wd.WorkflowID, + ) + } + existing := mergeHelpCommandDescription(byCommand[trimmed], trimmed, description, wd.WorkflowID, "/%s") + if wd.CommandCentralized { + existing.Centralized = true + } else { + existing.Decentralized = true + } + byCommand[trimmed] = existing + } +} - for _, labelName := range wd.LabelCommand { - trimmed := strings.TrimSpace(labelName) - if trimmed == "" { - continue - } - existing := byLabel[trimmed] - if existing.Description != "" && description != "" && existing.Description != description { - centralSlashCommandWorkflowLog.Printf( - "Conflicting descriptions for label %q: keeping %q from workflow %s, ignoring %q from workflow %s", - trimmed, - existing.Description, - existing.DescriptionBy, - description, - wd.WorkflowID, - ) - } - if existing.Description == "" && description != "" { - existing.Description = description - existing.DescriptionBy = wd.WorkflowID - } - if existing.SourceFile == "" { - existing.SourceFile = wd.WorkflowID - } - existing.Label = true - byLabel[trimmed] = existing +func addHelpLabelCommandAggregates(byLabel map[string]helpCommandAggregate, wd *WorkflowData, description string) { + for _, labelName := range wd.LabelCommand { + trimmed := strings.TrimSpace(labelName) + if trimmed == "" { + continue } + existing := mergeHelpCommandDescription(byLabel[trimmed], trimmed, description, wd.WorkflowID, "label %q") + existing.Label = true + byLabel[trimmed] = existing } +} - commands := sliceutil.SortedKeys(byCommand) +func mergeHelpCommandDescription(existing helpCommandAggregate, name, description, workflowID, labelFormat string) helpCommandAggregate { + if existing.Description != "" && description != "" && existing.Description != description { + centralSlashCommandWorkflowLog.Printf( + "Conflicting descriptions for "+labelFormat+": keeping %q from workflow %s, ignoring %q from workflow %s", + name, + existing.Description, + existing.DescriptionBy, + description, + workflowID, + ) + } + if existing.Description == "" && description != "" { + existing.Description = description + existing.DescriptionBy = workflowID + } + if existing.SourceFile == "" { + existing.SourceFile = workflowID + } + return existing +} +func buildHelpCommandEntriesFromAggregates( + byCommand map[string]helpCommandAggregate, + byLabel map[string]helpCommandAggregate, +) []helpCommandEntry { + commands := sliceutil.SortedKeys(byCommand) labels := sliceutil.SortedKeys(byLabel) - entries := make([]helpCommandEntry, 0, len(commands)+len(labels)) for _, command := range commands { item := byCommand[command] @@ -528,7 +537,6 @@ func buildHelpCommandEntries(workflowDataList []*WorkflowData) []helpCommandEntr SourceFile: item.SourceFile, }) } - return entries } diff --git a/pkg/workflow/checkout_config_parser.go b/pkg/workflow/checkout_config_parser.go index f03d40866b3..861328086ae 100644 --- a/pkg/workflow/checkout_config_parser.go +++ b/pkg/workflow/checkout_config_parser.go @@ -80,203 +80,221 @@ func ParseCheckoutConfigs(raw any) ([]*CheckoutConfig, error) { func checkoutConfigFromMap(m map[string]any) (*CheckoutConfig, error) { cfg := &CheckoutConfig{} + if err := parseCheckoutIdentityFields(cfg, m); err != nil { + return nil, err + } + if err := parseCheckoutAuthFields(cfg, m); err != nil { + return nil, err + } + if cfg.GitHubToken != "" && cfg.GitHubApp != nil { + checkoutManagerLog.Print("Rejecting checkout config: github-token and github-app are mutually exclusive") + return nil, errors.New("checkout: github-token and github-app are mutually exclusive; use one or the other") + } + checkoutManagerLog.Printf("Parsed checkout config: repo=%q, ref=%q, path=%q, current=%v, hasToken=%v, hasApp=%v", + cfg.Repository, cfg.Ref, cfg.Path, cfg.Current, cfg.GitHubToken != "", cfg.GitHubApp != nil) + if err := parseCheckoutGitFields(cfg, m); err != nil { + return nil, err + } + if err := parseCheckoutBooleanFields(cfg, m); err != nil { + return nil, err + } + + return cfg, nil +} + +func parseCheckoutIdentityFields(cfg *CheckoutConfig, m map[string]any) error { if v, ok := m["repository"]; ok { s, ok := v.(string) if !ok { - return nil, errors.New("checkout.repository must be a string") + return errors.New("checkout.repository must be a string") } cfg.Repository = s } - if v, ok := m["ref"]; ok { s, ok := v.(string) if !ok { - return nil, errors.New("checkout.ref must be a string") + return errors.New("checkout.ref must be a string") } cfg.Ref = s } - if v, ok := m["path"]; ok { s, ok := v.(string) if !ok { - return nil, errors.New("checkout.path must be a string") + return errors.New("checkout.path must be a string") } cfg.PathExplicit = true - // Normalize "." to empty string: both mean the workspace root and - // are treated identically by the checkout step generator. if s == "." { s = "" } cfg.Path = s } + return nil +} - // Support both "github-token" (preferred) and "token" (backward compat) +func parseCheckoutAuthFields(cfg *CheckoutConfig, m map[string]any) error { if v, ok := m["github-token"]; ok { s, ok := v.(string) if !ok { - return nil, errors.New("checkout.github-token must be a string") + return errors.New("checkout.github-token must be a string") } cfg.GitHubToken = s } else if v, ok := m["token"]; ok { - // Backward compatibility: "token" is accepted but "github-token" is preferred s, ok := v.(string) if !ok { - return nil, errors.New("checkout.token must be a string") + return errors.New("checkout.token must be a string") } cfg.GitHubToken = s } - - // Parse app configuration for GitHub App-based authentication if v, ok := m["github-app"]; ok { - appMap, ok := v.(map[string]any) - if !ok { - return nil, errors.New("checkout.github-app must be an object") - } - cfg.GitHubApp = parseAppConfig(appMap) - if cfg.GitHubApp.AppID == "" || cfg.GitHubApp.PrivateKey == "" { - return nil, errors.New("checkout.github-app requires both client-id (or app-id) and private-key") - } - } - - // Parse app configuration for safe_outputs-only GitHub App authentication. - parseSafeOutputAppConfig := func(fieldName string, value any) (*GitHubAppConfig, error) { - appMap, ok := value.(map[string]any) - if !ok { - return nil, fmt.Errorf("checkout.%s must be an object", fieldName) - } - appConfig := parseAppConfig(appMap) - if appConfig.AppID == "" || appConfig.PrivateKey == "" { - return nil, fmt.Errorf("checkout.%s requires both client-id (or app-id) and private-key", fieldName) + appConfig, err := parseCheckoutAppConfig("github-app", v) + if err != nil { + return err } - return appConfig, nil + cfg.GitHubApp = appConfig } if v, ok := m["safe-outputs-github-app"]; ok { - appConfig, err := parseSafeOutputAppConfig("safe-outputs-github-app", v) + appConfig, err := parseCheckoutAppConfig("safe-outputs-github-app", v) if err != nil { - return nil, err + return err } cfg.SafeOutputGitHubApp = appConfig } if _, ok := m["safe-output-github-app"]; ok { - return nil, errors.New("checkout.safe-output-github-app is not supported; use checkout.safe-outputs-github-app") + return errors.New("checkout.safe-output-github-app is not supported; use checkout.safe-outputs-github-app") } + return nil +} - // Validate mutual exclusivity of github-token and github-app - if cfg.GitHubToken != "" && cfg.GitHubApp != nil { - checkoutManagerLog.Print("Rejecting checkout config: github-token and github-app are mutually exclusive") - return nil, errors.New("checkout: github-token and github-app are mutually exclusive; use one or the other") +func parseCheckoutAppConfig(fieldName string, value any) (*GitHubAppConfig, error) { + appMap, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("checkout.%s must be an object", fieldName) } + appConfig := parseAppConfig(appMap) + if appConfig.AppID == "" || appConfig.PrivateKey == "" { + return nil, fmt.Errorf("checkout.%s requires both client-id (or app-id) and private-key", fieldName) + } + return appConfig, nil +} - checkoutManagerLog.Printf("Parsed checkout config: repo=%q, ref=%q, path=%q, current=%v, hasToken=%v, hasApp=%v", - cfg.Repository, cfg.Ref, cfg.Path, cfg.Current, cfg.GitHubToken != "", cfg.GitHubApp != nil) - +func parseCheckoutGitFields(cfg *CheckoutConfig, m map[string]any) error { if v, ok := m["fetch-depth"]; ok { - switch n := v.(type) { - case int: - depth := n - cfg.FetchDepth = &depth - case int64: - depth := int(n) - cfg.FetchDepth = &depth - case uint64: - depth := int(n) - cfg.FetchDepth = &depth - case float64: - if n != float64(int64(n)) { - return nil, errors.New("checkout.fetch-depth must be an integer") - } - depth := int(n) - cfg.FetchDepth = &depth - default: - return nil, errors.New("checkout.fetch-depth must be an integer") - } - if cfg.FetchDepth != nil && *cfg.FetchDepth < 0 { - return nil, errors.New("checkout.fetch-depth must be >= 0") + depth, err := parseCheckoutFetchDepth(v) + if err != nil { + return err } + cfg.FetchDepth = depth } - if v, ok := m["sparse-checkout"]; ok { s, ok := v.(string) if !ok { - return nil, errors.New("checkout.sparse-checkout must be a string") + return errors.New("checkout.sparse-checkout must be a string") } cfg.SparseCheckout = s } - if v, ok := m["submodules"]; ok { - switch sv := v.(type) { - case string: - cfg.Submodules = sv - case bool: - if sv { - cfg.Submodules = "true" - } else { - cfg.Submodules = "false" - } - default: - return nil, errors.New("checkout.submodules must be a string or boolean") + submodules, err := parseCheckoutSubmodules(v) + if err != nil { + return err } + cfg.Submodules = submodules } - - if v, ok := m["lfs"]; ok { - b, ok := v.(bool) - if !ok { - return nil, errors.New("checkout.lfs must be a boolean") + if v, ok := m["fetch"]; ok { + refs, err := parseCheckoutFetchRefs(v) + if err != nil { + return err } - cfg.LFS = b + cfg.Fetch = refs } + return nil +} - if v, ok := m["current"]; ok { - b, ok := v.(bool) - if !ok { - return nil, errors.New("checkout.current must be a boolean") +func parseCheckoutFetchDepth(v any) (*int, error) { + var depth int + switch n := v.(type) { + case int: + depth = n + case int64: + depth = int(n) + case uint64: + depth = int(n) + case float64: + if n != float64(int64(n)) { + return nil, errors.New("checkout.fetch-depth must be an integer") } - cfg.Current = b + depth = int(n) + default: + return nil, errors.New("checkout.fetch-depth must be an integer") } + if depth < 0 { + return nil, errors.New("checkout.fetch-depth must be >= 0") + } + return &depth, nil +} - if v, ok := m["fetch"]; ok { - switch fv := v.(type) { - case string: - // Single string shorthand: treat as a one-element list - if strings.TrimSpace(fv) == "" { - return nil, errors.New("checkout.fetch string value must not be empty") - } - cfg.Fetch = []string{fv} - case []any: - refs := make([]string, 0, len(fv)) - for i, item := range fv { - s, ok := item.(string) - if !ok { - return nil, fmt.Errorf("checkout.fetch[%d] must be a string, got %T", i, item) - } - if strings.TrimSpace(s) == "" { - return nil, fmt.Errorf("checkout.fetch[%d] must not be empty", i) - } - refs = append(refs, s) - } - cfg.Fetch = refs - default: - return nil, errors.New("checkout.fetch must be a string or an array of strings") - } +func parseCheckoutSubmodules(v any) (string, error) { + switch sv := v.(type) { + case string: + return sv, nil + case bool: + if sv { + return "true", nil + } + return "false", nil + default: + return "", errors.New("checkout.submodules must be a string or boolean") } +} - if v, ok := m["wiki"]; ok { - b, ok := v.(bool) - if !ok { - return nil, errors.New("checkout.wiki must be a boolean") - } - cfg.Wiki = b +func parseCheckoutFetchRefs(v any) ([]string, error) { + switch fv := v.(type) { + case string: + if strings.TrimSpace(fv) == "" { + return nil, errors.New("checkout.fetch string value must not be empty") + } + return []string{fv}, nil + case []any: + return parseCheckoutFetchRefArray(fv) + default: + return nil, errors.New("checkout.fetch must be a string or an array of strings") } +} - if v, ok := m["force-clean-git-credentials"]; ok { - b, ok := v.(bool) +func parseCheckoutFetchRefArray(fv []any) ([]string, error) { + refs := make([]string, 0, len(fv)) + for i, item := range fv { + s, ok := item.(string) if !ok { - return nil, errors.New("checkout.force-clean-git-credentials must be a boolean") + return nil, fmt.Errorf("checkout.fetch[%d] must be a string, got %T", i, item) } - cfg.CleanGitCredentials = b + if strings.TrimSpace(s) == "" { + return nil, fmt.Errorf("checkout.fetch[%d] must not be empty", i) + } + refs = append(refs, s) } + return refs, nil +} - return cfg, nil +func parseCheckoutBooleanFields(cfg *CheckoutConfig, m map[string]any) error { + boolFields := []struct { + key string + set func(bool) + }{ + {"lfs", func(v bool) { cfg.LFS = v }}, + {"current", func(v bool) { cfg.Current = v }}, + {"wiki", func(v bool) { cfg.Wiki = v }}, + {"force-clean-git-credentials", func(v bool) { cfg.CleanGitCredentials = v }}, + } + for _, field := range boolFields { + if v, ok := m[field.key]; ok { + b, ok := v.(bool) + if !ok { + return fmt.Errorf("checkout.%s must be a boolean", field.key) + } + field.set(b) + } + } + return nil } // buildCheckoutsPromptContent returns a markdown bullet list describing all user-configured @@ -304,59 +322,7 @@ func buildCheckoutsPromptContent(checkouts []*CheckoutConfig) string { continue } - // Build the full absolute path using $GITHUB_WORKSPACE as root. - // Normalize the path: strip "./" prefix; bare "." and "" both mean root. - relPath := strings.TrimPrefix(cfg.Path, "./") - if relPath == "." { - relPath = "" - } - isRoot := relPath == "" - absPath := "$GITHUB_WORKSPACE" - if !isRoot { - absPath += "/" + relPath - } - - // Determine repo: use configured value or fall back to the triggering repository expression. - // For wiki checkouts, append the ".wiki" suffix so the prompt accurately reflects what was checked out. - repo := cfg.Repository - if repo == "" { - repo = "${{ github.repository }}" - } - if cfg.Wiki { - if !strings.HasSuffix(repo, ".wiki") { - repo += ".wiki" - } - } - - line := fmt.Sprintf(" - repo `%s` → `%s`", repo, absPath) - if isRoot { - line += " (cwd)" - } - if cfg.Wiki { - line += " (wiki)" - } - if cfg.Current { - line += " (**current** - this is the repository you are working on; use this as the target for all GitHub operations unless otherwise specified)" - } - - // Annotate fetch-depth so the agent knows how much history is available - if cfg.FetchDepth != nil && *cfg.FetchDepth == 0 { - line += " [full history, all branches available as remote-tracking refs]" - } else if cfg.FetchDepth != nil { - line += fmt.Sprintf(" [shallow clone, fetch-depth=%d]", *cfg.FetchDepth) - } else { - line += " [shallow clone, fetch-depth=1 (default)]" - } - - // Annotate additionally fetched refs - if len(cfg.Fetch) > 0 { - line += fmt.Sprintf(" [additional refs fetched: %s]", strings.Join(cfg.Fetch, ", ")) - } - if strings.TrimSpace(cfg.SparseCheckout) != "" { - line += " [sparse checkout enabled]" - } - - sb.WriteString(line + "\n") + sb.WriteString(buildCheckoutPromptLine(cfg) + "\n") } // General guidance about unavailable branches @@ -371,3 +337,58 @@ func buildCheckoutsPromptContent(checkouts []*CheckoutConfig) string { return sb.String() } + +func buildCheckoutPromptLine(cfg *CheckoutConfig) string { + absPath, isRoot := checkoutPromptPath(cfg) + line := fmt.Sprintf(" - repo `%s` → `%s`", checkoutPromptRepo(cfg), absPath) + if isRoot { + line += " (cwd)" + } + if cfg.Wiki { + line += " (wiki)" + } + if cfg.Current { + line += " (**current** - this is the repository you are working on; use this as the target for all GitHub operations unless otherwise specified)" + } + line += checkoutPromptFetchDepth(cfg) + if len(cfg.Fetch) > 0 { + line += fmt.Sprintf(" [additional refs fetched: %s]", strings.Join(cfg.Fetch, ", ")) + } + if strings.TrimSpace(cfg.SparseCheckout) != "" { + line += " [sparse checkout enabled]" + } + return line +} + +func checkoutPromptPath(cfg *CheckoutConfig) (string, bool) { + relPath := strings.TrimPrefix(cfg.Path, "./") + if relPath == "." { + relPath = "" + } + isRoot := relPath == "" + if isRoot { + return "$GITHUB_WORKSPACE", true + } + return "$GITHUB_WORKSPACE/" + relPath, false +} + +func checkoutPromptRepo(cfg *CheckoutConfig) string { + repo := cfg.Repository + if repo == "" { + repo = "${{ github.repository }}" + } + if cfg.Wiki && !strings.HasSuffix(repo, ".wiki") { + repo += ".wiki" + } + return repo +} + +func checkoutPromptFetchDepth(cfg *CheckoutConfig) string { + if cfg.FetchDepth != nil && *cfg.FetchDepth == 0 { + return " [full history, all branches available as remote-tracking refs]" + } + if cfg.FetchDepth != nil { + return fmt.Sprintf(" [shallow clone, fetch-depth=%d]", *cfg.FetchDepth) + } + return " [shallow clone, fetch-depth=1 (default)]" +} diff --git a/pkg/workflow/checkout_manager.go b/pkg/workflow/checkout_manager.go index eccef6c590b..d9ffa7c138a 100644 --- a/pkg/workflow/checkout_manager.go +++ b/pkg/workflow/checkout_manager.go @@ -266,83 +266,93 @@ func (cm *CheckoutManager) add(cfg *CheckoutConfig) { return } - // Normalize path: "." and "" both refer to the workspace root. + key := normalizedCheckoutKey(cfg) + + if idx, exists := cm.index[key]; exists { + mergeCheckoutConfigIntoEntry(cm.ordered[idx], cfg) + checkoutManagerLog.Printf("Merged checkout for path=%q repository=%q", key.path, key.repository) + } else { + entry := newResolvedCheckoutFromConfig(key, cfg) + cm.index[key] = len(cm.ordered) + cm.ordered = append(cm.ordered, entry) + checkoutManagerLog.Printf("Added checkout for path=%q repository=%q", key.path, key.repository) + } +} + +func normalizedCheckoutKey(cfg *CheckoutConfig) checkoutKey { normalizedPath := cfg.Path if normalizedPath == "." { normalizedPath = "" } - // Normalize repository for wiki checkouts: strip a trailing ".wiki" suffix so that - // "owner/repo" and "owner/repo.wiki" with Wiki:true resolve to the same deduplication key. normalizedRepo := cfg.Repository if cfg.Wiki && strings.HasSuffix(normalizedRepo, ".wiki") { normalizedRepo = strings.TrimSuffix(normalizedRepo, ".wiki") } - key := checkoutKey{ + return checkoutKey{ repository: normalizedRepo, path: normalizedPath, wiki: cfg.Wiki, } +} - if idx, exists := cm.index[key]; exists { - // Merge into existing entry; first-seen wins for ref and token/app (auth is mutually exclusive: - // once either github-token or github-app is set for an entry, the other method is not added - // even if a later config provides it — this preserves the main workflow's auth choice). - entry := cm.ordered[idx] - entry.fetchDepth = deeperFetchDepth(entry.fetchDepth, cfg.FetchDepth) - if cfg.Ref != "" && entry.ref == "" { - entry.ref = cfg.Ref // first-seen ref wins - } - if cfg.GitHubToken != "" && entry.token == "" && entry.githubApp == nil { - entry.token = cfg.GitHubToken // first-seen auth wins (mutually exclusive with github-app) - } - if cfg.GitHubApp != nil && entry.githubApp == nil && entry.token == "" { - entry.githubApp = cfg.GitHubApp // first-seen auth wins (mutually exclusive with github-token) - } - if cfg.SafeOutputGitHubApp != nil && entry.safeOutputApp == nil { - entry.safeOutputApp = cfg.SafeOutputGitHubApp // first-seen safe_outputs auth wins - } - if cfg.SparseCheckout != "" { - entry.sparsePatterns = mergeSparsePatterns(entry.sparsePatterns, cfg.SparseCheckout) - } - if cfg.LFS { - entry.lfs = true - } - if cfg.Current { - entry.current = true - } - if cfg.Submodules != "" && entry.submodules == "" { - entry.submodules = cfg.Submodules - } - if len(cfg.Fetch) > 0 { - entry.fetchRefs = mergeFetchRefs(entry.fetchRefs, cfg.Fetch) - } - if cfg.CleanGitCredentials { - entry.cleanCreds = true - } - checkoutManagerLog.Printf("Merged checkout for path=%q repository=%q", key.path, key.repository) - } else { - entry := &resolvedCheckout{ - key: key, - ref: cfg.Ref, - token: cfg.GitHubToken, - githubApp: cfg.GitHubApp, - safeOutputApp: cfg.SafeOutputGitHubApp, - fetchDepth: cfg.FetchDepth, - submodules: cfg.Submodules, - lfs: cfg.LFS, - current: cfg.Current, - cleanCreds: cfg.CleanGitCredentials, - } - if cfg.SparseCheckout != "" { - entry.sparsePatterns = mergeSparsePatterns(nil, cfg.SparseCheckout) - } - if len(cfg.Fetch) > 0 { - entry.fetchRefs = mergeFetchRefs(nil, cfg.Fetch) - } - cm.index[key] = len(cm.ordered) - cm.ordered = append(cm.ordered, entry) - checkoutManagerLog.Printf("Added checkout for path=%q repository=%q", key.path, key.repository) +func mergeCheckoutConfigIntoEntry(entry *resolvedCheckout, cfg *CheckoutConfig) { + entry.fetchDepth = deeperFetchDepth(entry.fetchDepth, cfg.FetchDepth) + if cfg.Ref != "" && entry.ref == "" { + entry.ref = cfg.Ref + } + if cfg.GitHubToken != "" && entry.token == "" && entry.githubApp == nil { + entry.token = cfg.GitHubToken + } + if cfg.GitHubApp != nil && entry.githubApp == nil && entry.token == "" { + entry.githubApp = cfg.GitHubApp + } + if cfg.SafeOutputGitHubApp != nil && entry.safeOutputApp == nil { + entry.safeOutputApp = cfg.SafeOutputGitHubApp + } + mergeCheckoutConfigFeatures(entry, cfg) +} + +func mergeCheckoutConfigFeatures(entry *resolvedCheckout, cfg *CheckoutConfig) { + if cfg.SparseCheckout != "" { + entry.sparsePatterns = mergeSparsePatterns(entry.sparsePatterns, cfg.SparseCheckout) + } + if cfg.LFS { + entry.lfs = true + } + if cfg.Current { + entry.current = true + } + if cfg.Submodules != "" && entry.submodules == "" { + entry.submodules = cfg.Submodules + } + if len(cfg.Fetch) > 0 { + entry.fetchRefs = mergeFetchRefs(entry.fetchRefs, cfg.Fetch) + } + if cfg.CleanGitCredentials { + entry.cleanCreds = true + } +} + +func newResolvedCheckoutFromConfig(key checkoutKey, cfg *CheckoutConfig) *resolvedCheckout { + entry := &resolvedCheckout{ + key: key, + ref: cfg.Ref, + token: cfg.GitHubToken, + githubApp: cfg.GitHubApp, + safeOutputApp: cfg.SafeOutputGitHubApp, + fetchDepth: cfg.FetchDepth, + submodules: cfg.Submodules, + lfs: cfg.LFS, + current: cfg.Current, + cleanCreds: cfg.CleanGitCredentials, + } + if cfg.SparseCheckout != "" { + entry.sparsePatterns = mergeSparsePatterns(nil, cfg.SparseCheckout) + } + if len(cfg.Fetch) > 0 { + entry.fetchRefs = mergeFetchRefs(nil, cfg.Fetch) } + return entry } // GetDefaultCheckoutOverride returns the resolved checkout for the default workspace diff --git a/pkg/workflow/checkout_step_generator.go b/pkg/workflow/checkout_step_generator.go index 03e3a8aa660..1307a67113f 100644 --- a/pkg/workflow/checkout_step_generator.go +++ b/pkg/workflow/checkout_step_generator.go @@ -244,42 +244,37 @@ func (cm *CheckoutManager) GenerateGitHubFolderCheckoutStep(repository, ref, tok // condition. func (cm *CheckoutManager) GenerateConfigureGitCredentialsSteps(gitRemoteToken string, condition ConditionNode) []string { conditionStr := RenderCondition(condition) - - // Collect subdirectory cross-repo checkouts that need per-repo re-authentication. - type subRepo struct { - repository string - path string + subRepos := cm.gitCredentialSubRepos() + rootRepo := cm.gitCredentialRootRepo() + if len(subRepos) == 0 { + return buildSimpleGitCredentialsStep(conditionStr, rootRepo, gitRemoteToken) } - var subRepos []subRepo + return buildMultiRepoGitCredentialsStep(conditionStr, rootRepo, gitRemoteToken, buildSubRepoEnvVars(subRepos)) +} + +type gitCredentialSubRepo struct { + repository string + path string +} + +type gitCredentialSubRepoEnvVar struct { + repository string + path string + envVarName string + pathEnvVarName string +} + +func (cm *CheckoutManager) gitCredentialSubRepos() []gitCredentialSubRepo { + var subRepos []gitCredentialSubRepo for _, entry := range cm.ordered { if entry.key.repository != "" && entry.key.path != "" && !entry.key.wiki { - subRepos = append(subRepos, subRepo{repository: entry.key.repository, path: entry.key.path}) - } - } - - if len(subRepos) == 0 { - // Simple case: single root repo, call the script directly. - rootRepo := "${{ github.repository }}" - for _, entry := range cm.ordered { - // If a non-default checkout targets the workspace root (no path:), it will clobber - // the root checkout; configure git for the effective repo at the root. - if entry.key.wiki || entry.key.path != "" || entry.key.repository == "" { - continue - } - rootRepo = entry.key.repository - } - return []string{ - " - name: Configure Git credentials\n", - fmt.Sprintf(" if: %s\n", conditionStr), - " env:\n", - fmt.Sprintf(" GITHUB_REPOSITORY: %s\n", rootRepo), - " GITHUB_SERVER_URL: ${{ github.server_url }}\n", - fmt.Sprintf(" GIT_TOKEN: %s\n", gitRemoteToken), - " run: bash \"${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh\"\n", + subRepos = append(subRepos, gitCredentialSubRepo{repository: entry.key.repository, path: entry.key.path}) } } + return subRepos +} - // Multi-repo case: configure the root repo, then re-authenticate each subdirectory checkout. +func (cm *CheckoutManager) gitCredentialRootRepo() string { rootRepo := "${{ github.repository }}" for _, entry := range cm.ordered { if entry.key.wiki || entry.key.path != "" || entry.key.repository == "" { @@ -287,21 +282,25 @@ func (cm *CheckoutManager) GenerateConfigureGitCredentialsSteps(gitRemoteToken s } rootRepo = entry.key.repository } + return rootRepo +} - // Assign each sub-repo a dedicated env var so that GitHub Actions expressions - // (e.g. "${{ github.event.inputs.target_repo }}") are never inlined directly - // into the shell command, preventing template-injection scanner failures. - // pathEnvVarName is set only when path is expression-based; otherwise the - // literal path string is used directly in the shell command. - type subRepoEnvVar struct { - repository string - path string - envVarName string - pathEnvVarName string // non-empty only when path is a GitHub Actions expression +func buildSimpleGitCredentialsStep(conditionStr, rootRepo, gitRemoteToken string) []string { + return []string{ + " - name: Configure Git credentials\n", + fmt.Sprintf(" if: %s\n", conditionStr), + " env:\n", + fmt.Sprintf(" GITHUB_REPOSITORY: %s\n", rootRepo), + " GITHUB_SERVER_URL: ${{ github.server_url }}\n", + fmt.Sprintf(" GIT_TOKEN: %s\n", gitRemoteToken), + " run: bash \"${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh\"\n", } - subRepoEnvVars := make([]subRepoEnvVar, len(subRepos)) +} + +func buildSubRepoEnvVars(subRepos []gitCredentialSubRepo) []gitCredentialSubRepoEnvVar { + subRepoEnvVars := make([]gitCredentialSubRepoEnvVar, len(subRepos)) for i, repo := range subRepos { - ev := subRepoEnvVar{ + ev := gitCredentialSubRepoEnvVar{ repository: repo.repository, path: repo.path, envVarName: fmt.Sprintf("GH_AW_SUBREPO_%d", i), @@ -311,8 +310,27 @@ func (cm *CheckoutManager) GenerateConfigureGitCredentialsSteps(gitRemoteToken s } subRepoEnvVars[i] = ev } + return subRepoEnvVars +} - // Build the env block, including a dedicated var for every sub-repo. +func buildMultiRepoGitCredentialsStep(conditionStr, rootRepo, gitRemoteToken string, subRepoEnvVars []gitCredentialSubRepoEnvVar) []string { + steps := []string{ + " - name: Configure Git credentials\n", + fmt.Sprintf(" if: %s\n", conditionStr), + } + steps = append(steps, buildSubRepoEnvLines(rootRepo, gitRemoteToken, subRepoEnvVars)...) + steps = append(steps, + " run: |\n", + " bash \"${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh\"\n", + " GIT_SERVER_URL_STRIPPED=\"${GITHUB_SERVER_URL#https://}\"\n", + ) + for _, repo := range subRepoEnvVars { + steps = append(steps, buildSubRepoReauthLines(repo)...) + } + return append(steps, " echo \"Git configured with standard GitHub Actions identity\"\n") +} + +func buildSubRepoEnvLines(rootRepo, gitRemoteToken string, subRepoEnvVars []gitCredentialSubRepoEnvVar) []string { envLines := []string{ " env:\n", fmt.Sprintf(" GITHUB_REPOSITORY: %s\n", rootRepo), @@ -333,39 +351,22 @@ func (cm *CheckoutManager) GenerateConfigureGitCredentialsSteps(gitRemoteToken s envLines = append(envLines, fmt.Sprintf(" %s: %s\n", repo.pathEnvVarName, githubExpressionWhitespaceReplacer.Replace(repo.path))) } } + return envLines +} - steps := []string{ - " - name: Configure Git credentials\n", - fmt.Sprintf(" if: %s\n", conditionStr), +func buildSubRepoReauthLines(repo gitCredentialSubRepoEnvVar) []string { + gitDir := fmt.Sprintf("%q", repo.path) + if repo.pathEnvVarName != "" { + gitDir = fmt.Sprintf("\"${%s}\"", repo.pathEnvVarName) } - steps = append(steps, envLines...) - steps = append(steps, - " run: |\n", - " bash \"${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh\"\n", - " GIT_SERVER_URL_STRIPPED=\"${GITHUB_SERVER_URL#https://}\"\n", - ) - for _, repo := range subRepoEnvVars { - // Use the path env var reference when path is expression-based to avoid - // inlining ${{ }} into the run: block (template-injection scanner risk). - gitDir := fmt.Sprintf("%q", repo.path) - if repo.pathEnvVarName != "" { - gitDir = fmt.Sprintf("\"${%s}\"", repo.pathEnvVarName) - } - // Comment uses the path literal (or env var reference) — never the raw - // repository expression — so ${{ }} never appears in the run: block. - commentRef := repo.path - if repo.pathEnvVarName != "" { - commentRef = "${" + repo.pathEnvVarName + "}" - } - steps = append(steps, - fmt.Sprintf(" # Re-authenticate git for %s\n", commentRef), - fmt.Sprintf(" git -C %s remote set-url origin \"https://x-access-token:${GIT_TOKEN}@${GIT_SERVER_URL_STRIPPED}/${%s}.git\"\n", gitDir, repo.envVarName), - ) + commentRef := repo.path + if repo.pathEnvVarName != "" { + commentRef = "${" + repo.pathEnvVarName + "}" + } + return []string{ + fmt.Sprintf(" # Re-authenticate git for %s\n", commentRef), + fmt.Sprintf(" git -C %s remote set-url origin \"https://x-access-token:${GIT_TOKEN}@${GIT_SERVER_URL_STRIPPED}/${%s}.git\"\n", gitDir, repo.envVarName), } - steps = append(steps, - " echo \"Git configured with standard GitHub Actions identity\"\n", - ) - return steps } // GenerateDefaultCheckoutStep emits the default workspace checkout, applying any @@ -390,99 +391,19 @@ func (cm *CheckoutManager) GenerateDefaultCheckoutStep( sb.WriteString(" - name: Checkout repository\n") fmt.Fprintf(&sb, " uses: %s\n", getActionPin("actions/checkout")) sb.WriteString(" with:\n") - cleanCreds := override != nil && override.cleanCreds - if cm.keepCredentialsForPush { - // safe_outputs job: retain credentials so later git fetch/push can authenticate - // using the push-capable token installed at checkout time. - sb.WriteString(" persist-credentials: true\n") - } else if cleanCreds { - sb.WriteString(" persist-credentials: true\n") - } else { - // Security: default behavior disables credential persistence so the agent cannot - // exfiltrate credentials from disk. - sb.WriteString(" persist-credentials: false\n") - } - - // Track whether a token has been written to the checkout step so the safe_outputs - // push-token fallback below does not double-emit. + writeCheckoutPersistCredentials(&sb, cm.keepCredentialsForPush, cleanCreds) tokenEmitted := false - - // Apply trial mode overrides if trialMode { - if trialLogicalRepoSlug != "" { - fmt.Fprintf(&sb, " repository: %s\n", trialLogicalRepoSlug) - } - effectiveToken := getEffectiveGitHubToken("") - fmt.Fprintf(&sb, " token: %s\n", effectiveToken) + writeTrialCheckoutOverrides(&sb, trialLogicalRepoSlug) tokenEmitted = true } - - // Apply user overrides (only when NOT in trial mode to avoid conflicts) if !trialMode && override != nil { - if override.key.wiki { - // Wiki checkout: use "{repository}.wiki" as the effective repository. - fmt.Fprintf(&sb, " repository: %s\n", wikiRepository(override.key.repository)) - } else if override.key.repository != "" { - fmt.Fprintf(&sb, " repository: %s\n", override.key.repository) - } - if override.ref != "" { - fmt.Fprintf(&sb, " ref: %s\n", override.ref) - } - // Prevent actions/checkout from adding --filter=blob:none when sparse-checkout - // is specified. Blobless clones require credentials for lazy blob fetches, but - // agent jobs intentionally do not retain git credentials after checkout, making - // offline git operations fail. Using blob:limit=1073741824 (1 GiB) effectively - // fetches all blobs up front on GitHub-hosted repos (GitHub rejects blobs > 100 MiB), - // while keeping the filter non-empty so actions/checkout won't substitute blob:none. - // The subsequent repair step then clears partial-clone markers entirely. - if len(override.sparsePatterns) > 0 { - sb.WriteString(" filter: 'blob:limit=1073741824'\n") - } - // Determine effective token: github-app-minted token takes precedence - effectiveOverrideToken := override.token - if override.githubApp != nil { - // Determine the actual index of the default checkout to reference the correct - // app-token step ID. Do not assume it is always at index 0. - defaultIdx := 0 - if idx, ok := cm.index[override.key]; ok { - defaultIdx = idx - } - //nolint:gosec // G101: False positive - this is a GitHub Actions expression template placeholder, not a hardcoded credential - effectiveOverrideToken = fmt.Sprintf("${{ steps.checkout-app-token-%d.outputs.token }}", defaultIdx) - if override.githubApp.shouldIgnoreMissingKey() { - effectiveOverrideToken = combineTokenExpressions(effectiveOverrideToken, getEffectiveGitHubToken(override.token)) - } - } - if effectiveOverrideToken != "" { - fmt.Fprintf(&sb, " token: %s\n", effectiveOverrideToken) - tokenEmitted = true - } - if override.fetchDepth != nil { - fmt.Fprintf(&sb, " fetch-depth: %d\n", *override.fetchDepth) - } - if len(override.sparsePatterns) > 0 { - sb.WriteString(" sparse-checkout: |\n") - for _, pattern := range override.sparsePatterns { - fmt.Fprintf(&sb, " %s\n", strings.TrimSpace(pattern)) - } - } - if override.submodules != "" { - fmt.Fprintf(&sb, " submodules: %s\n", override.submodules) - } - if override.lfs { - sb.WriteString(" lfs: true\n") - } + tokenEmitted = cm.writeDefaultCheckoutOverride(&sb, override) } - - // safe_outputs job: when no explicit token was written above, persist the resolved - // push token so the credential retained in .git/config matches the token the - // safe-output handlers use to fetch/push (avoiding both a wrong-token push and the - // duplicate Authorization header that a separate per-command extraheader would add). if !trialMode && !tokenEmitted && cm.keepCredentialsForPush && cm.pushToken != "" { fmt.Fprintf(&sb, " token: %s\n", cm.pushToken) } - steps := []string{sb.String()} if override != nil && len(override.sparsePatterns) > 0 { steps = append(steps, generateSparseCheckoutPartialCloneResetStep("")) @@ -490,95 +411,120 @@ func (cm *CheckoutManager) GenerateDefaultCheckoutStep( if cleanCreds && !cm.keepCredentialsForPush { steps = append(steps, generateCheckoutCredentialsCleanupStep()) } - - // Emit a git fetch step if the user requested additional refs. - // In trial mode the fetch step is still emitted so the behaviour - // mirrors production as closely as possible. if override != nil && len(override.fetchRefs) > 0 { - defaultIdx := 0 - if idx, ok := cm.index[override.key]; ok { - defaultIdx = idx - } - if fetchStep := generateFetchStepLines(override, defaultIdx); fetchStep != "" { + if fetchStep := generateFetchStepLines(override, cm.defaultCheckoutIndex(override)); fetchStep != "" { steps = append(steps, fetchStep) } } - return steps } -// generateCheckoutStepLines generates YAML step lines for a single non-default checkout. -// The index parameter identifies the checkout's position in the ordered list, used to -// reference the correct app token minting step when app authentication is configured. -// When keepCredentialsForPush is true (safe_outputs job), credentials are retained -// (persist-credentials: true) and the post-checkout cleanup step is suppressed so a later -// git fetch/push can authenticate. -func generateCheckoutStepLines(entry *resolvedCheckout, index int, keepCredentialsForPush bool, pushToken string, getActionPin func(string) string) []string { - checkoutManagerLog.Printf("Generating checkout step lines: index=%d, repo=%q, path=%q, ref=%q, appAuth=%v", - index, entry.key.repository, entry.key.path, entry.ref, entry.githubApp != nil) - name := "Checkout " + checkoutStepName(entry.key) - var sb strings.Builder - fmt.Fprintf(&sb, " - name: %s\n", name) - fmt.Fprintf(&sb, " uses: %s\n", getActionPin("actions/checkout")) - sb.WriteString(" with:\n") - - if keepCredentialsForPush { - // safe_outputs job: retain credentials so later git fetch/push can authenticate. - sb.WriteString(" persist-credentials: true\n") - } else if entry.cleanCreds { +func writeCheckoutPersistCredentials(sb *strings.Builder, keepCredentialsForPush, cleanCreds bool) { + if keepCredentialsForPush || cleanCreds { sb.WriteString(" persist-credentials: true\n") } else { - // Security: default behavior disables credential persistence sb.WriteString(" persist-credentials: false\n") } +} + +func writeTrialCheckoutOverrides(sb *strings.Builder, trialLogicalRepoSlug string) { + if trialLogicalRepoSlug != "" { + fmt.Fprintf(sb, " repository: %s\n", trialLogicalRepoSlug) + } + fmt.Fprintf(sb, " token: %s\n", getEffectiveGitHubToken("")) +} +func (cm *CheckoutManager) writeDefaultCheckoutOverride(sb *strings.Builder, override *resolvedCheckout) bool { + writeCheckoutRepositoryRef(sb, override) + writeCheckoutSparseFilter(sb, len(override.sparsePatterns) > 0) + tokenEmitted := cm.writeDefaultCheckoutToken(sb, override) + writeCheckoutOptionalFields(sb, override.fetchDepth, override.sparsePatterns, override.submodules, override.lfs) + return tokenEmitted +} + +func writeCheckoutRepositoryRef(sb *strings.Builder, entry *resolvedCheckout) { if entry.key.wiki { - // Wiki checkout: use "{repository}.wiki" as the effective repository. - fmt.Fprintf(&sb, " repository: %s\n", wikiRepository(entry.key.repository)) + fmt.Fprintf(sb, " repository: %s\n", wikiRepository(entry.key.repository)) } else if entry.key.repository != "" { - fmt.Fprintf(&sb, " repository: %s\n", entry.key.repository) + fmt.Fprintf(sb, " repository: %s\n", entry.key.repository) } if entry.ref != "" { - fmt.Fprintf(&sb, " ref: %s\n", entry.ref) + fmt.Fprintf(sb, " ref: %s\n", entry.ref) } - if entry.key.path != "" { - fmt.Fprintf(&sb, " path: %s\n", entry.key.path) +} + +func writeCheckoutSparseFilter(sb *strings.Builder, hasSparsePatterns bool) { + if hasSparsePatterns { + sb.WriteString(" filter: 'blob:limit=1073741824'\n") } - // Determine effective token: github-app-minted token takes precedence - effectiveToken := resolveCheckoutTokenExpression(entry, index, false) - // safe_outputs job: when this checkout declares no token/app of its own, persist the - // resolved push token so the retained .git/config credential matches the token the - // safe-output handlers use to fetch/push. - if effectiveToken == "" && keepCredentialsForPush && pushToken != "" { - effectiveToken = pushToken +} + +func (cm *CheckoutManager) writeDefaultCheckoutToken(sb *strings.Builder, override *resolvedCheckout) bool { + effectiveOverrideToken := override.token + if override.githubApp != nil { + //nolint:gosec // G101: GitHub Actions expression placeholder, not a hardcoded credential + effectiveOverrideToken = fmt.Sprintf("${{ steps.checkout-app-token-%d.outputs.token }}", cm.defaultCheckoutIndex(override)) + if override.githubApp.shouldIgnoreMissingKey() { + effectiveOverrideToken = combineTokenExpressions(effectiveOverrideToken, getEffectiveGitHubToken(override.token)) + } } - if effectiveToken != "" { - fmt.Fprintf(&sb, " token: %s\n", effectiveToken) + if effectiveOverrideToken == "" { + return false } - if entry.fetchDepth != nil { - fmt.Fprintf(&sb, " fetch-depth: %d\n", *entry.fetchDepth) + fmt.Fprintf(sb, " token: %s\n", effectiveOverrideToken) + return true +} + +func (cm *CheckoutManager) defaultCheckoutIndex(override *resolvedCheckout) int { + if idx, ok := cm.index[override.key]; ok { + return idx } - if len(entry.sparsePatterns) > 0 { + return 0 +} + +func writeCheckoutOptionalFields(sb *strings.Builder, fetchDepth *int, sparsePatterns []string, submodules string, lfs bool) { + if fetchDepth != nil { + fmt.Fprintf(sb, " fetch-depth: %d\n", *fetchDepth) + } + if len(sparsePatterns) > 0 { sb.WriteString(" sparse-checkout: |\n") - for _, pattern := range entry.sparsePatterns { - fmt.Fprintf(&sb, " %s\n", strings.TrimSpace(pattern)) + for _, pattern := range sparsePatterns { + fmt.Fprintf(sb, " %s\n", strings.TrimSpace(pattern)) } - // Prevent actions/checkout from adding --filter=blob:none when sparse-checkout - // is specified. Blobless clones require credentials for lazy blob fetches, but - // agent jobs intentionally do not retain git credentials after checkout, making - // offline git operations fail. Using blob:limit=1073741824 (1 GiB) effectively - // fetches all blobs up front on GitHub-hosted repos (GitHub rejects blobs > 100 MiB), - // while keeping the filter non-empty so actions/checkout won't substitute blob:none. The subsequent repair step then - // clears partial-clone markers entirely. - sb.WriteString(" filter: 'blob:limit=1073741824'\n") } - if entry.submodules != "" { - fmt.Fprintf(&sb, " submodules: %s\n", entry.submodules) + if submodules != "" { + fmt.Fprintf(sb, " submodules: %s\n", submodules) } - if entry.lfs { + if lfs { sb.WriteString(" lfs: true\n") } +} +// generateCheckoutStepLines generates YAML step lines for a single non-default checkout. +// The index parameter identifies the checkout's position in the ordered list, used to +// reference the correct app token minting step when app authentication is configured. +// When keepCredentialsForPush is true (safe_outputs job), credentials are retained +// (persist-credentials: true) and the post-checkout cleanup step is suppressed so a later +// git fetch/push can authenticate. +func generateCheckoutStepLines(entry *resolvedCheckout, index int, keepCredentialsForPush bool, pushToken string, getActionPin func(string) string) []string { + checkoutManagerLog.Printf("Generating checkout step lines: index=%d, repo=%q, path=%q, ref=%q, appAuth=%v", + index, entry.key.repository, entry.key.path, entry.ref, entry.githubApp != nil) + name := "Checkout " + checkoutStepName(entry.key) + var sb strings.Builder + fmt.Fprintf(&sb, " - name: %s\n", name) + fmt.Fprintf(&sb, " uses: %s\n", getActionPin("actions/checkout")) + sb.WriteString(" with:\n") + writeCheckoutPersistCredentials(&sb, keepCredentialsForPush, entry.cleanCreds) + writeCheckoutRepositoryRef(&sb, entry) + if entry.key.path != "" { + fmt.Fprintf(&sb, " path: %s\n", entry.key.path) + } + writeCheckoutToken(&sb, entry, index, keepCredentialsForPush, pushToken) + writeCheckoutFetchDepthAndSparse(&sb, entry.fetchDepth, entry.sparsePatterns) + if len(entry.sparsePatterns) > 0 { + sb.WriteString(" filter: 'blob:limit=1073741824'\n") + } + writeCheckoutSubmodulesAndLFS(&sb, entry.submodules, entry.lfs) steps := []string{sb.String()} if len(entry.sparsePatterns) > 0 { steps = append(steps, generateSparseCheckoutPartialCloneResetStep(entry.key.path)) @@ -592,6 +538,37 @@ func generateCheckoutStepLines(entry *resolvedCheckout, index int, keepCredentia return steps } +func writeCheckoutToken(sb *strings.Builder, entry *resolvedCheckout, index int, keepCredentialsForPush bool, pushToken string) { + effectiveToken := resolveCheckoutTokenExpression(entry, index, false) + if effectiveToken == "" && keepCredentialsForPush && pushToken != "" { + effectiveToken = pushToken + } + if effectiveToken != "" { + fmt.Fprintf(sb, " token: %s\n", effectiveToken) + } +} + +func writeCheckoutFetchDepthAndSparse(sb *strings.Builder, fetchDepth *int, sparsePatterns []string) { + if fetchDepth != nil { + fmt.Fprintf(sb, " fetch-depth: %d\n", *fetchDepth) + } + if len(sparsePatterns) > 0 { + sb.WriteString(" sparse-checkout: |\n") + for _, pattern := range sparsePatterns { + fmt.Fprintf(sb, " %s\n", strings.TrimSpace(pattern)) + } + } +} + +func writeCheckoutSubmodulesAndLFS(sb *strings.Builder, submodules string, lfs bool) { + if submodules != "" { + fmt.Fprintf(sb, " submodules: %s\n", submodules) + } + if lfs { + sb.WriteString(" lfs: true\n") + } +} + func generateCheckoutCredentialsCleanupStep() string { return ` - name: Clean git credentials after checkout continue-on-error: true diff --git a/pkg/workflow/claude_engine.go b/pkg/workflow/claude_engine.go index 47c2db56a2d..be2890caead 100644 --- a/pkg/workflow/claude_engine.go +++ b/pkg/workflow/claude_engine.go @@ -170,63 +170,53 @@ func (e *ClaudeEngine) GetAgentManifestPathPrefixes() []string { func (e *ClaudeEngine) GetExecutionSteps(workflowData *WorkflowData, logFile string) []GitHubActionStep { claudeLog.Printf("Generating execution steps for Claude engine: workflow=%s, firewall=%v", workflowData.Name, isFirewallEnabled(workflowData)) - var steps []GitHubActionStep - - // Build claude CLI arguments based on configuration - var claudeArgs []string toolsWithMountedCLIs := withMountedCLIShellCommandsInRestrictedBash(workflowData) + cliConfig := e.buildClaudeCLIConfig(workflowData, toolsWithMountedCLIs, logFile) + claudeCommand := e.buildClaudeCommand(workflowData, cliConfig) + command := e.wrapClaudeCommand(workflowData, claudeCommand, logFile) + env := e.buildClaudeExecutionEnv(workflowData, cliConfig.modelConfigured) + step := e.buildClaudeExecutionStep(workflowData, command, env, cliConfig.allowedTools) + return []GitHubActionStep{step} +} - // Add print flag for non-interactive mode - claudeArgs = append(claudeArgs, "--print") - - // Disable Chrome integration for security and deterministic execution - claudeArgs = append(claudeArgs, "--no-chrome") - - // Model is always passed via the native ANTHROPIC_MODEL environment variable when configured. - // This avoids embedding the value directly in the shell command (which fails template injection - // validation for GitHub Actions expressions like ${{ inputs.model }}). - // Fallback for unconfigured model uses GH_AW_MODEL_AGENT_CLAUDE with shell expansion. - modelConfigured := workflowData.EngineConfig != nil && workflowData.EngineConfig.Model != "" +type claudeCLIConfig struct { + args []string + allowedTools string + mcpConfigArg string + modelConfigured bool + commandName string + harnessScript string +} - // Add max_turns if specified (in CLI it's max-turns) +func (e *ClaudeEngine) buildClaudeCLIConfig(workflowData *WorkflowData, toolsWithMountedCLIs map[string]any, logFile string) claudeCLIConfig { + config := claudeCLIConfig{ + args: []string{"--print", "--no-chrome"}, + modelConfigured: workflowData.EngineConfig != nil && workflowData.EngineConfig.Model != "", + commandName: "claude", + harnessScript: e.GetHarnessScriptName(), + } if workflowData.EngineConfig != nil && workflowData.EngineConfig.MaxTurns != "" { claudeLog.Printf("Setting max turns: %s", workflowData.EngineConfig.MaxTurns) - claudeArgs = append(claudeArgs, "--max-turns", workflowData.EngineConfig.MaxTurns) + config.args = append(config.args, "--max-turns", workflowData.EngineConfig.MaxTurns) } - - // Add MCP configuration only if there are MCP servers. - // Keep this argument outside shellJoinArgs so ${RUNNER_TEMP} expands at runtime. - mcpConfigArg := "" if HasMCPServers(workflowData) { claudeLog.Print("Adding MCP configuration") - mcpConfigArg = ` --mcp-config "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json"` - } - - // Add allowed tools configuration - // Note: Claude Code CLI v2.0.31 introduced a simpler --tools flag, but we continue to use - // --allowed-tools because it provides fine-grained control needed by gh-aw: - // - Specific bash commands: Bash(git:*), Bash(ls) - // - MCP tool prefixes: mcp__github__issue_read - // - Path-specific tools: Read(/tmp/gh-aw/cache-memory/*) - // The --tools flag only supports basic tool names (e.g., "Bash,Edit,Read") without patterns. - allowedTools := e.computeAllowedClaudeToolsString(toolsWithMountedCLIs, workflowData.SafeOutputs, workflowData.CacheMemoryConfig, workflowData.MCPScripts, workflowData.SandboxConfig) - if allowedTools != "" { - claudeArgs = append(claudeArgs, "--allowed-tools", allowedTools) - } - - // Add debug-file flag to write debug logs directly to file - // This implicitly enables debug mode and provides cleaner, more reliable log capture - // than shell redirection with 2>&1 | tee - claudeArgs = append(claudeArgs, "--debug-file", logFile) - - // Always add verbose flag for enhanced debugging output - claudeArgs = append(claudeArgs, "--verbose") - - // Add permission mode for non-interactive execution. - // - // Default to "acceptEdits" so Claude Code honours --allowed-tools as the effective - // MCP tool boundary. Workflows that explicitly set tools.edit=false default to - // "auto" because they don't rely on acceptEdits write auto-approval behavior. + config.mcpConfigArg = ` --mcp-config "${RUNNER_TEMP}/gh-aw/mcp-config/mcp-servers.json"` + } + config.allowedTools = e.computeAllowedClaudeToolsString(toolsWithMountedCLIs, workflowData.SafeOutputs, workflowData.CacheMemoryConfig, workflowData.MCPScripts, workflowData.SandboxConfig) + if config.allowedTools != "" { + config.args = append(config.args, "--allowed-tools", config.allowedTools) + } + config.args = append(config.args, "--debug-file", logFile, "--verbose") + config.args = appendClaudePermissionModeArgs(config.args, workflowData) + config.args = append(config.args, "--output-format", "stream-json") + config.args = appendClaudeBareAndEngineArgs(config.args, workflowData) + config.commandName = resolveClaudeCommandName(workflowData) + config.harnessScript = resolveClaudeHarnessScript(e, workflowData) + return config +} + +func appendClaudePermissionModeArgs(args []string, workflowData *WorkflowData) []string { permissionMode := "acceptEdits" if isEditToolExplicitlyDisabled(workflowData.Tools) { claudeLog.Print("tools.edit=false detected: using auto permission mode") @@ -236,184 +226,153 @@ func (e *ClaudeEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str permissionMode = workflowData.EngineConfig.PermissionMode claudeLog.Printf("Using engine.permission-mode override: %s", permissionMode) } - claudeArgs = append(claudeArgs, "--permission-mode", permissionMode) - permissionModeValueIndex := len(claudeArgs) - 1 - - // Add output format for structured output - // Use "stream-json" to output JSONL format (newline-delimited JSON objects) - // This format is compatible with the log parser which expects either JSON array or JSONL - claudeArgs = append(claudeArgs, "--output-format", "stream-json") + return append(args, "--permission-mode", permissionMode) +} - // Add --bare when bare mode is enabled to suppress automatic loading of memory - // files (CLAUDE.md, ~/.claude/) and other context injections. +func appendClaudeBareAndEngineArgs(args []string, workflowData *WorkflowData) []string { + permissionModeValueIndex := -1 + for i := 0; i < len(args)-1; i++ { + if args[i] == "--permission-mode" { + permissionModeValueIndex = i + 1 + } + } if workflowData.EngineConfig != nil && workflowData.EngineConfig.Bare { claudeLog.Print("Bare mode enabled: adding --bare") - claudeArgs = append(claudeArgs, "--bare") + args = append(args, "--bare") } - - // Add custom args from engine configuration before the prompt. - // Strip any user-supplied --permission-mode flags so exactly one flag is emitted. if workflowData.EngineConfig != nil && len(workflowData.EngineConfig.Args) > 0 { - // stripClaudePermissionModeArgs returns an empty permission-mode string when no - // override flag is present. engineArgs, permissionModeFromArgs := stripClaudePermissionModeArgs(workflowData.EngineConfig.Args) - if permissionModeFromArgs != "" && workflowData.EngineConfig.PermissionMode == "" { + if permissionModeFromArgs != "" && workflowData.EngineConfig.PermissionMode == "" && permissionModeValueIndex >= 0 { claudeLog.Printf("Using legacy engine.args permission mode override: %s", permissionModeFromArgs) - claudeArgs[permissionModeValueIndex] = permissionModeFromArgs + args[permissionModeValueIndex] = permissionModeFromArgs } - claudeArgs = append(claudeArgs, engineArgs...) + args = append(args, engineArgs...) } + return args +} - // The prompt is always read from prompt.txt, which is assembled by the compiler in the - // activation job. For engines that do not support native agent-file handling (including - // Claude), the compiler prepends the agent file content to prompt.txt so no special - // shell variable juggling is needed here. - - // Determine which command to use - var commandName string +func resolveClaudeCommandName(workflowData *WorkflowData) string { if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { - commandName = workflowData.EngineConfig.Command - claudeLog.Printf("Using custom command: %s", commandName) - } else { - // Use regular claude command - PATH is inherited via --env-all in AWF mode - commandName = "claude" + claudeLog.Printf("Using custom command: %s", workflowData.EngineConfig.Command) + return workflowData.EngineConfig.Command } + return "claude" +} - // Determine harness script to wrap claude execution. - // The built-in harness provides retry logic for transient Anthropic API errors - // (overload, rate limit). A custom engine.harness overrides the built-in one. +func resolveClaudeHarnessScript(e *ClaudeEngine, workflowData *WorkflowData) string { harnessScriptName := e.GetHarnessScriptName() if workflowData.EngineConfig != nil && workflowData.EngineConfig.HarnessScript != "" { harnessScriptName = workflowData.EngineConfig.HarnessScript claudeLog.Printf("Using custom harness script: %s", harnessScriptName) } + return harnessScriptName +} +func (e *ClaudeEngine) buildClaudeCommand(workflowData *WorkflowData, cliConfig claudeCLIConfig) string { var claudeCommand string - if harnessScriptName != "" { - // Harness-wrapped execution: the harness reads --prompt-file and passes its content - // as the last positional arg on the initial run. On --continue retries it omits the - // prompt so Claude Code resumes from its on-disk session state. - // The harness sets cwd=GITHUB_WORKSPACE when spawning the claude process, so no - // shell-level cd prefix is needed. - execPrefix := fmt.Sprintf(`%s %s/%s %s`, nodeRuntimeResolutionCommand, SetupActionDestinationShell, harnessScriptName, commandName) - claudeCommand = fmt.Sprintf("%s %s%s --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt", execPrefix, shellJoinArgs(claudeArgs), mcpConfigArg) + if cliConfig.harnessScript != "" { + execPrefix := fmt.Sprintf(`%s %s/%s %s`, nodeRuntimeResolutionCommand, SetupActionDestinationShell, cliConfig.harnessScript, cliConfig.commandName) + claudeCommand = fmt.Sprintf("%s %s%s --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt", execPrefix, shellJoinArgs(cliConfig.args), cliConfig.mcpConfigArg) } else { - // Without harness: use shell expansion for the prompt (no retry logic). - // Apply workspace prefix here since there is no JS harness to set the cwd. - // - // The prompt command is appended raw after shellJoinArgs because it contains - // shell variable references ("$(cat ...)") that must NOT be escaped — - // single-quoting them would prevent shell expansion at runtime. promptCommand := `"$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"` - claudeCommand = getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + fmt.Sprintf("%s%s %s", shellJoinArgs(append([]string{commandName}, claudeArgs...)), mcpConfigArg, promptCommand) + claudeCommand = getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + fmt.Sprintf("%s%s %s", shellJoinArgs(append([]string{cliConfig.commandName}, cliConfig.args...)), cliConfig.mcpConfigArg, promptCommand) } - - // When model is not configured, use the GH_AW_MODEL_AGENT_CLAUDE fallback env var - // via shell expansion so users can set a default via GitHub Actions variables. - // When model IS configured, ANTHROPIC_MODEL is set in the env block (see below) and the - // Claude CLI reads it natively - no --model flag in the shell command needed. - if !modelConfigured { - isDetectionJob := workflowData.SafeOutputs == nil - var modelEnvVar string - if isDetectionJob { + if !cliConfig.modelConfigured { + modelEnvVar := constants.EnvVarModelAgentClaude + if workflowData.SafeOutputs == nil { modelEnvVar = constants.EnvVarModelDetectionClaude - } else { - modelEnvVar = constants.EnvVarModelAgentClaude } claudeCommand = fmt.Sprintf(`%s${%s:+ --model "$%s"}`, claudeCommand, modelEnvVar, modelEnvVar) } + return claudeCommand +} - // Build the full command based on whether firewall is enabled - var command string +func (e *ClaudeEngine) wrapClaudeCommand(workflowData *WorkflowData, claudeCommand, logFile string) string { if isFirewallEnabled(workflowData) { - // Get allowed domains: prefer the pre-warmed cache on WorkflowData (populated by - // computeAllowedDomainsForSanitization before GetExecutionSteps is called) to avoid - // re-running the expensive map+sort operation. - var allowedDomains string - if workflowData.CachedAllowedDomainsComputed { - allowedDomains = workflowData.CachedAllowedDomainsStr - } else { - allowedDomains = GetAllowedDomainsForEngine(constants.ClaudeEngine, workflowData.NetworkPermissions, workflowData.Tools, workflowData.Runtimes) - } - // Add GHES/custom API target domains to the firewall allow-list when engine.api-target is set - if workflowData.EngineConfig != nil && workflowData.EngineConfig.APITarget != "" { - allowedDomains = mergeAPITargetDomains(allowedDomains, workflowData.EngineConfig.APITarget) - } - - // Build AWF command with all configuration - // AWF v0.15.0+ uses chroot mode by default, providing transparent access to host binaries - // AWF with --enable-chroot and --env-all handles most PATH setup natively: - // - GOROOT, JAVA_HOME, etc. are handled via AWF_HOST_PATH and entrypoint.sh - // However, npm-installed CLIs (like claude) need hostedtoolcache bin directories in PATH. - // We prepend GetNpmBinPathSetup() to the engine command so it runs inside the AWF container. - npmPathSetup := GetNpmBinPathSetup() - claudeCommandWithPath := fmt.Sprintf(`%s && %s`, npmPathSetup, claudeCommand) - if dockerSbxCLIPath := GetDockerSbxNpmCLIPathSetup(workflowData); dockerSbxCLIPath != "" { - claudeCommandWithPath = fmt.Sprintf("%s && %s", dockerSbxCLIPath, claudeCommandWithPath) - } - // Add MCP CLI bin directory to PATH when cli-proxy is enabled. - if mcpCLIPath := GetMCPCLIPathSetup(workflowData); mcpCLIPath != "" { - claudeCommandWithPath = fmt.Sprintf("%s && %s", mcpCLIPath, claudeCommandWithPath) - } - - command = BuildAWFCommand(AWFCommandConfig{ - EngineName: "claude", - EngineCommand: claudeCommandWithPath, // Command with npm PATH setup runs inside AWF - LogFile: logFile, - WorkflowData: workflowData, - UsesTTY: true, // Claude Code CLI requires TTY - AllowedDomains: allowedDomains, - PathSetup: "touch " + AgentStepSummaryPath, // Runs BEFORE AWF on the host - // Exclude every env var whose step-env value is a secret so the agent - // cannot read raw token values via bash tools (env / printenv). - ExcludeEnvVarNames: ComputeAWFExcludeEnvVarNames(workflowData, llmProviderSecretNames(e.ResolveLLMProvider(workflowData))), - }) - } else { - // Run Claude command without AWF wrapper - // Note: Claude Code CLI writes debug logs to --debug-file and JSON output to stdout - // Use tee to capture stdout (stream-json output) to the log file while also displaying on console - // The combined output (debug logs + JSON) will be in the log file for parsing - // PATH is already set correctly by actions/setup-* steps which prepend to PATH - command = fmt.Sprintf(`set -o pipefail + return e.buildAWFClaudeCommand(workflowData, claudeCommand, logFile) + } + return fmt.Sprintf(`set -o pipefail printf '%%s' "$(date +%%s%%3N)" > %s touch %s (umask 177 && touch %s) # Execute Claude Code CLI with prompt from file %s 2>&1 | tee -a %s`, AgentCLIStartMsPath, AgentStepSummaryPath, logFile, claudeCommand, logFile) +} + +func (e *ClaudeEngine) buildAWFClaudeCommand(workflowData *WorkflowData, claudeCommand, logFile string) string { + allowedDomains := allowedClaudeDomains(workflowData) + npmPathSetup := GetNpmBinPathSetup() + claudeCommandWithPath := fmt.Sprintf(`%s && %s`, npmPathSetup, claudeCommand) + if dockerSbxCLIPath := GetDockerSbxNpmCLIPathSetup(workflowData); dockerSbxCLIPath != "" { + claudeCommandWithPath = fmt.Sprintf("%s && %s", dockerSbxCLIPath, claudeCommandWithPath) + } + if mcpCLIPath := GetMCPCLIPathSetup(workflowData); mcpCLIPath != "" { + claudeCommandWithPath = fmt.Sprintf("%s && %s", mcpCLIPath, claudeCommandWithPath) + } + return BuildAWFCommand(AWFCommandConfig{ + EngineName: "claude", + EngineCommand: claudeCommandWithPath, + LogFile: logFile, + WorkflowData: workflowData, + UsesTTY: true, + AllowedDomains: allowedDomains, + PathSetup: "touch " + AgentStepSummaryPath, + ExcludeEnvVarNames: ComputeAWFExcludeEnvVarNames(workflowData, llmProviderSecretNames(e.ResolveLLMProvider(workflowData))), + }) +} + +func allowedClaudeDomains(workflowData *WorkflowData) string { + var allowedDomains string + if workflowData.CachedAllowedDomainsComputed { + allowedDomains = workflowData.CachedAllowedDomainsStr + } else { + allowedDomains = GetAllowedDomainsForEngine(constants.ClaudeEngine, workflowData.NetworkPermissions, workflowData.Tools, workflowData.Runtimes) } + if workflowData.EngineConfig != nil && workflowData.EngineConfig.APITarget != "" { + allowedDomains = mergeAPITargetDomains(allowedDomains, workflowData.EngineConfig.APITarget) + } + return allowedDomains +} - // Build environment variables map +func (e *ClaudeEngine) buildClaudeExecutionEnv(workflowData *WorkflowData, modelConfigured bool) map[string]string { provider := e.ResolveLLMProvider(workflowData) - env := map[string]string{ - "ANTHROPIC_API_KEY": llmProviderSecretExpression(provider, workflowData), - "DISABLE_TELEMETRY": "1", - "DISABLE_ERROR_REPORTING": "1", - "DISABLE_BUG_COMMAND": "1", - // Disable Claude Code's "fast mode" feature. Fast mode requires the - // server-side flagSettings.fastMode feature flag, which is not available - // in Agent SDK contexts (non-interactive --print sessions). Without this, - // Claude Code 2.1.120+ attempts to enable fast mode and fails with - // "Fast mode unavailable: Fast mode is not available in the Agent SDK", - // which crashes the agent mid-session on every API call. - "CLAUDE_CODE_DISABLE_FAST_MODE": "1", - "GH_AW_PROMPT": constants.AwPromptsFile, - // Tag the step as a GitHub AW agentic execution for discoverability by agents - "GITHUB_AW": "true", - // Override GITHUB_STEP_SUMMARY with a path that exists inside the sandbox. - // The runner's original path is unreachable within the AWF isolated filesystem; - // we create this file before the agent starts and append it to the real - // $GITHUB_STEP_SUMMARY after secret redaction. - "GITHUB_STEP_SUMMARY": AgentStepSummaryPath, - "GITHUB_WORKSPACE": "${{ github.workspace }}", - "RUNNER_TEMP": "${{ runner.temp }}", - } - env["GH_AW_LLM_PROVIDER"] = provider + env := baseClaudeExecutionEnv(provider, workflowData) if isFirewallEnabled(workflowData) && provider != LLMProviderAnthropic { env["ANTHROPIC_BASE_URL"] = llmProviderGatewayBaseURL(provider) } injectWorkflowCallNetworkAllowedEnv(env, workflowData) - // Indicate the phase: "agent" for the main run, "detection" for threat detection - // Include the compiler version so agents can identify which gh-aw version generated the workflow + applyClaudePhaseAndVersionEnv(env, workflowData) + applyClaudeMCPAndGitEnv(env, workflowData) + applyClaudeTimeoutEnv(env, workflowData) + applySafeOutputEnvToMap(env, workflowData) + applyTraceContextEnvToMap(env) + applyOptionalEngineToolTimeouts(env, workflowData) + applyEngineMaxTurnsEnv(env, workflowData) + applyEngineHarnessRetryEnv(env, workflowData) + applyClaudeModelEnv(env, workflowData, modelConfigured) + applyEngineCwdEnv(env, workflowData) + applyEngineAndAgentEnv(env, workflowData, claudeLog) + applyMCPScriptsSecretEnv(env, workflowData) + return env +} + +func baseClaudeExecutionEnv(provider string, workflowData *WorkflowData) map[string]string { + return map[string]string{ + "ANTHROPIC_API_KEY": llmProviderSecretExpression(provider, workflowData), + "DISABLE_TELEMETRY": "1", + "DISABLE_ERROR_REPORTING": "1", + "DISABLE_BUG_COMMAND": "1", + "CLAUDE_CODE_DISABLE_FAST_MODE": "1", + "GH_AW_PROMPT": constants.AwPromptsFile, + "GITHUB_AW": "true", + "GITHUB_STEP_SUMMARY": AgentStepSummaryPath, + "GITHUB_WORKSPACE": "${{ github.workspace }}", + "RUNNER_TEMP": "${{ runner.temp }}", + "GH_AW_LLM_PROVIDER": provider, + } +} + +func applyClaudePhaseAndVersionEnv(env map[string]string, workflowData *WorkflowData) { if workflowData.IsDetectionRun { env["GH_AW_PHASE"] = "detection" } else { @@ -424,113 +383,64 @@ func (e *ClaudeEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str } else { env["GH_AW_VERSION"] = "dev" } +} - // Add GH_AW_MCP_CONFIG for MCP server configuration only if there are MCP servers +func applyClaudeMCPAndGitEnv(env map[string]string, workflowData *WorkflowData) { if HasMCPServers(workflowData) { env["GH_AW_MCP_CONFIG"] = constants.McpServersJsonPathExpr } - - // In sandbox (AWF) mode, set git identity environment variables so the first git commit - // succeeds inside the container. AWF's --env-all forwards these to the container, ensuring - // git does not rely on the host-side ~/.gitconfig which is not visible in the sandbox. if isFirewallEnabled(workflowData) { maps.Copy(env, getGitIdentityEnvVars()) } +} - // Set timeout environment variables for Claude Code - // Use tools.startup-timeout if specified, otherwise default to DefaultMCPStartupTimeout - // For expressions, fall back to default (can't compute ms value at compile time) +func applyClaudeTimeoutEnv(env map[string]string, workflowData *WorkflowData) { startupTimeoutMs := int(constants.DefaultMCPStartupTimeout / time.Millisecond) if n := templatableIntValue(&workflowData.ToolsStartupTimeout); n > 0 { - startupTimeoutMs = n * 1000 // convert seconds to milliseconds + startupTimeoutMs = n * 1000 } - - // Use tools.timeout if specified, otherwise default to DefaultToolTimeout - // For expressions, fall back to default (can't compute ms value at compile time) timeoutMs := int(constants.DefaultToolTimeout / time.Millisecond) if n := templatableIntValue(&workflowData.ToolsTimeout); n > 0 { - timeoutMs = n * 1000 // convert seconds to milliseconds + timeoutMs = n * 1000 } - env["MCP_TIMEOUT"] = strconv.Itoa(startupTimeoutMs) env["MCP_TOOL_TIMEOUT"] = strconv.Itoa(timeoutMs) env["BASH_DEFAULT_TIMEOUT_MS"] = strconv.Itoa(timeoutMs) env["BASH_MAX_TIMEOUT_MS"] = strconv.Itoa(timeoutMs) +} - // Add GH_AW_SAFE_OUTPUTS if output is needed - applySafeOutputEnvToMap(env, workflowData) - - // Propagate W3C trace context so engine spans nest under the gh-aw.agent.setup span. - applyTraceContextEnvToMap(env) - - applyOptionalEngineToolTimeouts(env, workflowData) - applyEngineMaxTurnsEnv(env, workflowData) - applyEngineHarnessRetryEnv(env, workflowData) - - // Set the model environment variable. - // When model is configured, use the native ANTHROPIC_MODEL env var - the Claude CLI reads it - // directly, avoiding the need to embed the value in the shell command (which would fail - // template injection validation for GitHub Actions expressions like ${{ inputs.model }}). - // When model is not configured, fall back to GH_AW_MODEL_AGENT/DETECTION_CLAUDE so users - // can set a default via GitHub Actions variables. +func applyClaudeModelEnv(env map[string]string, workflowData *WorkflowData, modelConfigured bool) { if modelConfigured { claudeLog.Printf("Setting %s env var for model: %s", constants.ClaudeCLIModelEnvVar, workflowData.EngineConfig.Model) env[constants.ClaudeCLIModelEnvVar] = workflowData.EngineConfig.Model + return + } + if workflowData.SafeOutputs == nil { + env[constants.EnvVarModelDetectionClaude] = compilerenv.BuildModelOverrideExpressionEmptyFallback(constants.EnvVarModelDetectionClaude, compilerenv.DefaultModelClaude) } else { - // No model configured - use fallback GitHub variable with shell expansion - isDetectionJob := workflowData.SafeOutputs == nil - if isDetectionJob { - env[constants.EnvVarModelDetectionClaude] = compilerenv.BuildModelOverrideExpressionEmptyFallback(constants.EnvVarModelDetectionClaude, compilerenv.DefaultModelClaude) - } else { - env[constants.EnvVarModelAgentClaude] = compilerenv.BuildModelOverrideExpressionEmptyFallback(constants.EnvVarModelAgentClaude, compilerenv.DefaultModelClaude) - } + env[constants.EnvVarModelAgentClaude] = compilerenv.BuildModelOverrideExpressionEmptyFallback(constants.EnvVarModelAgentClaude, compilerenv.DefaultModelClaude) } +} - applyEngineCwdEnv(env, workflowData) - applyEngineAndAgentEnv(env, workflowData, claudeLog) - applyMCPScriptsSecretEnv(env, workflowData) - - // Generate the step for Claude CLI execution - stepName := "Execute Claude Code CLI" - var stepLines []string - - stepLines = append(stepLines, " - name: "+stepName) - stepLines = append(stepLines, " id: agentic_execution") - - // Add allowed tools comment before the run section - // Reuse the already-computed allowedTools string (computed earlier for --allowed-tools flag) - // to avoid redundant allocations from calling computeAllowedClaudeToolsString twice. - allowedToolsComment := e.generateAllowedToolsComment(allowedTools, " ") - if allowedToolsComment != "" { - // Split the comment into lines and add each line +func (e *ClaudeEngine) buildClaudeExecutionStep(workflowData *WorkflowData, command string, env map[string]string, allowedTools string) GitHubActionStep { + stepLines := []string{ + " - name: Execute Claude Code CLI", + " id: agentic_execution", + } + if allowedToolsComment := e.generateAllowedToolsComment(allowedTools, " "); allowedToolsComment != "" { commentLines := strings.Split(strings.TrimSuffix(allowedToolsComment, "\n"), "\n") stepLines = append(stepLines, commentLines...) } - - // Add timeout at step level (GitHub Actions standard) if workflowData.TimeoutMinutes != "" { - // Strip timeout-minutes prefix timeoutValue := strings.TrimPrefix(workflowData.TimeoutMinutes, "timeout-minutes: ") stepLines = append(stepLines, " timeout-minutes: "+timeoutValue) } else { - stepLines = append(stepLines, fmt.Sprintf(" timeout-minutes: %d", int(constants.DefaultAgenticWorkflowTimeout/time.Minute))) // Default timeout for agentic workflows + stepLines = append(stepLines, fmt.Sprintf(" timeout-minutes: %d", int(constants.DefaultAgenticWorkflowTimeout/time.Minute))) } - - // Filter environment variables to only include allowed secrets - // This is a security measure to prevent exposing unnecessary secrets to the AWF container allowedSecrets := e.GetRequiredSecretNames(workflowData) filteredEnv := FilterEnvForSecrets(env, allowedSecrets) - - // Inject GH_TOKEN for CLI proxy (added after filtering since it uses a special - // fallback expression that is always allowed when cli-proxy is enabled) addCliProxyGHTokenToEnv(filteredEnv, workflowData) - - // Format step with command and filtered environment variables using shared helper - stepLines = FormatStepWithCommandAndEnv(stepLines, command, filteredEnv) - - steps = append(steps, GitHubActionStep(stepLines)) - - return steps + return GitHubActionStep(FormatStepWithCommandAndEnv(stepLines, command, filteredEnv)) } // GetLogParserScriptId returns the JavaScript script name for parsing Claude logs diff --git a/pkg/workflow/claude_logs.go b/pkg/workflow/claude_logs.go index 4857346aac1..ef5aa435e53 100644 --- a/pkg/workflow/claude_logs.go +++ b/pkg/workflow/claude_logs.go @@ -146,188 +146,193 @@ func (e *ClaudeEngine) parseClaudeJSONLog(logContent string, verbose bool) LogMe claudeLogsLog.Print("Attempting to parse Claude JSON log") var metrics LogMetrics - // Try to parse the entire log as a JSON array first (old format) - var logEntries []map[string]any - if err := json.Unmarshal([]byte(logContent), &logEntries); err != nil { - // If that fails, try to parse as mixed format (debug logs + JSONL) - claudeLogsLog.Print("JSON array parse failed, trying JSONL format") - if verbose { - fmt.Fprintf(os.Stderr, "Failed to parse Claude log as JSON array, trying JSONL format: %v\n", err) - } + logEntries, ok := parseClaudeLogEntries(logContent, verbose) + if !ok { + return metrics + } - logEntries = []map[string]any{} - lines := strings.Split(logContent, "\n") + toolCallMap := make(map[string]*ToolCallInfo) + var currentSequence []string - for i := 0; i < len(lines); i++ { - line := lines[i] - trimmedLine := strings.TrimSpace(line) - if trimmedLine == "" { - continue // Skip empty lines + for _, entry := range logEntries { + if typeStr, _ := entry["type"].(string); typeStr == "result" { + applyClaudeResultEntryMetrics(entry, &metrics) + if verbose { + fmt.Fprintf(os.Stderr, "Extracted from Claude result payload: tokens=%d, cost=%.4f, turns=%d\n", + metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns) } + break + } else if typeStr == "assistant" { + currentSequence = e.appendClaudeAssistantToolSequence(entry, toolCallMap, currentSequence) + } - // If a line looks like a JSON array (starts with '['), try to parse it as an array - if strings.HasPrefix(trimmedLine, "[") { - buf := trimmedLine - // If the closing bracket is not on the same line, accumulate subsequent lines - if !strings.Contains(trimmedLine, "]") { - j := i + 1 - var sb strings.Builder - for j < len(lines) { - sb.WriteString("\n" + lines[j]) - if strings.Contains(lines[j], "]") { - // Advance outer loop to the line we consumed - i = j - break - } - j++ - } - buf += sb.String() - } - - var arr []map[string]any - if err := json.Unmarshal([]byte(buf), &arr); err == nil { - logEntries = append(logEntries, arr...) - continue - } + if entry["type"] == "user" { + e.parseClaudeUserToolResults(entry, toolCallMap) + } + } - // If parsing as a single-line or multi-line array failed, attempt to extract a JSON array substring - openIdx := strings.Index(buf, "[") - closeIdx := strings.LastIndex(buf, "]") - if openIdx != -1 && closeIdx != -1 && closeIdx > openIdx { - sub := buf[openIdx : closeIdx+1] - var arr2 []map[string]any - if err2 := json.Unmarshal([]byte(sub), &arr2); err2 == nil { - logEntries = append(logEntries, arr2...) - continue - } - } - } + FinalizeToolCallsAndSequence(&metrics, toolCallMap, currentSequence) + claudeLogsLog.Printf("Parsed %d log entries: tokens=%d, cost=$%.4f, turns=%d, tool_types=%d", + len(logEntries), metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns, len(metrics.ToolCalls)) + logClaudeToolSequenceSummary(metrics, verbose) + return metrics +} - // Skip debug log lines that don't start with '{' - if !strings.HasPrefix(trimmedLine, "{") { - continue - } +func parseClaudeLogEntries(logContent string, verbose bool) ([]map[string]any, bool) { + var logEntries []map[string]any + if err := json.Unmarshal([]byte(logContent), &logEntries); err == nil { + return logEntries, true + } else { + claudeLogsLog.Print("JSON array parse failed, trying JSONL format") + if verbose { + fmt.Fprintf(os.Stderr, "Failed to parse Claude log as JSON array, trying JSONL format: %v\n", err) + } + } + logEntries = parseClaudeMixedLogEntries(logContent, verbose) + if len(logEntries) == 0 { + if verbose { + fmt.Fprintf(os.Stderr, "No valid JSON entries found in Claude log\n") + } + return nil, false + } + if verbose { + fmt.Fprintf(os.Stderr, "Extracted %d JSON entries from mixed format Claude log\n", len(logEntries)) + } + return logEntries, true +} - // Try to parse each line as JSON - var jsonEntry map[string]any - if err := json.Unmarshal([]byte(trimmedLine), &jsonEntry); err != nil { - // Skip invalid JSON lines (could be partial debug output) - if verbose { - fmt.Fprintf(os.Stderr, "Skipping invalid JSON line: %s\n", trimmedLine) - } +func parseClaudeMixedLogEntries(logContent string, verbose bool) []map[string]any { + var logEntries []map[string]any + lines := strings.Split(logContent, "\n") + for i := 0; i < len(lines); i++ { + trimmedLine := strings.TrimSpace(lines[i]) + if trimmedLine == "" { + continue + } + if strings.HasPrefix(trimmedLine, "[") { + arr, consumedTo, ok := parseClaudeJSONArrayFromLines(lines, i, trimmedLine) + if ok { + logEntries = append(logEntries, arr...) + i = consumedTo continue } - - logEntries = append(logEntries, jsonEntry) } - - if len(logEntries) == 0 { + if !strings.HasPrefix(trimmedLine, "{") { + continue + } + var jsonEntry map[string]any + if err := json.Unmarshal([]byte(trimmedLine), &jsonEntry); err != nil { if verbose { - fmt.Fprintf(os.Stderr, "No valid JSON entries found in Claude log\n") + fmt.Fprintf(os.Stderr, "Skipping invalid JSON line: %s\n", trimmedLine) } - return metrics - } - - if verbose { - fmt.Fprintf(os.Stderr, "Extracted %d JSON entries from mixed format Claude log\n", len(logEntries)) + continue } + logEntries = append(logEntries, jsonEntry) } + return logEntries +} - // Look for the result entry with type: "result" - toolCallMap := make(map[string]*ToolCallInfo) // Track tool calls across entries - var currentSequence []string // Track tool sequence within current context - - for _, entry := range logEntries { - if entryType, exists := entry["type"]; exists { - if typeStr, ok := entryType.(string); ok && typeStr == "result" { - // Found the result payload, extract cost and token data - if totalCost, exists := entry["total_cost_usd"]; exists { - if cost := typeutil.ConvertToFloat(totalCost); cost > 0 { - metrics.EstimatedCost = cost - } - } - - // Extract usage information with all token types - if usage, exists := entry["usage"]; exists { - if usageMap, ok := usage.(map[string]any); ok { - inputTokens := typeutil.ConvertToInt(usageMap["input_tokens"]) - outputTokens := typeutil.ConvertToInt(usageMap["output_tokens"]) - cacheCreationTokens := typeutil.ConvertToInt(usageMap["cache_creation_input_tokens"]) - cacheReadTokens := typeutil.ConvertToInt(usageMap["cache_read_input_tokens"]) - - totalTokens := inputTokens + outputTokens + cacheCreationTokens + cacheReadTokens - if totalTokens > 0 { - metrics.TokenUsage = totalTokens - } - } - } - - // Extract number of turns - if numTurns, exists := entry["num_turns"]; exists { - if turns := typeutil.ConvertToInt(numTurns); turns > 0 { - metrics.Turns = turns - } - } - - // Note: duration_ms in the result payload is the total elapsed job time, not per-tool timing. - // Claude logs do not provide per-tool execution durations, so MaxDuration is left as 0 - // for tools whose timing cannot be measured individually. Assigning the total job - // duration as a per-tool MaxDuration would be misleading (it produced the bug where - // all bash tools appeared to take as long as the entire job). - - if verbose { - fmt.Fprintf(os.Stderr, "Extracted from Claude result payload: tokens=%d, cost=%.4f, turns=%d\n", - metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns) - } +func parseClaudeJSONArrayFromLines(lines []string, start int, trimmedLine string) ([]map[string]any, int, bool) { + buf := trimmedLine + consumedTo := start + if !strings.Contains(trimmedLine, "]") { + var sb strings.Builder + for j := start + 1; j < len(lines); j++ { + sb.WriteString("\n" + lines[j]) + if strings.Contains(lines[j], "]") { + consumedTo = j break - } else if typeStr == "assistant" { - // Parse tool_use entries for tool call statistics and sequence - if message, exists := entry["message"]; exists { - if messageMap, ok := message.(map[string]any); ok { - if content, exists := messageMap["content"]; exists { - if contentArray, ok := content.([]any); ok { - sequenceInMessage := e.parseToolCallsWithSequence(contentArray, toolCallMap) - if len(sequenceInMessage) > 0 { - currentSequence = append(currentSequence, sequenceInMessage...) - } - } - } - } - } } } + buf += sb.String() + } + if arr, ok := unmarshalClaudeJSONArray(buf); ok { + return arr, consumedTo, true + } + openIdx := strings.Index(buf, "[") + closeIdx := strings.LastIndex(buf, "]") + if openIdx != -1 && closeIdx != -1 && closeIdx > openIdx { + if arr, ok := unmarshalClaudeJSONArray(buf[openIdx : closeIdx+1]); ok { + return arr, consumedTo, true + } + } + return nil, consumedTo, false +} - // Parse tool results from user entries for output sizes - if entry["type"] == "user" { - if message, exists := entry["message"]; exists { - if messageMap, ok := message.(map[string]any); ok { - if content, exists := messageMap["content"]; exists { - if contentArray, ok := content.([]any); ok { - // Sequence return value intentionally discarded; only toolCallMap is needed here. - e.parseToolCallsWithSequence(contentArray, toolCallMap) - } - } - } +func unmarshalClaudeJSONArray(raw string) ([]map[string]any, bool) { + var arr []map[string]any + if err := json.Unmarshal([]byte(raw), &arr); err != nil { + return nil, false + } + return arr, true +} + +func applyClaudeResultEntryMetrics(entry map[string]any, metrics *LogMetrics) { + if totalCost, exists := entry["total_cost_usd"]; exists { + if cost := typeutil.ConvertToFloat(totalCost); cost > 0 { + metrics.EstimatedCost = cost + } + } + if usage, exists := entry["usage"]; exists { + if usageMap, ok := usage.(map[string]any); ok { + inputTokens := typeutil.ConvertToInt(usageMap["input_tokens"]) + outputTokens := typeutil.ConvertToInt(usageMap["output_tokens"]) + cacheCreationTokens := typeutil.ConvertToInt(usageMap["cache_creation_input_tokens"]) + cacheReadTokens := typeutil.ConvertToInt(usageMap["cache_read_input_tokens"]) + if totalTokens := inputTokens + outputTokens + cacheCreationTokens + cacheReadTokens; totalTokens > 0 { + metrics.TokenUsage = totalTokens } } } + if numTurns, exists := entry["num_turns"]; exists { + if turns := typeutil.ConvertToInt(numTurns); turns > 0 { + metrics.Turns = turns + } + } +} - // Finalize tool calls and sequences using shared helper - FinalizeToolCallsAndSequence(&metrics, toolCallMap, currentSequence) +func (e *ClaudeEngine) appendClaudeAssistantToolSequence(entry map[string]any, toolCallMap map[string]*ToolCallInfo, currentSequence []string) []string { + if contentArray, ok := claudeLogContentArray(entry); ok { + if sequenceInMessage := e.parseToolCallsWithSequence(contentArray, toolCallMap); len(sequenceInMessage) > 0 { + currentSequence = append(currentSequence, sequenceInMessage...) + } + } + return currentSequence +} - claudeLogsLog.Printf("Parsed %d log entries: tokens=%d, cost=$%.4f, turns=%d, tool_types=%d", - len(logEntries), metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns, len(metrics.ToolCalls)) +func (e *ClaudeEngine) parseClaudeUserToolResults(entry map[string]any, toolCallMap map[string]*ToolCallInfo) { + if contentArray, ok := claudeLogContentArray(entry); ok { + e.parseToolCallsWithSequence(contentArray, toolCallMap) + } +} - if verbose && len(metrics.ToolSequences) > 0 { - totalTools := 0 - for _, seq := range metrics.ToolSequences { - totalTools += len(seq) - } - fmt.Fprintf(os.Stderr, "Claude parser extracted %d tool sequences with %d total tool calls\n", - len(metrics.ToolSequences), totalTools) +func claudeLogContentArray(entry map[string]any) ([]any, bool) { + message, exists := entry["message"] + if !exists { + return nil, false + } + messageMap, ok := message.(map[string]any) + if !ok { + return nil, false } + content, exists := messageMap["content"] + if !exists { + return nil, false + } + contentArray, ok := content.([]any) + return contentArray, ok +} - return metrics +func logClaudeToolSequenceSummary(metrics LogMetrics, verbose bool) { + if !verbose || len(metrics.ToolSequences) == 0 { + return + } + totalTools := 0 + for _, seq := range metrics.ToolSequences { + totalTools += len(seq) + } + fmt.Fprintf(os.Stderr, "Claude parser extracted %d tool sequences with %d total tool calls\n", + len(metrics.ToolSequences), totalTools) } // parseToolCallsWithSequence extracts tool call information from Claude log content array and returns sequence @@ -347,82 +352,68 @@ func (e *ClaudeEngine) parseToolCallsWithSequence(contentArray []any, toolCallMa switch typeStr { case "tool_use": - nameStr, ok := typeutil.LookupString(contentMap, "name") - if !ok { - continue - } - - // Skip internal tools as per existing JavaScript logic (disabled for tool graph visualization) - // internalTools := []string{ - // "Read", "Write", "Edit", "MultiEdit", "LS", "Grep", "Glob", "TodoWrite", - // } - // if slices.Contains(internalTools, nameStr) { - // continue - // } - - prettifiedName := PrettifyToolName(nameStr) - - // Special handling for bash - each invocation is unique - if nameStr == "Bash" { - if commandStr, ok := typeutil.LookupStringPath(contentMap, "input", "command"); ok { - // Create unique bash entry with command info, avoiding colons for - // filesystem-safe names in downstream summaries/artifacts. - prettifiedName = "bash_" + ShortenCommand(commandStr) - } - // If command is missing or non-string, preserve the default "bash" fallback name. - // This occurs with partial/malformed tool_use payloads and keeps parsing robust. - } - - // Add to sequence - sequence = append(sequence, prettifiedName) - - // Calculate input size from the input field - inputSize := 0 - if input, exists := contentMap["input"]; exists { - inputSize = e.estimateInputSize(input) - } - - // Initialize or update tool call info - if toolInfo, exists := toolCallMap[prettifiedName]; exists { - toolInfo.CallCount++ - if inputSize > toolInfo.MaxInputSize { - toolInfo.MaxInputSize = inputSize - } - } else { - toolCallMap[prettifiedName] = &ToolCallInfo{ - Name: prettifiedName, - CallCount: 1, - MaxInputSize: inputSize, - MaxOutputSize: 0, // Will be updated when we find tool results - MaxDuration: 0, // Will be updated when we find execution timing - } + if prettifiedName, ok := e.parseClaudeToolUse(contentMap, toolCallMap); ok { + sequence = append(sequence, prettifiedName) } case "tool_result": - contentStr, ok := typeutil.LookupString(contentMap, "content") - if !ok { - continue - } + updateClaudeToolResultOutputSizes(contentMap, toolCallMap) + } + } - // Estimate token count (rough approximation: 1 token = ~4 characters) - outputSize := len(contentStr) / 4 + return sequence +} - // tool_use_id confirms this tool_result corresponds to a tool invocation. - // We currently do not map IDs back to a single tool, so we still update all tools. - if _, ok := typeutil.LookupString(contentMap, "tool_use_id"); !ok { - continue - } +func (e *ClaudeEngine) parseClaudeToolUse(contentMap map[string]any, toolCallMap map[string]*ToolCallInfo) (string, bool) { + nameStr, ok := typeutil.LookupString(contentMap, "name") + if !ok { + return "", false + } + prettifiedName := claudePrettifiedToolName(contentMap, nameStr) + inputSize := 0 + if input, exists := contentMap["input"]; exists { + inputSize = e.estimateInputSize(input) + } + if toolInfo, exists := toolCallMap[prettifiedName]; exists { + toolInfo.CallCount++ + if inputSize > toolInfo.MaxInputSize { + toolInfo.MaxInputSize = inputSize + } + } else { + toolCallMap[prettifiedName] = &ToolCallInfo{ + Name: prettifiedName, + CallCount: 1, + MaxInputSize: inputSize, + MaxOutputSize: 0, + MaxDuration: 0, + } + } + return prettifiedName, true +} - // This is simplified - in a full implementation we'd track tool_use_id to tool name mapping - // For now, we'll update the max output size for all tools (conservative estimate) - for _, toolInfo := range toolCallMap { - if outputSize > toolInfo.MaxOutputSize { - toolInfo.MaxOutputSize = outputSize - } - } +func claudePrettifiedToolName(contentMap map[string]any, nameStr string) string { + prettifiedName := PrettifyToolName(nameStr) + if nameStr == "Bash" { + if commandStr, ok := typeutil.LookupStringPath(contentMap, "input", "command"); ok { + prettifiedName = "bash_" + ShortenCommand(commandStr) } } + return prettifiedName +} - return sequence +func updateClaudeToolResultOutputSizes(contentMap map[string]any, toolCallMap map[string]*ToolCallInfo) { + contentStr, ok := typeutil.LookupString(contentMap, "content") + if !ok { + return + } + outputSize := len(contentStr) / 4 + if _, ok := typeutil.LookupString(contentMap, "tool_use_id"); !ok { + return + } + for _, toolInfo := range toolCallMap { + if outputSize > toolInfo.MaxOutputSize { + toolInfo.MaxOutputSize = outputSize + } + } } // estimateInputSize estimates the input size in tokens from a tool input object diff --git a/pkg/workflow/claude_tools.go b/pkg/workflow/claude_tools.go index bdfcc56742c..b28095bc4df 100644 --- a/pkg/workflow/claude_tools.go +++ b/pkg/workflow/claude_tools.go @@ -19,80 +19,78 @@ const defaultClaudeTmpWritePath = "/tmp" // expandNeutralToolsToClaudeTools converts neutral tool names to Claude-specific tool configurations func (e *ClaudeEngine) expandNeutralToolsToClaudeTools(tools map[string]any) map[string]any { claudeToolsLog.Printf("Starting neutral tools expansion: input_tools=%d", len(tools)) - result := make(map[string]any) - neutralToolCount := 0 - // Count neutral tools - for key := range tools { - switch key { - case "bash", "web-fetch", "web-search", "edit", "playwright": - neutralToolCount++ - } + if neutralToolCount := countNeutralClaudeTools(tools); neutralToolCount > 0 { + claudeToolsLog.Printf("Expanding %d neutral tools to Claude-specific tools", neutralToolCount) } - if neutralToolCount > 0 { - claudeToolsLog.Printf("Expanding %d neutral tools to Claude-specific tools", neutralToolCount) + result := copyNonNeutralClaudeTools(tools) + claudeSection := getOrCreateToolMap(result, "claude") + claudeAllowed := getOrCreateToolMap(claudeSection, "allowed") + addNeutralClaudeTools(tools, result, claudeAllowed) + + claudeToolsLog.Printf("Expansion complete: result_tools=%d, claude_allowed=%d", len(result), len(claudeAllowed)) + return result +} + +func countNeutralClaudeTools(tools map[string]any) int { + count := 0 + for key := range tools { + if isNeutralClaudeTool(key) { + count++ + } } + return count +} - // Copy existing tools that are not neutral tools +func copyNonNeutralClaudeTools(tools map[string]any) map[string]any { + result := make(map[string]any) for key, value := range tools { - switch key { - case "bash", "web-fetch", "web-search", "edit", "playwright": - // These are neutral tools that need conversion - skip copying, will be converted below - continue - default: - // Copy MCP servers and other non-neutral tools as-is + if !isNeutralClaudeTool(key) { result[key] = value } } + return result +} - // Create or get existing claude section and allowed tools map - claudeSection := getOrCreateToolMap(result, "claude") - claudeAllowed := getOrCreateToolMap(claudeSection, "allowed") +func isNeutralClaudeTool(key string) bool { + switch key { + case "bash", "web-fetch", "web-search", "edit", "playwright": + return true + default: + return false + } +} - // Convert neutral tools to Claude tools +func addNeutralClaudeTools(tools, result, claudeAllowed map[string]any) { if bashTool, hasBash := tools["bash"]; hasBash { - // bash -> Bash, KillBash, BashOutput if bashCommands, ok := bashTool.([]any); ok { claudeAllowed["Bash"] = bashCommands } else { - claudeAllowed["Bash"] = nil // Allow all bash commands + claudeAllowed["Bash"] = nil } } - if _, hasWebFetch := tools["web-fetch"]; hasWebFetch { - // web-fetch -> WebFetch claudeAllowed["WebFetch"] = nil } - if _, hasWebSearch := tools["web-search"]; hasWebSearch { - // web-search -> WebSearch claudeAllowed["WebSearch"] = nil } + addEditAndPlaywrightClaudeTools(tools, result, claudeAllowed) +} +func addEditAndPlaywrightClaudeTools(tools, result, claudeAllowed map[string]any) { if editTool, hasEdit := tools["edit"]; hasEdit && !isExplicitlyDisabledTool(editTool) { - // edit -> Edit, MultiEdit, NotebookEdit, Write claudeAllowed["Edit"] = nil claudeAllowed["MultiEdit"] = nil claudeAllowed["NotebookEdit"] = nil claudeAllowed["Write"] = nil - - // If edit tool has specific configuration, we could handle it here - // For now, treating it as enabling all edit capabilities - _ = editTool } - - // Handle playwright tool by converting it to an MCP tool configuration if _, hasPlaywright := tools["playwright"]; hasPlaywright { - // Create playwright as an MCP tool with the same tools available as copilot agent - playwrightMCP := map[string]any{ + result["playwright"] = map[string]any{ "allowed": GetPlaywrightTools(), } - result["playwright"] = playwrightMCP } - - claudeToolsLog.Printf("Expansion complete: result_tools=%d, claude_allowed=%d", len(result), len(claudeAllowed)) - return result } func isExplicitlyDisabledTool(tool any) bool { diff --git a/pkg/workflow/codex_engine.go b/pkg/workflow/codex_engine.go index c85586d9e3c..17b978c1306 100644 --- a/pkg/workflow/codex_engine.go +++ b/pkg/workflow/codex_engine.go @@ -141,34 +141,34 @@ func (e *CodexEngine) GetInstallationSteps(workflowData *WorkflowData) []GitHubA // Add AWF installation step if firewall is enabled if isFirewallEnabled(workflowData) { - firewallConfig := getFirewallConfig(workflowData) - agentConfig := getAgentConfig(workflowData) - var awfVersion string - if firewallConfig != nil { - awfVersion = firewallConfig.Version - } - - // gVisor must be installed and registered BEFORE AWF starts the agent container. - if isGVisorRuntime(workflowData) { - steps = append(steps, generateGVisorInstallStep()) - } + steps = append(steps, generateCodexAWFInstallationSteps(workflowData)...) + } - // docker-sbx must be installed, authenticated, and smoke-tested BEFORE AWF. - if isDockerSbxRuntime(workflowData) { - steps = append(steps, generateDockerSbxKVMCheckStep()) - steps = append(steps, generateDockerSbxSecretsCheckStep()) - steps = append(steps, generateDockerSbxInstallStep()) - steps = append(steps, generateDockerSbxAuthAndDaemonStep()) - steps = append(steps, generateDockerSbxPreFlightStep()) - } + return steps +} - // Install AWF binary (or skip if custom command is specified) - awfInstall := generateAWFInstallationStep(awfVersion, agentConfig) - if len(awfInstall) > 0 { - steps = append(steps, awfInstall) - } +func generateCodexAWFInstallationSteps(workflowData *WorkflowData) []GitHubActionStep { + firewallConfig := getFirewallConfig(workflowData) + agentConfig := getAgentConfig(workflowData) + var awfVersion string + if firewallConfig != nil { + awfVersion = firewallConfig.Version } + var steps []GitHubActionStep + if isGVisorRuntime(workflowData) { + steps = append(steps, generateGVisorInstallStep()) + } + if isDockerSbxRuntime(workflowData) { + steps = append(steps, generateDockerSbxKVMCheckStep()) + steps = append(steps, generateDockerSbxSecretsCheckStep()) + steps = append(steps, generateDockerSbxInstallStep()) + steps = append(steps, generateDockerSbxAuthAndDaemonStep()) + steps = append(steps, generateDockerSbxPreFlightStep()) + } + if awfInstall := generateAWFInstallationStep(awfVersion, agentConfig); len(awfInstall) > 0 { + steps = append(steps, awfInstall) + } return steps } @@ -212,247 +212,238 @@ func (e *CodexEngine) GetExecutionSteps(workflowData *WorkflowData, logFile stri var steps []GitHubActionStep - // Codex does not support a native model environment variable, so model selection - // always uses GH_AW_MODEL_AGENT_CODEX or GH_AW_MODEL_DETECTION_CODEX with shell expansion - // via the --model flag. This also correctly handles GitHub Actions expressions like ${{ inputs.model }}. - // Note: Codex also supports config-layer model selection (config key `model`, including `-c model="..."`), - // but `--model` is a direct CLI flag and avoids TOML quoting/parsing edge cases in automation. isDetectionJob := workflowData.SafeOutputs == nil - var modelEnvVar string - if isDetectionJob { - modelEnvVar = constants.EnvVarModelDetectionCodex - } else { - modelEnvVar = constants.EnvVarModelAgentCodex - } + modelEnvVar := codexModelEnvVar(isDetectionJob) modelParam := fmt.Sprintf(`${%s:+ --model "$%s"}`, modelEnvVar, modelEnvVar) - // Build search parameter: disable web search by default, enable only if web-search tool is present. - // Codex enables web search by default, so we must explicitly set web_search="disabled" to disable it. - // The --no-search flag does not exist; use the -c web_search="disabled" config option instead. - // See https://developers.openai.com/codex/cli/features#web-search - // Leading space is intentional: these params are concatenated directly and need their own separator. - webSearchParam := ` -c web_search="disabled"` - if workflowData.ParsedTools != nil && workflowData.ParsedTools.WebSearch != nil { - // Web search is enabled by default in Codex; no extra flag needed. - webSearchParam = "" + codexCommandConfig := e.buildCodexCommandConfig(workflowData, modelParam, firewallEnabled) + command := buildCodexExecutionCommand(workflowData, logFile, firewallEnabled, codexCommandConfig) + + // Get effective GitHub token based on precedence: custom token > default + env := buildCodexExecutionEnv(workflowData) + injectWorkflowCallNetworkAllowedEnv(env, workflowData) + setCodexPhaseAndVersionEnv(env, workflowData) + applySafeOutputEnvToMap(env, workflowData) + applyTraceContextEnvToMap(env) + + if firewallEnabled { + maps.Copy(env, getGitIdentityEnvVars()) } - // Build fetch parameter: enforce AWF default-deny for fetch unless web-fetch tool is present. - // Codex enables fetch by default, so this code explicitly sets fetch="disabled" unless web-fetch is configured. - // Leading space is intentional: these params are concatenated directly and need their own separator. - webFetchParam := ` -c fetch="disabled"` - if workflowData.ParsedTools != nil && workflowData.ParsedTools.WebFetch != nil { - // When web-fetch is configured, omit override so Codex default fetch behavior remains enabled. - webFetchParam = "" + applyOptionalEngineToolTimeouts(env, workflowData) + applyEngineMaxTurnsEnv(env, workflowData) + applyEngineHarnessRetryEnv(env, workflowData) + + // Set the model environment variable. + // Codex has no native model env var, so model selection always goes through + // GH_AW_MODEL_AGENT_CODEX / GH_AW_MODEL_DETECTION_CODEX with shell expansion. + // When model is configured (static or GitHub Actions expression), set the env var directly. + // When not configured, use the GitHub variable fallback so users can set a default. + if modelConfigured { + codexEngineLog.Printf("Setting %s env var for model: %s", modelEnvVar, workflowData.EngineConfig.Model) + env[modelEnvVar] = workflowData.EngineConfig.Model + } else { + env[modelEnvVar] = compilerenv.BuildModelOverrideExpression(modelEnvVar, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) } - // See https://github.com/github/gh-aw/issues/892 - // In AWF mode we bypass Codex approvals/sandboxing because AWF provides the sandbox layer. - // Outside AWF, keep Codex sandboxing enabled and disable approvals for non-interactive execution. - executionPolicyParam := ` --sandbox workspace-write --skip-git-repo-check -c approval_policy="never" ` - if firewallEnabled { - executionPolicyParam = " --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check " + applyEngineCwdEnv(env, workflowData) + applyEngineAndAgentEnv(env, workflowData, codexEngineLog) + applyMCPScriptsSecretEnv(env, workflowData) + + steps = append(steps, e.codexExecutionStep(workflowData, command, env)) + + return steps +} + +type codexCommandConfig struct { + commandName string + harnessScriptName string + codexCommand string + detectionSchemaWriteCmd string +} + +func codexModelEnvVar(isDetectionJob bool) string { + if isDetectionJob { + return constants.EnvVarModelDetectionCodex } + return constants.EnvVarModelAgentCodex +} - // Build custom args parameter if specified in engineConfig - var customArgsParam string - if workflowData.EngineConfig != nil && len(workflowData.EngineConfig.Args) > 0 { - var customArgsParamSb strings.Builder - for _, arg := range workflowData.EngineConfig.Args { - customArgsParamSb.WriteString(arg + " ") - } - customArgsParam += customArgsParamSb.String() - } - - // Build structured output parameter for detection runs. - // Use --output-schema to constrain Codex output to the threat detection JSON schema, - // and -o (--output-last-message) to write the final structured verdict directly to a - // file. The parser (parse_threat_detection_results.cjs) reads detection_result.json - // first, bypassing the noisy log stream that caused false parse_error warnings. - // - // The schema file is written to detectionSchemaFilePath before Codex runs: - // - AWF mode: in PathSetup (runs on host before the AWF container starts) - // - Non-AWF mode: in the command preamble (inline shell command) - // Because /tmp/gh-aw/ is the read-write runtime tree mounted in both the host and - // the AWF container, the schema file is accessible inside the container and the - // result file written inside the container is accessible on the host after exit. - var structuredOutputParam string - var detectionSchemaWriteCmd string - if workflowData.IsDetectionRun { - // --output-schema : constrain model output to the threat detection schema - // -o : write the final structured verdict to a file for direct parsing - structuredOutputParam = fmt.Sprintf(` --output-schema %s -o %s`, detectionSchemaFilePath, detectionResultFilePath) - // Shell command to write the schema file before Codex runs. - // printf '%s' avoids the need to escape the JSON (no single quotes in schema). - detectionSchemaWriteCmd = fmt.Sprintf("mkdir -p /tmp/gh-aw/threat-detection && printf '%%s' '%s' > %s", detectionResponseSchema, detectionSchemaFilePath) - codexEngineLog.Printf("Enabling structured outputs for Codex detection run") - } - - // Build the Codex command - // Determine which command to use - var commandName string +func (e *CodexEngine) buildCodexCommandConfig(workflowData *WorkflowData, modelParam string, firewallEnabled bool) codexCommandConfig { + commandName := codexCommandName(workflowData) + harnessScriptName := e.codexHarnessScriptName(workflowData) + structuredOutputParam, detectionSchemaWriteCmd := codexStructuredOutputParams(workflowData) + codexCommand := buildCodexCommand(workflowData, commandName, harnessScriptName, modelParam, structuredOutputParam, firewallEnabled) + return codexCommandConfig{commandName, harnessScriptName, codexCommand, detectionSchemaWriteCmd} +} + +func codexCommandName(workflowData *WorkflowData) string { if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { - commandName = workflowData.EngineConfig.Command - codexEngineLog.Printf("Using custom command: %s", commandName) - } else { - // Use regular codex command - PATH is inherited via --env-all in AWF mode - commandName = "codex" + codexEngineLog.Printf("Using custom command: %s", workflowData.EngineConfig.Command) + return workflowData.EngineConfig.Command } + return "codex" +} - // Determine harness script to wrap codex execution. - // The built-in harness provides retry logic for transient OpenAI API errors - // (rate limits, server errors). A custom engine.harness overrides the built-in one. +func (e *CodexEngine) codexHarnessScriptName(workflowData *WorkflowData) string { harnessScriptName := e.GetHarnessScriptName() if workflowData.EngineConfig != nil && workflowData.EngineConfig.HarnessScript != "" { harnessScriptName = workflowData.EngineConfig.HarnessScript codexEngineLog.Printf("Using custom harness script: %s", harnessScriptName) } + return harnessScriptName +} - // Build the Codex command. - // The default harness (codex_harness.cjs) wraps execution with retry logic and reads the - // prompt via --prompt-file. The else branch is a defensive fallback for the case where - // harnessScriptName is empty (e.g. a future code path that does not set a harness). - var codexCommand string +func codexStructuredOutputParams(workflowData *WorkflowData) (string, string) { + if !workflowData.IsDetectionRun { + return "", "" + } + structuredOutputParam := fmt.Sprintf(` --output-schema %s -o %s`, detectionSchemaFilePath, detectionResultFilePath) + detectionSchemaWriteCmd := fmt.Sprintf("mkdir -p /tmp/gh-aw/threat-detection && printf '%%s' '%s' > %s", detectionResponseSchema, detectionSchemaFilePath) + codexEngineLog.Printf("Enabling structured outputs for Codex detection run") + return structuredOutputParam, detectionSchemaWriteCmd +} + +func buildCodexCommand(workflowData *WorkflowData, commandName, harnessScriptName, modelParam, structuredOutputParam string, firewallEnabled bool) string { + webSearchParam, webFetchParam := codexWebParams(workflowData) + executionPolicyParam := codexExecutionPolicyParam(firewallEnabled) + customArgsParam := codexCustomArgsParam(workflowData) if harnessScriptName != "" { - // Harness-wrapped execution: the harness reads --prompt-file and passes its content - // as the last positional arg. The harness also provides retry logic. - // The harness sets cwd=GITHUB_WORKSPACE when spawning the codex process, so no - // shell-level cd prefix is needed. execPrefix := fmt.Sprintf(`%s %s/%s %s`, nodeRuntimeResolutionCommand, SetupActionDestinationShell, harnessScriptName, commandName) - codexCommand = fmt.Sprintf("%s exec%s%s%s%s%s%s --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt", + return fmt.Sprintf("%s exec%s%s%s%s%s%s --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt", execPrefix, modelParam, webSearchParam, webFetchParam, executionPolicyParam, structuredOutputParam, customArgsParam) - } else { - // Without harness: use shell expansion for the prompt (no retry logic). - // Apply workspace prefix here since there is no JS harness to set the cwd. - codexCommand = getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + fmt.Sprintf("%s exec%s%s%s%s%s%s \"$INSTRUCTION\"", - commandName, modelParam, webSearchParam, webFetchParam, executionPolicyParam, structuredOutputParam, customArgsParam) } + return getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + fmt.Sprintf("%s exec%s%s%s%s%s%s \"$INSTRUCTION\"", + commandName, modelParam, webSearchParam, webFetchParam, executionPolicyParam, structuredOutputParam, customArgsParam) +} + +func codexWebParams(workflowData *WorkflowData) (string, string) { + webSearchParam := ` -c web_search="disabled"` + if workflowData.ParsedTools != nil && workflowData.ParsedTools.WebSearch != nil { + webSearchParam = "" + } + webFetchParam := ` -c fetch="disabled"` + if workflowData.ParsedTools != nil && workflowData.ParsedTools.WebFetch != nil { + webFetchParam = "" + } + return webSearchParam, webFetchParam +} - // Build the full command with agent file handling and AWF wrapping if enabled - var command string +func codexExecutionPolicyParam(firewallEnabled bool) string { if firewallEnabled { - // Build AWF-wrapped command using helper function - // Get allowed domains: prefer the pre-warmed cache on WorkflowData to avoid - // re-running the expensive map+sort operation. - var allowedDomains string - if workflowData.CachedAllowedDomainsComputed { - allowedDomains = workflowData.CachedAllowedDomainsStr - } else { - allowedDomains = GetAllowedDomainsForEngine(constants.CodexEngine, workflowData.NetworkPermissions, workflowData.Tools, workflowData.Runtimes) - } - // Add GHES/custom API target domains to the firewall allow-list when engine.api-target is set - if workflowData.EngineConfig != nil && workflowData.EngineConfig.APITarget != "" { - allowedDomains = mergeAPITargetDomains(allowedDomains, workflowData.EngineConfig.APITarget) - } + return " --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check " + } + return ` --sandbox workspace-write --skip-git-repo-check -c approval_policy="never" ` +} - // AWF v0.15.0+ with --env-all handles most PATH setup natively (chroot mode is default): - // - GOROOT, JAVA_HOME, etc. are handled via AWF_HOST_PATH and entrypoint.sh - // However, npm-installed CLIs (like codex) need hostedtoolcache bin directories in PATH. - npmPathSetup := GetNpmBinPathSetup() - - // Build the codex command with PATH setup inside the AWF container. - // For engines that do not support native agent-file handling (including Codex), - // the compiler prepends the agent file content to prompt.txt. - // When using the harness, --prompt-file is passed directly; otherwise the prompt - // is read via shell variable expansion. - var codexCommandWithSetup string - if harnessScriptName != "" { - // Harness handles prompt reading via --prompt-file; no INSTRUCTION variable needed. - codexCommandWithSetup = fmt.Sprintf(`%s && %s`, npmPathSetup, codexCommand) - } else { - codexCommandWithSetup = fmt.Sprintf(`%s && INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" && %s`, npmPathSetup, codexCommand) - } - if dockerSbxCLIPath := GetDockerSbxNpmCLIPathSetup(workflowData); dockerSbxCLIPath != "" { - codexCommandWithSetup = fmt.Sprintf("%s && %s", dockerSbxCLIPath, codexCommandWithSetup) - } - // Add MCP CLI bin directory to PATH when cli-proxy is enabled. - if mcpCLIPath := GetMCPCLIPathSetup(workflowData); mcpCLIPath != "" { - codexCommandWithSetup = fmt.Sprintf("%s && %s", mcpCLIPath, codexCommandWithSetup) - } +func codexCustomArgsParam(workflowData *WorkflowData) string { + if workflowData.EngineConfig == nil || len(workflowData.EngineConfig.Args) == 0 { + return "" + } + var sb strings.Builder + for _, arg := range workflowData.EngineConfig.Args { + sb.WriteString(arg + " ") + } + return sb.String() +} - command = BuildAWFCommand(AWFCommandConfig{ - EngineName: "codex", - EngineCommand: codexCommandWithSetup, - LogFile: logFile, - WorkflowData: workflowData, - UsesTTY: false, // Codex is not a TUI, outputs to stdout/stderr - AllowedDomains: allowedDomains, - // Create logs directory and agent step summary file before AWF. - // For detection runs, also write the JSON schema file that --output-schema - // references. PathSetup runs on the host before the AWF container starts; - // /tmp/gh-aw/ is the read-write runtime tree mounted in both environments, - // so the schema file is accessible inside the container. - PathSetup: func() string { - base := "mkdir -p \"$CODEX_HOME/logs\" && touch " + AgentStepSummaryPath - if workflowData.IsDetectionRun { - return base + " && " + detectionSchemaWriteCmd - } - return base - }(), - // Exclude Codex/OpenAI API key env vars from the AWF container. - // AWF's API proxy handles auth, so raw token values should not be - // visible to in-container tools (e.g., env/printenv). - ExcludeEnvVarNames: ComputeAWFExcludeEnvVarNames(workflowData, []string{"CODEX_API_KEY", "OPENAI_API_KEY"}), - }) +func buildCodexExecutionCommand(workflowData *WorkflowData, logFile string, firewallEnabled bool, config codexCommandConfig) string { + if firewallEnabled { + return buildAWFCodexExecutionCommand(workflowData, logFile, config) + } + return buildDirectCodexExecutionCommand(workflowData, logFile, config) +} + +func buildAWFCodexExecutionCommand(workflowData *WorkflowData, logFile string, config codexCommandConfig) string { + return BuildAWFCommand(AWFCommandConfig{ + EngineName: "codex", + EngineCommand: codexCommandWithSandboxPathSetup(workflowData, config.codexCommand, config.harnessScriptName), + LogFile: logFile, + WorkflowData: workflowData, + UsesTTY: false, + AllowedDomains: codexAllowedDomains(workflowData), + PathSetup: codexAWFPathSetup(workflowData, config.detectionSchemaWriteCmd), + ExcludeEnvVarNames: ComputeAWFExcludeEnvVarNames(workflowData, []string{"CODEX_API_KEY", "OPENAI_API_KEY"}), + }) +} + +func codexAllowedDomains(workflowData *WorkflowData) string { + var allowedDomains string + if workflowData.CachedAllowedDomainsComputed { + allowedDomains = workflowData.CachedAllowedDomainsStr } else { - // Build the command without AWF wrapping. - // For engines that do not support native agent-file handling (including Codex), - // the compiler prepends the agent file content to prompt.txt so no special - // shell variable juggling is needed here. - - // Optionally prefix the detection schema write command for detection runs. - // Keep it chained with "&&" so a schema write failure stops before codex runs. - schemaWritePrefix := "" - if workflowData.IsDetectionRun { - schemaWritePrefix = detectionSchemaWriteCmd + " && " - } + allowedDomains = GetAllowedDomainsForEngine(constants.CodexEngine, workflowData.NetworkPermissions, workflowData.Tools, workflowData.Runtimes) + } + if workflowData.EngineConfig != nil && workflowData.EngineConfig.APITarget != "" { + allowedDomains = mergeAPITargetDomains(allowedDomains, workflowData.EngineConfig.APITarget) + } + return allowedDomains +} - if harnessScriptName != "" { - // Harness handles prompt reading via --prompt-file; no INSTRUCTION variable needed. - command = fmt.Sprintf(`set -o pipefail +func codexCommandWithSandboxPathSetup(workflowData *WorkflowData, codexCommand, harnessScriptName string) string { + var commandWithSetup string + if harnessScriptName != "" { + commandWithSetup = fmt.Sprintf(`%s && %s`, GetNpmBinPathSetup(), codexCommand) + } else { + commandWithSetup = fmt.Sprintf(`%s && INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" && %s`, GetNpmBinPathSetup(), codexCommand) + } + if dockerSbxCLIPath := GetDockerSbxNpmCLIPathSetup(workflowData); dockerSbxCLIPath != "" { + commandWithSetup = fmt.Sprintf("%s && %s", dockerSbxCLIPath, commandWithSetup) + } + if mcpCLIPath := GetMCPCLIPathSetup(workflowData); mcpCLIPath != "" { + commandWithSetup = fmt.Sprintf("%s && %s", mcpCLIPath, commandWithSetup) + } + return commandWithSetup +} + +func codexAWFPathSetup(workflowData *WorkflowData, detectionSchemaWriteCmd string) string { + base := "mkdir -p \"$CODEX_HOME/logs\" && touch " + AgentStepSummaryPath + if workflowData.IsDetectionRun { + return base + " && " + detectionSchemaWriteCmd + } + return base +} + +func buildDirectCodexExecutionCommand(workflowData *WorkflowData, logFile string, config codexCommandConfig) string { + schemaWritePrefix := "" + if workflowData.IsDetectionRun { + schemaWritePrefix = config.detectionSchemaWriteCmd + " && " + } + if config.harnessScriptName != "" { + return fmt.Sprintf(`set -o pipefail printf '%%s' "$(date +%%s%%3N)" > %s touch %s (umask 177 && touch %s) mkdir -p "$CODEX_HOME/logs" -%s%s 2>&1 | tee %s`, AgentCLIStartMsPath, AgentStepSummaryPath, logFile, schemaWritePrefix, codexCommand, logFile) - } else { - command = fmt.Sprintf(`set -o pipefail +%s%s 2>&1 | tee %s`, AgentCLIStartMsPath, AgentStepSummaryPath, logFile, schemaWritePrefix, config.codexCommand, logFile) + } + return fmt.Sprintf(`set -o pipefail printf '%%s' "$(date +%%s%%3N)" > %s touch %s (umask 177 && touch %s) INSTRUCTION="$(cat "$GH_AW_PROMPT")" mkdir -p "$CODEX_HOME/logs" -%s%s 2>&1 | tee %s`, AgentCLIStartMsPath, AgentStepSummaryPath, logFile, schemaWritePrefix, codexCommand, logFile) - } - } +%s%s 2>&1 | tee %s`, AgentCLIStartMsPath, AgentStepSummaryPath, logFile, schemaWritePrefix, config.codexCommand, logFile) +} - // Get effective GitHub token based on precedence: custom token > default +func buildCodexExecutionEnv(workflowData *WorkflowData) map[string]string { effectiveGitHubToken := getEffectiveGitHubToken("") - - env := map[string]string{ - "CODEX_API_KEY": "${{ secrets.CODEX_API_KEY || secrets.OPENAI_API_KEY }}", - // Override GITHUB_STEP_SUMMARY with a path that exists inside the sandbox. - // The runner's original path is unreachable within the AWF isolated filesystem; - // we create this file before the agent starts and append it to the real - // $GITHUB_STEP_SUMMARY after secret redaction. - "GITHUB_STEP_SUMMARY": AgentStepSummaryPath, - "GH_AW_PROMPT": constants.AwPromptsFile, - // Tag the step as a GitHub AW agentic execution for discoverability by agents - "GITHUB_AW": "true", - "RUNNER_TEMP": "${{ runner.temp }}", - "GH_AW_MCP_CONFIG": constants.CodexMcpConfigTomlPath, - // Keep Codex runtime state in /tmp/gh-aw because ${RUNNER_TEMP}/gh-aw is - // mounted read-only inside the AWF chroot sandbox. - "CODEX_HOME": constants.TmpMcpConfigDir, - // Enable verbose RUST_LOG only in debug mode (runner.debug == 1); default to warn to avoid noisy output. + return map[string]string{ + "CODEX_API_KEY": "${{ secrets.CODEX_API_KEY || secrets.OPENAI_API_KEY }}", + "GITHUB_STEP_SUMMARY": AgentStepSummaryPath, + "GH_AW_PROMPT": constants.AwPromptsFile, + "GITHUB_AW": "true", + "RUNNER_TEMP": "${{ runner.temp }}", + "GH_AW_MCP_CONFIG": constants.CodexMcpConfigTomlPath, + "CODEX_HOME": constants.TmpMcpConfigDir, "RUST_LOG": "${{ runner.debug == 1 && 'trace,hyper_util=info,mio=info,reqwest=info,os_info=info,codex_otel=warn,codex_core=debug,ocodex_exec=debug' || 'warn' }}", "GH_AW_GITHUB_TOKEN": effectiveGitHubToken, - "GITHUB_PERSONAL_ACCESS_TOKEN": effectiveGitHubToken, // Used by GitHub MCP server via env_vars - "OPENAI_API_KEY": "${{ secrets.CODEX_API_KEY || secrets.OPENAI_API_KEY }}", // Fallback for CODEX_API_KEY + "GITHUB_PERSONAL_ACCESS_TOKEN": effectiveGitHubToken, + "OPENAI_API_KEY": "${{ secrets.CODEX_API_KEY || secrets.OPENAI_API_KEY }}", } - injectWorkflowCallNetworkAllowedEnv(env, workflowData) - // Indicate the phase: "agent" for the main run, "detection" for threat detection - // Include the compiler version so agents can identify which gh-aw version generated the workflow +} + +func setCodexPhaseAndVersionEnv(env map[string]string, workflowData *WorkflowData) { if workflowData.IsDetectionRun { env["GH_AW_PHASE"] = "detection" } else { @@ -463,62 +454,16 @@ mkdir -p "$CODEX_HOME/logs" } else { env["GH_AW_VERSION"] = "dev" } +} - // Add GH_AW_SAFE_OUTPUTS if output is needed - applySafeOutputEnvToMap(env, workflowData) - - // Propagate W3C trace context so engine spans nest under the gh-aw.agent.setup span. - applyTraceContextEnvToMap(env) - - // In sandbox (AWF) mode, set git identity environment variables so the first git commit - // succeeds inside the container. AWF's --env-all forwards these to the container, ensuring - // git does not rely on the host-side ~/.gitconfig which is not visible in the sandbox. - if firewallEnabled { - maps.Copy(env, getGitIdentityEnvVars()) +func (e *CodexEngine) codexExecutionStep(workflowData *WorkflowData, command string, env map[string]string) GitHubActionStep { + stepLines := []string{ + " - name: Execute Codex CLI", + " id: agentic_execution", } - - applyOptionalEngineToolTimeouts(env, workflowData) - applyEngineMaxTurnsEnv(env, workflowData) - applyEngineHarnessRetryEnv(env, workflowData) - - // Set the model environment variable. - // Codex has no native model env var, so model selection always goes through - // GH_AW_MODEL_AGENT_CODEX / GH_AW_MODEL_DETECTION_CODEX with shell expansion. - // When model is configured (static or GitHub Actions expression), set the env var directly. - // When not configured, use the GitHub variable fallback so users can set a default. - if modelConfigured { - codexEngineLog.Printf("Setting %s env var for model: %s", modelEnvVar, workflowData.EngineConfig.Model) - env[modelEnvVar] = workflowData.EngineConfig.Model - } else { - env[modelEnvVar] = compilerenv.BuildModelOverrideExpression(modelEnvVar, compilerenv.DefaultModelCodex, constants.CodexDefaultModel) - } - - applyEngineCwdEnv(env, workflowData) - applyEngineAndAgentEnv(env, workflowData, codexEngineLog) - applyMCPScriptsSecretEnv(env, workflowData) - - // Generate the step for Codex execution - stepName := "Execute Codex CLI" - var stepLines []string - - stepLines = append(stepLines, " - name: "+stepName) - stepLines = append(stepLines, " id: agentic_execution") - - // Filter environment variables to only include allowed secrets - // This is a security measure to prevent exposing unnecessary secrets to the AWF container - allowedSecrets := e.GetRequiredSecretNames(workflowData) - filteredEnv := FilterEnvForSecrets(env, allowedSecrets) - - // Inject GH_TOKEN for CLI proxy (added after filtering since it uses a special - // fallback expression that is always allowed when cli-proxy is enabled) + filteredEnv := FilterEnvForSecrets(env, e.GetRequiredSecretNames(workflowData)) addCliProxyGHTokenToEnv(filteredEnv, workflowData) - - // Format step with command and filtered environment variables using shared helper - stepLines = FormatStepWithCommandAndEnv(stepLines, command, filteredEnv) - - steps = append(steps, GitHubActionStep(stepLines)) - - return steps + return GitHubActionStep(FormatStepWithCommandAndEnv(stepLines, command, filteredEnv)) } // GetSquidLogsSteps returns the steps for uploading and parsing Squid logs (after secret redaction) @@ -537,7 +482,17 @@ func (e *CodexEngine) expandNeutralToolsToCodexTools(toolsConfig *ToolsConfig) * } } - // Create a copy of the tools config + result := cloneCodexToolsConfig(toolsConfig) + + // Handle playwright tool by converting it to an MCP tool configuration with copilot agent tools + if toolsConfig.Playwright != nil { + applyCodexPlaywrightConfig(result, toolsConfig.Playwright) + } + + return result +} + +func cloneCodexToolsConfig(toolsConfig *ToolsConfig) *ToolsConfig { result := &ToolsConfig{ GitHub: toolsConfig.GitHub, Bash: toolsConfig.Bash, @@ -552,47 +507,30 @@ func (e *CodexEngine) expandNeutralToolsToCodexTools(toolsConfig *ToolsConfig) * Custom: make(map[string]MCPServerConfig), raw: make(map[string]any), } - - // Copy custom tools maps.Copy(result.Custom, toolsConfig.Custom) - - // Copy raw map maps.Copy(result.raw, toolsConfig.raw) + return result +} - // Handle playwright tool by converting it to an MCP tool configuration with copilot agent tools - if toolsConfig.Playwright != nil { - // Create an updated Playwright config preserving all fields including Mode - playwrightConfig := &PlaywrightToolConfig{ - Version: toolsConfig.Playwright.Version, - Args: toolsConfig.Playwright.Args, - Mode: toolsConfig.Playwright.Mode, - } - - result.Playwright = playwrightConfig - - // In CLI mode, playwright is not an MCP server — remove from raw map and skip MCP config entry. - // result.raw is populated by maps.Copy(result.raw, toolsConfig.raw) earlier in this function, - // so delete is safe regardless of whether the key was originally present. - if playwrightConfig.IsCLIMode() { - delete(result.raw, "playwright") - } else { - // Also update the Custom map entry for playwright with allowed tools list - playwrightMCP := map[string]any{ - "allowed": GetPlaywrightTools(), - } - if playwrightConfig.Version != "" { - playwrightMCP["version"] = playwrightConfig.Version - } - if len(playwrightConfig.Args) > 0 { - playwrightMCP["args"] = playwrightConfig.Args - } - - // Update raw map for backward compatibility - result.raw["playwright"] = playwrightMCP - } +func applyCodexPlaywrightConfig(result *ToolsConfig, source *PlaywrightToolConfig) { + playwrightConfig := &PlaywrightToolConfig{ + Version: source.Version, + Args: source.Args, + Mode: source.Mode, } - - return result + result.Playwright = playwrightConfig + if playwrightConfig.IsCLIMode() { + delete(result.raw, "playwright") + return + } + playwrightMCP := map[string]any{"allowed": GetPlaywrightTools()} + if playwrightConfig.Version != "" { + playwrightMCP["version"] = playwrightConfig.Version + } + if len(playwrightConfig.Args) > 0 { + playwrightMCP["args"] = playwrightConfig.Args + } + result.raw["playwright"] = playwrightMCP } // expandNeutralToolsToCodexToolsFromMap is a backward compatibility wrapper diff --git a/pkg/workflow/codex_logs.go b/pkg/workflow/codex_logs.go index b130bc9fc3e..db36a42460a 100644 --- a/pkg/workflow/codex_logs.go +++ b/pkg/workflow/codex_logs.go @@ -16,184 +16,168 @@ var codexLogsLog = logger.New("workflow:codex_logs") func (e *CodexEngine) ParseLogMetrics(logContent string, verbose bool) LogMetrics { codexLogsLog.Printf("Parsing Codex log metrics: log_size=%d bytes, lines=%d", len(logContent), strings.Count(logContent, "\n")+1) - var metrics LogMetrics - var totalTokenUsage int - lines := strings.Split(logContent, "\n") - turns := 0 - inThinkingSection := false - toolCallMap := make(map[string]*ToolCallInfo) // Track tool calls - var currentSequence []string // Track tool sequence - var lastToolName string // Track most recent tool for output size extraction + state := &codexLogParseState{ + toolCallMap: make(map[string]*ToolCallInfo), + } for i := range lines { - line := lines[i] - - // Skip empty lines - if strings.TrimSpace(line) == "" { - continue - } + e.processCodexLogLine(lines, i, state) + } - // Detect thinking sections as indicators of turns - // Support both old format: "] thinking" and new Rust format: "thinking" (standalone line) - trimmedLine := strings.TrimSpace(line) - if strings.Contains(line, "] thinking") || trimmedLine == "thinking" { - if !inThinkingSection { - turns++ - inThinkingSection = true - // Start of a new thinking section, save previous sequence if any - if len(currentSequence) > 0 { - metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) - currentSequence = []string{} - } - } - } else if strings.Contains(line, "] tool") || strings.Contains(line, "] exec") || strings.Contains(line, "] codex") || - strings.HasPrefix(trimmedLine, "tool ") || strings.HasPrefix(trimmedLine, "exec ") { - inThinkingSection = false - } + // Finalize metrics using shared helper + FinalizeToolMetrics(FinalizeToolMetricsOptions{ + Metrics: &state.metrics, + ToolCallMap: state.toolCallMap, + CurrentSequence: state.currentSequence, + Turns: state.turns, + TokenUsage: state.totalTokenUsage, + }) - // Extract tool calls from Codex logs and add to sequence - if toolName := e.parseCodexToolCallsWithSequence(line, toolCallMap); toolName != "" { - currentSequence = append(currentSequence, toolName) - lastToolName = toolName - } + codexLogsLog.Printf("Parsed Codex metrics: turns=%d, token_usage=%d, tool_calls=%d", + state.metrics.Turns, state.metrics.TokenUsage, len(state.metrics.ToolCalls)) - // Extract output size from success/failure lines followed by JSON blocks - if outputSize := e.extractOutputSizeFromResult(line, lines, i); outputSize > 0 && lastToolName != "" { - if toolInfo, exists := toolCallMap[lastToolName]; exists { - if outputSize > toolInfo.MaxOutputSize { - toolInfo.MaxOutputSize = outputSize - codexLogsLog.Printf("Updated %s MaxOutputSize to %d characters", lastToolName, outputSize) - } - } - } + return state.metrics +} - // Extract Codex-specific token usage (always sum for Codex) - if tokenUsage := e.extractCodexTokenUsage(line); tokenUsage > 0 { - totalTokenUsage += tokenUsage - } +type codexLogParseState struct { + metrics LogMetrics + totalTokenUsage int + turns int + inThinking bool + toolCallMap map[string]*ToolCallInfo + currentSequence []string + lastToolName string +} - // Basic processing - error/warning counting moved to end of function +func (e *CodexEngine) processCodexLogLine(lines []string, index int, state *codexLogParseState) { + line := lines[index] + if strings.TrimSpace(line) == "" { + return } - // Finalize metrics using shared helper - FinalizeToolMetrics(FinalizeToolMetricsOptions{ - Metrics: &metrics, - ToolCallMap: toolCallMap, - CurrentSequence: currentSequence, - Turns: turns, - TokenUsage: totalTokenUsage, - }) + e.updateCodexThinkingState(line, state) + if toolName := e.parseCodexToolCallsWithSequence(line, state.toolCallMap); toolName != "" { + state.currentSequence = append(state.currentSequence, toolName) + state.lastToolName = toolName + } + e.updateCodexOutputSize(line, lines, index, state) + if tokenUsage := e.extractCodexTokenUsage(line); tokenUsage > 0 { + state.totalTokenUsage += tokenUsage + } +} - codexLogsLog.Printf("Parsed Codex metrics: turns=%d, token_usage=%d, tool_calls=%d", - metrics.Turns, metrics.TokenUsage, len(metrics.ToolCalls)) +func (e *CodexEngine) updateCodexThinkingState(line string, state *codexLogParseState) { + trimmedLine := strings.TrimSpace(line) + if strings.Contains(line, "] thinking") || trimmedLine == "thinking" { + if !state.inThinking { + state.turns++ + state.inThinking = true + if len(state.currentSequence) > 0 { + state.metrics.ToolSequences = append(state.metrics.ToolSequences, state.currentSequence) + state.currentSequence = []string{} + } + } + } else if strings.Contains(line, "] tool") || strings.Contains(line, "] exec") || strings.Contains(line, "] codex") || + strings.HasPrefix(trimmedLine, "tool ") || strings.HasPrefix(trimmedLine, "exec ") { + state.inThinking = false + } +} - return metrics +func (e *CodexEngine) updateCodexOutputSize(line string, lines []string, index int, state *codexLogParseState) { + outputSize := e.extractOutputSizeFromResult(line, lines, index) + if outputSize <= 0 || state.lastToolName == "" { + return + } + if toolInfo, exists := state.toolCallMap[state.lastToolName]; exists && outputSize > toolInfo.MaxOutputSize { + toolInfo.MaxOutputSize = outputSize + codexLogsLog.Printf("Updated %s MaxOutputSize to %d characters", state.lastToolName, outputSize) + } } // parseCodexToolCallsWithSequence extracts tool call information from Codex log lines and returns tool name func (e *CodexEngine) parseCodexToolCallsWithSequence(line string, toolCallMap map[string]*ToolCallInfo) string { trimmedLine := strings.TrimSpace(line) - // Parse tool calls: "] tool provider.method(...)" (old format) - // or "tool provider.method(...)" (new Rust format) - var toolName string + if toolName := parseCodexToolName(line, trimmedLine); toolName != "" { + return recordCodexToolCall(toolName, toolCallMap) + } + if execCommand := parseCodexExecCommand(line, trimmedLine); execCommand != "" { + return recordCodexExecCommand(execCommand, toolCallMap) + } + e.updateCodexDuration(line, toolCallMap) + return "" // No tool call found +} - // Try old format first: "] tool provider.method(...)" +func parseCodexToolName(line, trimmedLine string) string { if strings.Contains(line, "] tool ") && strings.Contains(line, "(") { if match := codexToolCallOldFormat.FindStringSubmatch(line); len(match) > 1 { - toolName = strings.TrimSpace(match[1]) + return strings.TrimSpace(match[1]) } } - - // Try new Rust format: "tool provider.method(...)" - if toolName == "" && strings.HasPrefix(trimmedLine, "tool ") && strings.Contains(trimmedLine, "(") { + if strings.HasPrefix(trimmedLine, "tool ") && strings.Contains(trimmedLine, "(") { if match := codexToolCallNewFormat.FindStringSubmatch(trimmedLine); len(match) > 1 { - toolName = strings.TrimSpace(match[1]) + return strings.TrimSpace(match[1]) } } + return "" +} - if toolName != "" { - prettifiedName := PrettifyToolName(toolName) - - // For Codex, format provider.method as provider_method (avoiding colons) - if strings.Contains(toolName, ".") { - parts := strings.Split(toolName, ".") - if len(parts) >= 2 { - provider := parts[0] - method := strings.Join(parts[1:], "_") - prettifiedName = fmt.Sprintf("%s_%s", provider, method) - } - } - - // Initialize or update tool call info - if toolInfo, exists := toolCallMap[prettifiedName]; exists { - toolInfo.CallCount++ - } else { - toolCallMap[prettifiedName] = &ToolCallInfo{ - Name: prettifiedName, - CallCount: 1, - MaxOutputSize: 0, // Will be updated when output is extracted from result lines - MaxDuration: 0, // Will be updated when duration is found - } +func recordCodexToolCall(toolName string, toolCallMap map[string]*ToolCallInfo) string { + prettifiedName := PrettifyToolName(toolName) + if strings.Contains(toolName, ".") { + parts := strings.Split(toolName, ".") + if len(parts) >= 2 { + prettifiedName = fmt.Sprintf("%s_%s", parts[0], strings.Join(parts[1:], "_")) } - - return prettifiedName } + incrementCodexToolInfo(prettifiedName, toolCallMap) + return prettifiedName +} - // Parse exec commands: "] exec command" (old format) - // or "exec command in" (new Rust format) - treat as bash calls - var execCommand string - - // Try old format: "] exec command in" +func parseCodexExecCommand(line, trimmedLine string) string { if strings.Contains(line, "] exec ") { if match := codexExecCommandOldFormat.FindStringSubmatch(line); len(match) > 1 { - execCommand = strings.TrimSpace(match[1]) + return strings.TrimSpace(match[1]) } } - - // Try new Rust format: "exec command in" - if execCommand == "" && strings.HasPrefix(trimmedLine, "exec ") { + if strings.HasPrefix(trimmedLine, "exec ") { if match := codexExecCommandNewFormat.FindStringSubmatch(trimmedLine); len(match) > 1 { - execCommand = strings.TrimSpace(match[1]) + return strings.TrimSpace(match[1]) } } + return "" +} - if execCommand != "" { - // Create unique bash entry with command info, avoiding colons - uniqueBashName := "bash_" + ShortenCommand(execCommand) - - // Initialize or update tool call info - if toolInfo, exists := toolCallMap[uniqueBashName]; exists { - toolInfo.CallCount++ - } else { - toolCallMap[uniqueBashName] = &ToolCallInfo{ - Name: uniqueBashName, - CallCount: 1, - MaxOutputSize: 0, - MaxDuration: 0, // Will be updated when duration is found - } - } +func recordCodexExecCommand(execCommand string, toolCallMap map[string]*ToolCallInfo) string { + uniqueBashName := "bash_" + ShortenCommand(execCommand) + incrementCodexToolInfo(uniqueBashName, toolCallMap) + return uniqueBashName +} - return uniqueBashName +func incrementCodexToolInfo(name string, toolCallMap map[string]*ToolCallInfo) { + if toolInfo, exists := toolCallMap[name]; exists { + toolInfo.CallCount++ + return } + toolCallMap[name] = &ToolCallInfo{ + Name: name, + CallCount: 1, + MaxOutputSize: 0, + MaxDuration: 0, + } +} - // Parse duration from success/failure lines: "] success in 0.2s" or "] failure in 1.5s" - if strings.Contains(line, "success in") || strings.Contains(line, "failure in") || strings.Contains(line, "failed in") { - // Extract duration pattern like "in 0.2s", "in 1.5s" - if match := codexDurationPattern.FindStringSubmatch(line); len(match) > 1 { - if durationSeconds, err := strconv.ParseFloat(match[1], 64); err == nil { - duration := time.Duration(durationSeconds * float64(time.Second)) - - // Find the most recent tool call to associate with this duration - // Since we don't have direct association, we'll update the most recent entry - // This is a limitation of the log format, but it's the best we can do - e.updateMostRecentToolWithDuration(toolCallMap, duration) - } +func (e *CodexEngine) updateCodexDuration(line string, toolCallMap map[string]*ToolCallInfo) { + if !strings.Contains(line, "success in") && !strings.Contains(line, "failure in") && !strings.Contains(line, "failed in") { + return + } + if match := codexDurationPattern.FindStringSubmatch(line); len(match) > 1 { + if durationSeconds, err := strconv.ParseFloat(match[1], 64); err == nil { + duration := time.Duration(durationSeconds * float64(time.Second)) + e.updateMostRecentToolWithDuration(toolCallMap, duration) } } - - return "" // No tool call found } // updateMostRecentToolWithDuration updates the tool with maximum duration diff --git a/pkg/workflow/codex_mcp.go b/pkg/workflow/codex_mcp.go index 81b82924f1c..7f508dd70cf 100644 --- a/pkg/workflow/codex_mcp.go +++ b/pkg/workflow/codex_mcp.go @@ -22,72 +22,66 @@ func (e *CodexEngine) RenderMCPConfig(yaml *strings.Builder, tools map[string]an if codexMCPLog.Enabled() { codexMCPLog.Printf("Rendering MCP config for Codex: mcp_tools=%v, tool_count=%d", mcpTools, len(tools)) } + mcpConfigContent := e.buildCodexMCPConfigContent(tools, mcpTools, workflowData) + writeCodexMCPConfigHeredoc(yaml, mcpConfigContent.String()) + if err := e.writeCodexGatewayJSONConfig(yaml, tools, mcpTools, workflowData); err != nil { + return err + } + e.writeCodexConvertedConfigSync(yaml, tools, mcpTools, workflowData) + return nil +} - // Create unified renderer with Codex-specific options - // Codex uses TOML format without Copilot-specific fields and multi-line args +func (e *CodexEngine) buildCodexMCPConfigContent(tools map[string]any, mcpTools []string, workflowData *WorkflowData) strings.Builder { createRenderer := func(isLast bool) *MCPConfigRendererUnified { return NewMCPConfigRenderer(MCPRendererOptions{ - IncludeCopilotFields: false, // Codex doesn't use "type" and "tools" fields - InlineArgs: false, // Codex uses multi-line args format + IncludeCopilotFields: false, + InlineArgs: false, Format: "toml", IsLast: isLast, ActionMode: GetActionModeFromWorkflowData(workflowData), WriteSinkGuardPolicies: deriveWriteSinkGuardPolicyFromWorkflow(workflowData), }) } - - // Build the heredoc content into a temporary buffer so we can derive the - // delimiter from a SHA-256 hash of the content before writing it to the YAML output. var mcpConfigContent strings.Builder - - // Add history configuration to disable persistence mcpConfigContent.WriteString(" [history]\n") mcpConfigContent.WriteString(" persistence = \"none\"\n") - - // Add shell environment policy to control which environment variables are passed through - // This is a security feature to prevent accidental exposure of secrets e.renderShellEnvironmentPolicy(&mcpConfigContent, tools, mcpTools) - - // Expand neutral tools (like playwright: null) to include the copilot agent tools expandedTools := e.expandNeutralToolsToCodexToolsFromMap(tools) - - // Generate [mcp_servers] section for _, toolName := range mcpTools { - renderer := createRenderer(false) // isLast is always false in TOML format - switch toolName { - case "github": - githubTool, _ := expandedTools["github"].(map[string]any) - renderer.RenderGitHubMCP(&mcpConfigContent, githubTool, workflowData) - case "playwright": - playwrightTool := expandedTools["playwright"] - renderer.RenderPlaywrightMCP(&mcpConfigContent, playwrightTool) - case "agentic-workflows": - renderer.RenderAgenticWorkflowsMCP(&mcpConfigContent) - case "safe-outputs": - // Add safe-outputs MCP server if safe-outputs are configured - hasSafeOutputs := workflowData != nil && workflowData.SafeOutputs != nil && HasSafeOutputsEnabled(workflowData.SafeOutputs) - if hasSafeOutputs { - renderer.RenderSafeOutputsMCP(&mcpConfigContent, workflowData) - } - case "mcp-scripts": - // Add mcp-scripts MCP server if mcp-scripts are configured and feature flag is enabled - hasMCPScripts := workflowData != nil && IsMCPScriptsEnabled(workflowData.MCPScripts) - if hasMCPScripts { - renderer.RenderMCPScriptsMCP(&mcpConfigContent, workflowData.MCPScripts, workflowData) - } - default: - // Handle custom MCP tools using shared helper (with adapter for isLast parameter) - HandleCustomMCPToolInSwitch(&mcpConfigContent, toolName, expandedTools, false, func(yaml *strings.Builder, toolName string, toolConfig map[string]any, isLast bool) error { - return e.renderCodexMCPConfigWithContext(yaml, toolName, toolConfig, workflowData) - }) + e.renderCodexMCPTool(&mcpConfigContent, toolName, expandedTools, workflowData, createRenderer(false)) + } + e.appendCodexCustomConfig(&mcpConfigContent, workflowData) + return mcpConfigContent +} + +func (e *CodexEngine) renderCodexMCPTool(mcpConfigContent *strings.Builder, toolName string, expandedTools map[string]any, workflowData *WorkflowData, renderer *MCPConfigRendererUnified) { + switch toolName { + case "github": + githubTool, _ := expandedTools["github"].(map[string]any) + renderer.RenderGitHubMCP(mcpConfigContent, githubTool, workflowData) + case "playwright": + renderer.RenderPlaywrightMCP(mcpConfigContent, expandedTools["playwright"]) + case "agentic-workflows": + renderer.RenderAgenticWorkflowsMCP(mcpConfigContent) + case "safe-outputs": + if workflowData != nil && workflowData.SafeOutputs != nil && HasSafeOutputsEnabled(workflowData.SafeOutputs) { + renderer.RenderSafeOutputsMCP(mcpConfigContent, workflowData) + } + case "mcp-scripts": + if workflowData != nil && IsMCPScriptsEnabled(workflowData.MCPScripts) { + renderer.RenderMCPScriptsMCP(mcpConfigContent, workflowData.MCPScripts, workflowData) } + default: + HandleCustomMCPToolInSwitch(mcpConfigContent, toolName, expandedTools, false, func(yaml *strings.Builder, toolName string, toolConfig map[string]any, isLast bool) error { + return e.renderCodexMCPConfigWithContext(yaml, toolName, toolConfig, workflowData) + }) } +} - // Append custom config if provided +func (e *CodexEngine) appendCodexCustomConfig(mcpConfigContent *strings.Builder, workflowData *WorkflowData) { if workflowData.EngineConfig != nil && workflowData.EngineConfig.Config != "" { mcpConfigContent.WriteString(" \n") mcpConfigContent.WriteString(" # Custom configuration\n") - // Write the custom config line by line with proper indentation configLines := strings.SplitSeq(workflowData.EngineConfig.Config, "\n") for line := range configLines { if strings.TrimSpace(line) != "" { @@ -97,23 +91,19 @@ func (e *CodexEngine) RenderMCPConfig(yaml *strings.Builder, tools map[string]an } } } +} - // Derive the delimiter from the content so it is stable across builds. - delimiter := GenerateHeredocDelimiterFromContent("MCP_CONFIG", mcpConfigContent.String()) +func writeCodexMCPConfigHeredoc(yaml *strings.Builder, mcpConfigContent string) { + delimiter := GenerateHeredocDelimiterFromContent("MCP_CONFIG", mcpConfigContent) yaml.WriteString(" cat > \"${RUNNER_TEMP}/gh-aw/mcp-config/config.toml\" << " + delimiter + "\n") - yaml.WriteString(mcpConfigContent.String()) - - // End the heredoc for config.toml + yaml.WriteString(mcpConfigContent) yaml.WriteString(" " + delimiter + "\n") +} - // Also generate JSON config for MCP gateway - // Per MCP Gateway Specification v1.0.0 section 4.1, the gateway requires JSON input - // This JSON config is used by the gateway, while the TOML config above is used by Codex +func (e *CodexEngine) writeCodexGatewayJSONConfig(yaml *strings.Builder, tools map[string]any, mcpTools []string, workflowData *WorkflowData) error { yaml.WriteString(" \n") yaml.WriteString(" # Generate JSON config for MCP gateway\n") - - // Gateway uses JSON format without Copilot-specific fields and multi-line args - if err := renderStandardJSONMCPConfig(yaml, renderStandardJSONMCPConfigOptions{ + return renderStandardJSONMCPConfig(yaml, renderStandardJSONMCPConfigOptions{ tools: tools, mcpTools: mcpTools, workflowData: workflowData, @@ -121,20 +111,27 @@ func (e *CodexEngine) RenderMCPConfig(yaml *strings.Builder, tools map[string]an renderCustom: func(yaml *strings.Builder, toolName string, toolConfig map[string]any, isLast bool) error { return e.renderCodexJSONMCPConfigWithContext(yaml, toolName, toolConfig, isLast, workflowData) }, - }); err != nil { - return err - } + }) +} - // start_mcp_gateway.cjs converts the gateway output and writes Codex config to - // ${RUNNER_TEMP}/gh-aw/mcp-config/config.toml. Codex reads config from - // $CODEX_HOME/config.toml, so copy the converted config into writable CODEX_HOME - // and prepend shell policy (converter output does not include this section). +func (e *CodexEngine) writeCodexConvertedConfigSync(yaml *strings.Builder, tools map[string]any, mcpTools []string, workflowData *WorkflowData) { yaml.WriteString(" \n") yaml.WriteString(" # Sync converter output to writable CODEX_HOME for Codex\n") yaml.WriteString(" mkdir -p /tmp/gh-aw/mcp-config\n") + e.writeCodexShellPolicyConfig(yaml, tools, mcpTools, workflowData) + if isFirewallEnabled(workflowData) { + e.renderAppendConvertedConfigWithoutOpenAIProxy(yaml) + } else { + yaml.WriteString(" cat \"${RUNNER_TEMP}/gh-aw/mcp-config/config.toml\" >> \"/tmp/gh-aw/mcp-config/config.toml\"\n") + } + appendCodexEngineCustomConfig(yaml, workflowData) + yaml.WriteString(" chmod 600 \"/tmp/gh-aw/mcp-config/config.toml\"\n") + yaml.WriteString(" mkdir -p \"${CODEX_HOME}\"\n") + yaml.WriteString(" if [ \"/tmp/gh-aw/mcp-config/config.toml\" != \"${CODEX_HOME}/config.toml\" ]; then cp \"/tmp/gh-aw/mcp-config/config.toml\" \"${CODEX_HOME}/config.toml\"; fi\n") + yaml.WriteString(" chmod 600 \"${CODEX_HOME}/config.toml\"\n") +} - // Build the shell-policy heredoc content into a temp buffer so the delimiter - // can be derived from a SHA-256 hash of the content for build stability. +func (e *CodexEngine) writeCodexShellPolicyConfig(yaml *strings.Builder, tools map[string]any, mcpTools []string, workflowData *WorkflowData) { var shellPolicyContent strings.Builder if isFirewallEnabled(workflowData) { e.renderOpenAIProxyProviderToml(&shellPolicyContent, " ") @@ -144,11 +141,9 @@ func (e *CodexEngine) RenderMCPConfig(yaml *strings.Builder, tools map[string]an yaml.WriteString(" cat > \"/tmp/gh-aw/mcp-config/config.toml\" << " + shellPolicyDelimiter + "\n") yaml.WriteString(shellPolicyContent.String()) yaml.WriteString(" " + shellPolicyDelimiter + "\n") - if isFirewallEnabled(workflowData) { - e.renderAppendConvertedConfigWithoutOpenAIProxy(yaml) - } else { - yaml.WriteString(" cat \"${RUNNER_TEMP}/gh-aw/mcp-config/config.toml\" >> \"/tmp/gh-aw/mcp-config/config.toml\"\n") - } +} + +func appendCodexEngineCustomConfig(yaml *strings.Builder, workflowData *WorkflowData) { if workflowData.EngineConfig != nil && strings.TrimSpace(workflowData.EngineConfig.Config) != "" { customConfigDelimiter := GenerateHeredocDelimiterFromContent("CODEX_CUSTOM_CONFIG", workflowData.EngineConfig.Config) yaml.WriteString(" \n") @@ -160,12 +155,6 @@ func (e *CodexEngine) RenderMCPConfig(yaml *strings.Builder, tools map[string]an } yaml.WriteString(" " + customConfigDelimiter + "\n") } - yaml.WriteString(" chmod 600 \"/tmp/gh-aw/mcp-config/config.toml\"\n") - yaml.WriteString(" mkdir -p \"${CODEX_HOME}\"\n") - yaml.WriteString(" if [ \"/tmp/gh-aw/mcp-config/config.toml\" != \"${CODEX_HOME}/config.toml\" ]; then cp \"/tmp/gh-aw/mcp-config/config.toml\" \"${CODEX_HOME}/config.toml\"; fi\n") - yaml.WriteString(" chmod 600 \"${CODEX_HOME}/config.toml\"\n") - - return nil } func (e *CodexEngine) renderOpenAIProxyProviderToml(yaml *strings.Builder, indent string) { diff --git a/pkg/workflow/command.go b/pkg/workflow/command.go index 1a777a9dcf7..5fbf5123eff 100644 --- a/pkg/workflow/command.go +++ b/pkg/workflow/command.go @@ -32,164 +32,92 @@ func buildEventAwareCommandCondition(commandNames []string, commandEvents []stri eventNames := GetCommentEventNames(filteredEvents) commandLog.Printf("Filtered command events: commands=%v, filtered_count=%d", commandNames, len(eventNames)) - // Build command checks for different content sources based on filtered events - var commandChecks []ConditionNode + commandChecks := buildCommandEventChecks(commandNames, eventNames) - // Check which events are enabled - hasIssues := slices.Contains(eventNames, "issues") - hasIssueComment := slices.Contains(eventNames, "issue_comment") - hasPRComment := slices.Contains(eventNames, "pull_request_comment") - hasPR := slices.Contains(eventNames, "pull_request") - hasPRReview := slices.Contains(eventNames, "pull_request_review_comment") - hasDiscussion := slices.Contains(eventNames, "discussion") - hasDiscussionComment := slices.Contains(eventNames, "discussion_comment") - - // Helper function to build OR condition for multiple command checks - // Uses strict matching: command at start of line only - buildMultiCommandCheck := func(bodyAccessor string) ConditionNode { - var commandOrChecks []ConditionNode - for _, commandName := range commandNames { - if isWildcardCommandName(commandName) { - commandPrefix := "/" + strings.TrimSuffix(commandName, "*") - commandOrChecks = append(commandOrChecks, BuildFunctionCall("startsWith", - BuildPropertyAccess(bodyAccessor), - BuildStringLiteral(commandPrefix), - )) - continue - } - - commandText := "/" + commandName - commandWithSpace := fmt.Sprintf("/%s ", commandName) - commandWithNewline := fmt.Sprintf("/%s\n", commandName) - - // Check for exact match (command without arguments) - exactMatch := BuildEquals( - BuildPropertyAccess(bodyAccessor), - BuildStringLiteral(commandText), - ) - - // Check for command with arguments (starts with "/ ") - startsWithMatch := BuildFunctionCall("startsWith", - BuildPropertyAccess(bodyAccessor), - BuildStringLiteral(commandWithSpace), - ) - - // Check for command followed by a newline (e.g. bot comments that append metadata after the command) - startsWithNewlineMatch := BuildFunctionCall("startsWith", - BuildPropertyAccess(bodyAccessor), - BuildStringLiteral(commandWithNewline), - ) - - // Combine: exact match OR starts with space OR starts with newline - commandCheck := &OrNode{ - Left: &OrNode{ - Left: startsWithMatch, - Right: startsWithNewlineMatch, - }, - Right: exactMatch, - } - - commandOrChecks = append(commandOrChecks, commandCheck) - } - // If only one command, return it directly; otherwise combine with OR - if len(commandOrChecks) == 1 { - return commandOrChecks[0] - } - return BuildDisjunction(false, commandOrChecks...) + if len(commandChecks) == 0 { + return nil, fmt.Errorf("no valid comment events specified for commands %v - at least one event must be enabled", commandNames) } + commandLog.Printf("Built %d command check(s) for commands: %v", len(commandChecks), commandNames) + commandCondition := BuildDisjunction(false, commandChecks...) - if hasIssues { - // issues event - check github.event.issue.body only when event is 'issues' - issueBodyCheck := &AndNode{ - Left: BuildEventTypeEquals("issues"), - Right: buildMultiCommandCheck("github.event.issue.body"), - } - commandChecks = append(commandChecks, issueBodyCheck) - } - - if hasIssueComment { - // issue_comment event only on issues (not PRs) - commentBodyCheck := &AndNode{ - Left: BuildEventTypeEquals("issue_comment"), - Right: &AndNode{ - Left: buildMultiCommandCheck("github.event.comment.body"), - Right: BuildEquals( - BuildPropertyAccess("github.event.issue.pull_request"), - BuildNullLiteral(), - ), - }, - } - commandChecks = append(commandChecks, commentBodyCheck) - } - - if hasPRComment { - // pull_request_comment event only on PRs - prCommentBodyCheck := &AndNode{ - Left: BuildEventTypeEquals("issue_comment"), - Right: &AndNode{ - Left: buildMultiCommandCheck("github.event.comment.body"), - Right: BuildNotEquals( - BuildPropertyAccess("github.event.issue.pull_request"), - BuildNullLiteral(), - ), - }, - } - commandChecks = append(commandChecks, prCommentBodyCheck) + if !hasOtherEvents { + commandLog.Print("Using simple command condition (no other events)") + return commandCondition, nil } + commandLog.Print("Using event-aware condition (mixed command and non-command events)") - if hasPRReview { - // pull_request_review_comment uses github.event.comment.body - reviewCommentBodyCheck := &AndNode{ - Left: BuildEventTypeEquals("pull_request_review_comment"), - Right: buildMultiCommandCheck("github.event.comment.body"), - } - commandChecks = append(commandChecks, reviewCommentBodyCheck) - } + return buildMixedEventCommandCondition(commandCondition, eventNames), nil +} - if hasPR { - // pull_request event - check github.event.pull_request.body - prBodyCheck := &AndNode{ - Left: BuildEventTypeEquals("pull_request"), - Right: buildMultiCommandCheck("github.event.pull_request.body"), - } - commandChecks = append(commandChecks, prBodyCheck) +func buildMultiCommandCheck(commandNames []string, bodyAccessor string) ConditionNode { + var commandOrChecks []ConditionNode + for _, commandName := range commandNames { + commandOrChecks = append(commandOrChecks, buildSingleCommandCheck(commandName, bodyAccessor)) } - - if hasDiscussion { - // discussion event - check github.event.discussion.body - discussionBodyCheck := &AndNode{ - Left: BuildEventTypeEquals("discussion"), - Right: buildMultiCommandCheck("github.event.discussion.body"), - } - commandChecks = append(commandChecks, discussionBodyCheck) + if len(commandOrChecks) == 1 { + return commandOrChecks[0] } + return BuildDisjunction(false, commandOrChecks...) +} - if hasDiscussionComment { - // discussion_comment event - check github.event.comment.body - discussionCommentBodyCheck := &AndNode{ - Left: BuildEventTypeEquals("discussion_comment"), - Right: buildMultiCommandCheck("github.event.comment.body"), - } - commandChecks = append(commandChecks, discussionCommentBodyCheck) +func buildSingleCommandCheck(commandName string, bodyAccessor string) ConditionNode { + if isWildcardCommandName(commandName) { + commandPrefix := "/" + strings.TrimSuffix(commandName, "*") + return BuildFunctionCall("startsWith", BuildPropertyAccess(bodyAccessor), BuildStringLiteral(commandPrefix)) } - // Combine all command checks with OR - var commandCondition ConditionNode - if len(commandChecks) == 0 { - // No events enabled - this indicates a configuration error - return nil, fmt.Errorf("no valid comment events specified for commands %v - at least one event must be enabled", commandNames) + commandText := "/" + commandName + startsWithMatch := BuildFunctionCall("startsWith", BuildPropertyAccess(bodyAccessor), BuildStringLiteral(fmt.Sprintf("/%s ", commandName))) + startsWithNewlineMatch := BuildFunctionCall("startsWith", BuildPropertyAccess(bodyAccessor), BuildStringLiteral(fmt.Sprintf("/%s\n", commandName))) + exactMatch := BuildEquals(BuildPropertyAccess(bodyAccessor), BuildStringLiteral(commandText)) + return &OrNode{Left: &OrNode{Left: startsWithMatch, Right: startsWithNewlineMatch}, Right: exactMatch} +} + +func buildCommandEventChecks(commandNames []string, eventNames []string) []ConditionNode { + var commandChecks []ConditionNode + addEventCheck := func(event, accessor string) { + commandChecks = append(commandChecks, &AndNode{Left: BuildEventTypeEquals(event), Right: buildMultiCommandCheck(commandNames, accessor)}) } - commandLog.Printf("Built %d command check(s) for commands: %v", len(commandChecks), commandNames) - commandCondition = BuildDisjunction(false, commandChecks...) + if slices.Contains(eventNames, "issues") { + addEventCheck("issues", "github.event.issue.body") + } + if slices.Contains(eventNames, "issue_comment") { + commandChecks = append(commandChecks, buildIssueCommentCommandCheck(commandNames, false)) + } + if slices.Contains(eventNames, "pull_request_comment") { + commandChecks = append(commandChecks, buildIssueCommentCommandCheck(commandNames, true)) + } + if slices.Contains(eventNames, "pull_request_review_comment") { + addEventCheck("pull_request_review_comment", "github.event.comment.body") + } + if slices.Contains(eventNames, "pull_request") { + addEventCheck("pull_request", "github.event.pull_request.body") + } + if slices.Contains(eventNames, "discussion") { + addEventCheck("discussion", "github.event.discussion.body") + } + if slices.Contains(eventNames, "discussion_comment") { + addEventCheck("discussion_comment", "github.event.comment.body") + } + return commandChecks +} - if !hasOtherEvents { - // If there are no other events, just use the simple command condition - commandLog.Print("Using simple command condition (no other events)") - return commandCondition, nil +func buildIssueCommentCommandCheck(commandNames []string, isPR bool) ConditionNode { + issuePRAccessor := BuildPropertyAccess("github.event.issue.pull_request") + var prCheck ConditionNode = BuildEquals(issuePRAccessor, BuildNullLiteral()) + if isPR { + prCheck = BuildNotEquals(issuePRAccessor, BuildNullLiteral()) } - commandLog.Print("Using event-aware condition (mixed command and non-command events)") + return &AndNode{ + Left: BuildEventTypeEquals("issue_comment"), + Right: &AndNode{ + Left: buildMultiCommandCheck(commandNames, "github.event.comment.body"), + Right: prCheck, + }, + } +} - // Define which events should be checked for command +func buildMixedEventCommandCondition(commandCondition ConditionNode, eventNames []string) ConditionNode { var commentEventTerms []ConditionNode actualEventNames := make(map[string]struct { // Use map to deduplicate }) @@ -213,9 +141,8 @@ func buildEventAwareCommandCondition(commandNames []string, commandEvents []stri // Allow all non-comment events to run nonCommentEvents := &NotNode{Child: commentEventChecks} - // Combine: (comment events && command check) || (non-comment events) return &OrNode{ Left: commentEventCheck, Right: nonCommentEvents, - }, nil + } } diff --git a/pkg/workflow/compiler.go b/pkg/workflow/compiler.go index 198855c0108..01dacffe2aa 100644 --- a/pkg/workflow/compiler.go +++ b/pkg/workflow/compiler.go @@ -138,13 +138,8 @@ func (c *Compiler) generateAndValidateYAML(workflowData *WorkflowData, markdownP // that cannot be bypassed, so we validate it unconditionally workflowLog.Print("Validating expression sizes") if err := c.validateExpressionSizes(yamlContent); err != nil { - // Store error first so we can write invalid YAML before returning formattedErr := formatCompilerError(markdownPath, "error", fmt.Sprintf("expression size validation failed: %v", err), err) - // Write the invalid YAML to a .invalid.yml file for inspection - invalidFile := strings.TrimSuffix(lockFile, ".lock.yml") + ".invalid.yml" - if writeErr := os.WriteFile(invalidFile, []byte(yamlContent), constants.FilePermPublic); writeErr == nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Invalid workflow YAML written to: "+console.ToRelativePath(invalidFile))) - } + writeInvalidWorkflowYAML(lockFile, yamlContent) return "", nil, nil, formattedErr } @@ -158,17 +153,7 @@ func (c *Compiler) generateAndValidateYAML(workflowData *WorkflowData, markdownP // fast-path checks avoid an unnecessary yaml.Unmarshal. needsSchemaCheck := !c.skipValidation - var parsedWorkflow map[string]any - if needsSchemaCheck { - // Schema validation requires parsed YAML; parse once and share with the - // template injection validator below. - workflowLog.Print("Parsing compiled YAML for validation") - if parseErr := yaml.Unmarshal([]byte(yamlContent), &parsedWorkflow); parseErr != nil { - // If parsing fails here the subsequent validators would also fail; keep going - // so we surface the root error from the right validator. - parsedWorkflow = nil - } - } + parsedWorkflow := parseCompiledWorkflowForValidation(yamlContent, needsSchemaCheck) // Validate for template injection vulnerabilities (unsafe expression usage in run: commands). // @@ -183,68 +168,107 @@ func (c *Compiler) generateAndValidateYAML(workflowData *WorkflowData, markdownP // Validate against GitHub Actions schema (unless skipped) if needsSchemaCheck { - workflowLog.Print("Validating workflow against GitHub Actions schema") - var schemaErr error - if parsedWorkflow != nil { - schemaErr = c.validateGitHubActionsSchemaFromParsed(parsedWorkflow) - } else { - schemaErr = c.validateGitHubActionsSchema(yamlContent) - } - if schemaErr != nil { - // Try to point at the exact line of the failing field in the source markdown. - // extractSchemaErrorField unwraps the error chain to find the top-level field - // name (e.g. "timeout-minutes"), which findFrontmatterFieldLine then locates in - // the source frontmatter so the error is IDE-navigable. - fieldLine := 1 - if fieldName := extractSchemaErrorField(schemaErr); fieldName != "" { - frontmatterLines := strings.Split(workflowData.FrontmatterYAML, "\n") - if line := findFrontmatterFieldLine(frontmatterLines, 2, fieldName); line > 0 { - fieldLine = line - } - } - // Store error first so we can write invalid YAML before returning - formattedErr := formatCompilerErrorWithPosition(markdownPath, fieldLine, 1, "error", - fmt.Sprintf("invalid workflow: %v", schemaErr), schemaErr) - // Write the invalid YAML to a .invalid.yml file for inspection - invalidFile := strings.TrimSuffix(lockFile, ".lock.yml") + ".invalid.yml" - if writeErr := os.WriteFile(invalidFile, []byte(yamlContent), constants.FilePermPublic); writeErr == nil { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Invalid workflow YAML written to: "+console.ToRelativePath(invalidFile))) - } - return "", nil, nil, formattedErr + if err := c.validateCompiledWorkflowSchemaAndResources(workflowData, markdownPath, lockFile, yamlContent, parsedWorkflow); err != nil { + return "", nil, nil, err } + } else if c.verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Schema validation available but skipped (use SetSkipValidation(false) to enable)")) + c.IncrementWarningCount() + } - // Validate container images used in MCP configurations - workflowLog.Print("Validating container images") - if err := c.validateContainerImages(workflowData); err != nil { - // Treat container image validation failures as warnings, not errors - // This is because validation may fail due to auth issues locally (e.g., private registries) - fmt.Fprintln(os.Stderr, formatCompilerMessage(markdownPath, "warning", fmt.Sprintf("container image validation failed: %v", err))) - c.IncrementWarningCount() - } + return yamlContent, bodySecrets, bodyActions, nil +} - // Validate runtime packages (npx, uv) - workflowLog.Print("Validating runtime packages") - if err := c.validateRuntimePackages(workflowData); err != nil { - return "", nil, nil, formatCompilerError(markdownPath, "error", fmt.Sprintf("runtime package validation failed: %v", err), err) - } +func writeInvalidWorkflowYAML(lockFile, yamlContent string) { + invalidFile := strings.TrimSuffix(lockFile, ".lock.yml") + ".invalid.yml" + if writeErr := os.WriteFile(invalidFile, []byte(yamlContent), constants.FilePermPublic); writeErr == nil { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Invalid workflow YAML written to: "+console.ToRelativePath(invalidFile))) + } +} - // Validate firewall configuration (log-level enum) - workflowLog.Print("Validating firewall configuration") - if err := c.validateFirewallConfig(workflowData); err != nil { - return "", nil, nil, formatCompilerError(markdownPath, "error", fmt.Sprintf("firewall configuration validation failed: %v", err), err) - } +func parseCompiledWorkflowForValidation(yamlContent string, needsSchemaCheck bool) map[string]any { + if !needsSchemaCheck { + return nil + } + workflowLog.Print("Parsing compiled YAML for validation") + var parsedWorkflow map[string]any + if parseErr := yaml.Unmarshal([]byte(yamlContent), &parsedWorkflow); parseErr != nil { + return nil + } + return parsedWorkflow +} + +func (c *Compiler) validateCompiledWorkflowSchemaAndResources(workflowData *WorkflowData, markdownPath string, lockFile string, yamlContent string, parsedWorkflow map[string]any) error { + if err := c.validateCompiledWorkflowSchema(workflowData, markdownPath, lockFile, yamlContent, parsedWorkflow); err != nil { + return err + } + if err := c.validateCompiledWorkflowRuntimeResources(workflowData, markdownPath); err != nil { + return err + } + return nil +} + +func (c *Compiler) validateCompiledWorkflowSchema(workflowData *WorkflowData, markdownPath string, lockFile string, yamlContent string, parsedWorkflow map[string]any) error { + workflowLog.Print("Validating workflow against GitHub Actions schema") + var schemaErr error + if parsedWorkflow != nil { + schemaErr = c.validateGitHubActionsSchemaFromParsed(parsedWorkflow) + } else { + schemaErr = c.validateGitHubActionsSchema(yamlContent) + } + if schemaErr == nil { + return nil + } + fieldLine := schemaErrorFrontmatterLine(workflowData, schemaErr) + formattedErr := formatCompilerErrorWithPosition(markdownPath, fieldLine, 1, "error", fmt.Sprintf("invalid workflow: %v", schemaErr), schemaErr) + writeInvalidWorkflowYAML(lockFile, yamlContent) + return formattedErr +} - // Validate repository features (discussions, issues) - workflowLog.Print("Validating repository features") - if err := c.validateRepositoryFeatures(workflowData); err != nil { - return "", nil, nil, formatCompilerError(markdownPath, "error", fmt.Sprintf("repository feature validation failed: %v", err), err) +func schemaErrorFrontmatterLine(workflowData *WorkflowData, schemaErr error) int { + fieldLine := 1 + if fieldName := extractSchemaErrorField(schemaErr); fieldName != "" { + frontmatterLines := strings.Split(workflowData.FrontmatterYAML, "\n") + if line := findFrontmatterFieldLine(frontmatterLines, 2, fieldName); line > 0 { + fieldLine = line } - } else if c.verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage("Schema validation available but skipped (use SetSkipValidation(false) to enable)")) + } + return fieldLine +} + +func (c *Compiler) validateCompiledWorkflowRuntimeResources(workflowData *WorkflowData, markdownPath string) error { + workflowLog.Print("Validating container images") + if err := c.validateContainerImages(workflowData); err != nil { + fmt.Fprintln(os.Stderr, formatCompilerMessage(markdownPath, "warning", fmt.Sprintf("container image validation failed: %v", err))) c.IncrementWarningCount() } + if err := c.validateRuntimePackagesWithContext(workflowData, markdownPath); err != nil { + return err + } + if err := c.validateFirewallConfigWithContext(workflowData, markdownPath); err != nil { + return err + } + workflowLog.Print("Validating repository features") + if err := c.validateRepositoryFeatures(workflowData); err != nil { + return formatCompilerError(markdownPath, "error", fmt.Sprintf("repository feature validation failed: %v", err), err) + } + return nil +} - return yamlContent, bodySecrets, bodyActions, nil +func (c *Compiler) validateRuntimePackagesWithContext(workflowData *WorkflowData, markdownPath string) error { + workflowLog.Print("Validating runtime packages") + if err := c.validateRuntimePackages(workflowData); err != nil { + return formatCompilerError(markdownPath, "error", fmt.Sprintf("runtime package validation failed: %v", err), err) + } + return nil +} + +func (c *Compiler) validateFirewallConfigWithContext(workflowData *WorkflowData, markdownPath string) error { + workflowLog.Print("Validating firewall configuration") + if err := c.validateFirewallConfig(workflowData); err != nil { + return formatCompilerError(markdownPath, "error", fmt.Sprintf("firewall configuration validation failed: %v", err), err) + } + return nil } // writeWorkflowOutput writes the compiled workflow to the lock file @@ -419,166 +443,178 @@ func (c *Compiler) CompileWorkflowData(workflowData *WorkflowData, markdownPath workflowLog.Printf("Compilation completed in %v", time.Since(startTime)) }() - // Reset the step order tracker for this compilation - c.stepOrderTracker = NewStepOrderTracker() + c.resetCompilationState() + c.configureGHESArtifactCompatibility() - // Reset schedule friendly formats for this compilation - c.scheduleFriendlyFormats = nil + lockFile := cleanedWorkflowLockFile(markdownPath) + workflowLog.Printf("Starting compilation: %s -> %s", markdownPath, lockFile) - // Reset the artifact manager for this compilation - if c.artifactManager == nil { - c.artifactManager = NewArtifactManager() - } else { - c.artifactManager.Reset() + safeUpdateEnabled := c.effectiveSafeUpdate(workflowData) + oldManifest, oldHasPR, oldHasPRTarget := c.safeUpdateBaselineIfEnabled(lockFile, safeUpdateEnabled) + + if err := c.validateWorkflowDataForCompile(workflowData, markdownPath); err != nil { + return err } - // Enable GHES artifact compatibility from CLI flag or aw.json (CLI flag wins). - // c.ghesCompatFromCLI is set once per compiler instance via SetGHESCompat(). - c.ghesArtifactCompat = c.ghesCompatFromCLI - if !c.ghesArtifactCompat { - // Fall back to aw.json ghes field when CLI flag was not passed. - if repoConfig, err := c.loadRepoConfig(); err == nil && repoConfig != nil { - c.ghesArtifactCompat = repoConfig.GHES - } + workflowLog.Printf("Workflow: %s, Tools: %d", workflowData.Name, len(workflowData.Tools)) + + yamlContent, bodySecrets, bodyActions, err := c.generateYAMLForCompile(workflowData, markdownPath, lockFile) + if err != nil { + return err } - if c.ghesArtifactCompat { - actionPinsLog.Print("GHES compatibility mode enabled: artifact actions continue using latest non-v3 pins") + + if safeUpdateEnabled { + c.enforceSafeUpdateWarning(workflowData, markdownPath, oldManifest, oldHasPR, oldHasPRTarget, bodySecrets, bodyActions) } - // Generate lock file name - lockFile := stringutil.MarkdownToLockFile(markdownPath) + if resolver := c.GetSharedActionResolver(); resolver != nil { + resolver.MarkCompilerGeneratedActionsAsUsed() + } - // Sanitize the lock file path to prevent path traversal attacks - lockFile = filepath.Clean(lockFile) + // Write output + if err := c.writeWorkflowOutput(lockFile, yamlContent, markdownPath); err != nil { + return err + } - workflowLog.Printf("Starting compilation: %s -> %s", markdownPath, lockFile) + return nil +} - // Resolve and cache the baseline manifest only when safe update mode is active. - // This avoids unnecessary git/filesystem reads on compile paths that skip safe update - // enforcement (e.g., --approve or strict: false). - safeUpdateEnabled := c.effectiveSafeUpdate(workflowData) - var oldManifest *GHAWManifest - var oldHasPR bool - var oldHasPRTarget bool - if safeUpdateEnabled { - // Read the existing lock file to extract the previous gh-aw-manifest for safe update - // enforcement. - // - // Priority (highest to lowest): - // 1. Pre-cached manifest supplied by the caller (e.g. MCP server collected at startup - // before any agent interaction, making it tamper-proof without requiring git access). - // 2. Content from the last git commit (HEAD) – prevents a local agent from modifying - // the .lock.yml file on disk to forge an approved manifest. - // 3. Filesystem read – fallback for first-time compilations or non-git environments. - if cached, ok := c.priorManifests[lockFile]; ok { - oldManifest = cached - if committedContent, readErr := c.readLockFileFromHEAD(lockFile); readErr == nil { - oldHasPR, oldHasPRTarget = extractPullRequestEventPresenceFromCompiledWorkflow(committedContent) - } - secretCount := 0 - if cached != nil { - secretCount = len(cached.Secrets) - } - workflowLog.Printf("Using pre-cached gh-aw-manifest for %s: %d secret(s)", lockFile, secretCount) - } else if committedContent, readErr := c.readLockFileFromHEAD(lockFile); readErr == nil { - oldHasPR, oldHasPRTarget = extractPullRequestEventPresenceFromCompiledWorkflow(committedContent) - if m, parseErr := ExtractGHAWManifestFromLockFile(committedContent); parseErr == nil { - oldManifest = m - if oldManifest != nil { - workflowLog.Printf("Loaded committed gh-aw-manifest from HEAD: %d secret(s)", len(oldManifest.Secrets)) - } - } else { - workflowLog.Printf("Failed to parse committed gh-aw-manifest: %v. Safe update enforcement will proceed without baseline comparison (all secrets will be considered new).", parseErr) - } - } else { - workflowLog.Printf("Lock file %s not found in HEAD commit (%v); falling back to filesystem read.", lockFile, readErr) - if existingContent, fsErr := os.ReadFile(lockFile); fsErr == nil { - oldHasPR, oldHasPRTarget = extractPullRequestEventPresenceFromCompiledWorkflow(string(existingContent)) - if m, parseErr := ExtractGHAWManifestFromLockFile(string(existingContent)); parseErr == nil { - oldManifest = m - if oldManifest != nil { - workflowLog.Printf("Loaded gh-aw-manifest from filesystem: %d secret(s)", len(oldManifest.Secrets)) - } - } else { - workflowLog.Printf("Failed to parse filesystem gh-aw-manifest: %v. Safe update enforcement will treat as empty manifest.", parseErr) - } - } else { - // No lock file anywhere — this is a brand-new workflow. Use an empty - // (non-nil) manifest so EnforceSafeUpdate applies enforcement and flags - // any newly introduced secrets or actions for review. - workflowLog.Printf("Lock file %s not found (new workflow). Safe update enforcement will use an empty baseline.", lockFile) - oldManifest = &GHAWManifest{Version: currentGHAWManifestVersion} - } - } - // Keep the first non-nil baseline seen by this compiler instance. - // This intentionally does not overwrite an existing cache entry so repeated - // compiles in the same process continue to compare against the same trusted - // baseline rather than a just-generated local lock file. - // Nil baselines (e.g., legacy lock files without gh-aw-manifest) are not - // cached so future compiles can pick up a newly available manifest. - if oldManifest != nil { - if _, ok := c.priorManifests[lockFile]; !ok { - c.priorManifests[lockFile] = oldManifest - } - } +func cleanedWorkflowLockFile(markdownPath string) string { + return filepath.Clean(stringutil.MarkdownToLockFile(markdownPath)) +} + +func (c *Compiler) safeUpdateBaselineIfEnabled(lockFile string, enabled bool) (*GHAWManifest, bool, bool) { + if !enabled { + return nil, false, false } + return c.safeUpdateBaseline(lockFile) +} - // Validate workflow data +func (c *Compiler) validateWorkflowDataForCompile(workflowData *WorkflowData, markdownPath string) error { if err := c.validateWorkflowData(workflowData, markdownPath); err != nil { - // validateWorkflowData always returns formatCompilerError results; pass through directly. - // If an unformatted error somehow slips through, wrap it with compiler context. if isFormattedCompilerError(err) { return err } return formatCompilerError(markdownPath, "error", "workflow validation: "+err.Error(), err) } + return nil +} - // Note: Markdown content size is now handled by splitting into multiple steps in generatePrompt - workflowLog.Printf("Workflow: %s, Tools: %d", workflowData.Name, len(workflowData.Tools)) - - // Note: compute-text functionality is now inlined directly in the task job - // instead of using a shared action file - - // Generate and validate YAML (also embeds the new gh-aw-manifest in the header). - // Returns the collected body secrets and action refs to avoid duplicate scans for - // safe update enforcement. +func (c *Compiler) generateYAMLForCompile(workflowData *WorkflowData, markdownPath string, lockFile string) (string, []string, []string, error) { yamlContent, bodySecrets, bodyActions, err := c.generateAndValidateYAML(workflowData, markdownPath, lockFile) if err != nil { - // generateAndValidateYAML always returns formatCompilerError results; pass through directly. - // If an unformatted error somehow slips through, wrap it with compiler context. if isFormattedCompilerError(err) { - return err + return "", nil, nil, err } - return formatCompilerError(markdownPath, "error", "YAML generation: "+err.Error(), err) + return "", nil, nil, formatCompilerError(markdownPath, "error", "YAML generation: "+err.Error(), err) } + return yamlContent, bodySecrets, bodyActions, nil +} - // Enforce safe update mode: emit a warning prompt (not a hard error) when unapproved - // secrets or action changes are detected. body* vars contain data collected from the - // workflow body only (not the header) to avoid matching the gh-aw-manifest JSON comment. - // - // Emitting a warning instead of failing allows compilation to succeed so that the lock - // file is written and the agent receives the actionable guidance embedded in the warning. - if safeUpdateEnabled { - currentHasPR, currentHasPRTarget := extractPullRequestEventPresenceFromOnField(workflowData.RawFrontmatter["on"]) - if enforceErr := EnforceSafeUpdate(oldManifest, bodySecrets, bodyActions, workflowData.Redirect, oldHasPR, oldHasPRTarget, currentHasPR, currentHasPRTarget); enforceErr != nil { - warningMsg := buildSafeUpdateWarningPrompt(enforceErr.Error()) - c.AddSafeUpdateWarning(warningMsg) - fmt.Fprintln(os.Stderr, formatCompilerMessage(markdownPath, "warning", enforceErr.Error())) - c.IncrementWarningCount() +func (c *Compiler) resetCompilationState() { + c.stepOrderTracker = NewStepOrderTracker() + c.scheduleFriendlyFormats = nil + if c.artifactManager == nil { + c.artifactManager = NewArtifactManager() + } else { + c.artifactManager.Reset() + } +} + +func (c *Compiler) configureGHESArtifactCompatibility() { + c.ghesArtifactCompat = c.ghesCompatFromCLI + if !c.ghesArtifactCompat { + if repoConfig, err := c.loadRepoConfig(); err == nil && repoConfig != nil { + c.ghesArtifactCompat = repoConfig.GHES } } + if c.ghesArtifactCompat { + actionPinsLog.Print("GHES compatibility mode enabled: artifact actions continue using latest non-v3 pins") + } +} - // Mark compiler-generated actions as used to prevent pruning. - // This handles actions that are hardcoded in code generators (cache.go, - // checkout_step_generator.go, etc.) rather than resolved from markdown workflows. - if resolver := c.GetSharedActionResolver(); resolver != nil { - resolver.MarkCompilerGeneratedActionsAsUsed() +func (c *Compiler) safeUpdateBaseline(lockFile string) (*GHAWManifest, bool, bool) { + if cached, ok := c.priorManifests[lockFile]; ok { + return c.safeUpdateBaselineFromCache(lockFile, cached) + } + if committedContent, readErr := c.readLockFileFromHEAD(lockFile); readErr == nil { + return c.safeUpdateBaselineFromCommitted(lockFile, committedContent) + } else { + return c.safeUpdateBaselineFromFilesystem(lockFile, readErr) } +} - // Write output - if err := c.writeWorkflowOutput(lockFile, yamlContent, markdownPath); err != nil { - return err +func (c *Compiler) safeUpdateBaselineFromCache(lockFile string, cached *GHAWManifest) (*GHAWManifest, bool, bool) { + var oldHasPR, oldHasPRTarget bool + if committedContent, readErr := c.readLockFileFromHEAD(lockFile); readErr == nil { + oldHasPR, oldHasPRTarget = extractPullRequestEventPresenceFromCompiledWorkflow(committedContent) } + secretCount := 0 + if cached != nil { + secretCount = len(cached.Secrets) + } + workflowLog.Printf("Using pre-cached gh-aw-manifest for %s: %d secret(s)", lockFile, secretCount) + return cached, oldHasPR, oldHasPRTarget +} - return nil +func (c *Compiler) safeUpdateBaselineFromCommitted(lockFile string, committedContent string) (*GHAWManifest, bool, bool) { + oldHasPR, oldHasPRTarget := extractPullRequestEventPresenceFromCompiledWorkflow(committedContent) + oldManifest, _ := parseSafeUpdateManifest("committed", committedContent) + c.cacheSafeUpdateBaseline(lockFile, oldManifest) + return oldManifest, oldHasPR, oldHasPRTarget +} + +func (c *Compiler) safeUpdateBaselineFromFilesystem(lockFile string, readErr error) (*GHAWManifest, bool, bool) { + workflowLog.Printf("Lock file %s not found in HEAD commit (%v); falling back to filesystem read.", lockFile, readErr) + if existingContent, fsErr := os.ReadFile(lockFile); fsErr == nil { + oldHasPR, oldHasPRTarget := extractPullRequestEventPresenceFromCompiledWorkflow(string(existingContent)) + oldManifest, _ := parseSafeUpdateManifest("filesystem", string(existingContent)) + c.cacheSafeUpdateBaseline(lockFile, oldManifest) + return oldManifest, oldHasPR, oldHasPRTarget + } + workflowLog.Printf("Lock file %s not found (new workflow). Safe update enforcement will use an empty baseline.", lockFile) + oldManifest := &GHAWManifest{Version: currentGHAWManifestVersion} + c.cacheSafeUpdateBaseline(lockFile, oldManifest) + return oldManifest, false, false +} + +func parseSafeUpdateManifest(source string, content string) (*GHAWManifest, bool) { + manifest, parseErr := ExtractGHAWManifestFromLockFile(content) + if parseErr != nil { + if source == "committed" { + workflowLog.Printf("Failed to parse committed gh-aw-manifest: %v. Safe update enforcement will proceed without baseline comparison (all secrets will be considered new).", parseErr) + } else { + workflowLog.Printf("Failed to parse filesystem gh-aw-manifest: %v. Safe update enforcement will treat as empty manifest.", parseErr) + } + return nil, false + } + if manifest != nil { + if source == "committed" { + workflowLog.Printf("Loaded committed gh-aw-manifest from HEAD: %d secret(s)", len(manifest.Secrets)) + } else { + workflowLog.Printf("Loaded gh-aw-manifest from filesystem: %d secret(s)", len(manifest.Secrets)) + } + } + return manifest, true +} + +func (c *Compiler) cacheSafeUpdateBaseline(lockFile string, oldManifest *GHAWManifest) { + if oldManifest == nil { + return + } + if _, ok := c.priorManifests[lockFile]; !ok { + c.priorManifests[lockFile] = oldManifest + } +} + +func (c *Compiler) enforceSafeUpdateWarning(workflowData *WorkflowData, markdownPath string, oldManifest *GHAWManifest, oldHasPR bool, oldHasPRTarget bool, bodySecrets []string, bodyActions []string) { + currentHasPR, currentHasPRTarget := extractPullRequestEventPresenceFromOnField(workflowData.RawFrontmatter["on"]) + enforceErr := EnforceSafeUpdate(oldManifest, bodySecrets, bodyActions, workflowData.Redirect, oldHasPR, oldHasPRTarget, currentHasPR, currentHasPRTarget) + if enforceErr == nil { + return + } + warningMsg := buildSafeUpdateWarningPrompt(enforceErr.Error()) + c.AddSafeUpdateWarning(warningMsg) + fmt.Fprintln(os.Stderr, formatCompilerMessage(markdownPath, "warning", enforceErr.Error())) + c.IncrementWarningCount() } diff --git a/pkg/workflow/compiler_activation_daily_aic.go b/pkg/workflow/compiler_activation_daily_aic.go index 99534aa05a5..e9700317a69 100644 --- a/pkg/workflow/compiler_activation_daily_aic.go +++ b/pkg/workflow/compiler_activation_daily_aic.go @@ -105,66 +105,81 @@ func (c *Compiler) buildActivationDailyAICGuardrailStep(data *WorkflowData) []st compilerActivationJobLog.Print("Prepending dedicated daily-AIC app-token mint step") steps = append(steps, c.buildDailyAICAppTokenMintStep(data.MaxDailyAICreditsGitHubApp)...) } - // Prepend cache restore step so cached AIC values from prior runs are available - // when the guardrail script runs, allowing it to skip artifact downloads. if data.WorkflowID != "" { - sanitized := SanitizeWorkflowIDForCacheKey(data.WorkflowID) - cacheKeyPrefix := fmt.Sprintf("agentic-workflow-usage-%s-", sanitized) - steps = append(steps, " - name: Restore daily AIC usage cache\n") - steps = append(steps, " id: restore-daily-aic-cache\n") - steps = append(steps, fmt.Sprintf(" if: %s\n", maxDailyAICreditsConfiguredIfExpr)) - steps = append(steps, " continue-on-error: true\n") - steps = append(steps, fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/cache/restore", data))) - steps = append(steps, " with:\n") - steps = append(steps, fmt.Sprintf(" key: %s${{ github.run_id }}\n", cacheKeyPrefix)) - steps = append(steps, fmt.Sprintf(" restore-keys: %s\n", cacheKeyPrefix)) - steps = append(steps, " path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl\n") - // Artifact-based fallback for cross-branch cache misses. - // GitHub Actions actions/cache is branch-scoped: caches written by the conclusion job - // on one PR branch are invisible to the activation job running on a different PR branch. - // This step downloads the most recent aic-usage-cache artifact uploaded by a prior - // conclusion job so that the guardrail script can skip per-run artifact downloads. - // Cache-miss detection is performed inside restore_aic_usage_cache_fallback.cjs using - // the cache restore outputs forwarded via env vars. - steps = append(steps, " - name: Restore daily AIC usage cache (artifact fallback)\n") - steps = append(steps, " id: restore-daily-aic-cache-fallback\n") - steps = append(steps, fmt.Sprintf(" if: %s\n", maxDailyAICreditsConfiguredIfExpr)) - steps = append(steps, " continue-on-error: true\n") - steps = append(steps, fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data))) - steps = append(steps, " env:\n") - steps = append(steps, " GH_AW_RESTORE_DAILY_AIC_CACHE_HIT: ${{ steps.restore-daily-aic-cache.outputs.cache-hit }}\n") - steps = append(steps, " GH_AW_RESTORE_DAILY_AIC_CACHE_MATCHED_KEY: ${{ steps.restore-daily-aic-cache.outputs.cache-matched-key }}\n") - steps = append(steps, " with:\n") - steps = append(steps, fmt.Sprintf(" github-token: %s\n", c.resolveDailyAICToken(data))) - steps = append(steps, " script: |\n") - steps = append(steps, " const { setupGlobals } = require('"+SetupActionDestination+"/setup_globals.cjs');\n") - steps = append(steps, " setupGlobals(core, github, context, exec, io, getOctokit);\n") - steps = append(steps, " const { main } = require('"+SetupActionDestination+"/restore_aic_usage_cache_fallback.cjs');\n") - steps = append(steps, " await main();\n") + steps = append(steps, c.buildDailyAICCacheRestoreSteps(data)...) } - steps = append(steps, " - name: Check daily workflow token guardrail\n") - steps = append(steps, " id: daily-effective-workflow-guardrail\n") - steps = append(steps, fmt.Sprintf(" if: %s\n", maxDailyAICreditsConfiguredIfExpr)) - steps = append(steps, fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data))) - steps = append(steps, " env:\n") - steps = append(steps, fmt.Sprintf(" GH_AW_WORKFLOW_NAME: %q\n", data.Name)) - steps = append(steps, fmt.Sprintf(" GH_AW_WORKFLOW_ID: %q\n", data.WorkflowID)) - steps = append(steps, " GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\n") - steps = append(steps, " GH_AW_WORKFLOW_DISPATCH_AW_CONTEXT: ${{ github.event.inputs.aw_context || '' }}\n") - steps = append(steps, fmt.Sprintf(" GH_AW_HAS_SLASH_COMMAND: %q\n", strconv.FormatBool(len(data.Command) > 0))) - steps = append(steps, fmt.Sprintf(" GH_AW_HAS_LABEL_COMMAND: %q\n", strconv.FormatBool(len(data.LabelCommand) > 0))) - steps = append(steps, fmt.Sprintf(" GH_AW_GITHUB_TOKEN: %s\n", c.resolveDailyAICToken(data))) - steps = append(steps, buildTemplatableIntEnvVar(maxDailyAICreditsEnvVar, data.MaxDailyAICredits)...) - steps = append(steps, " with:\n") - steps = append(steps, fmt.Sprintf(" github-token: %s\n", c.resolveDailyAICToken(data))) - steps = append(steps, " script: |\n") - steps = append(steps, " const { setupGlobals } = require('"+SetupActionDestination+"/setup_globals.cjs');\n") - steps = append(steps, " setupGlobals(core, github, context, exec, io, getOctokit);\n") - steps = append(steps, " const { main } = require('"+SetupActionDestination+"/check_daily_aic_workflow_guardrail.cjs');\n") - steps = append(steps, " await main();\n") + steps = append(steps, c.buildDailyAICGuardrailScriptStep(data)...) return steps } +func (c *Compiler) buildDailyAICCacheRestoreSteps(data *WorkflowData) []string { + sanitized := SanitizeWorkflowIDForCacheKey(data.WorkflowID) + cacheKeyPrefix := fmt.Sprintf("agentic-workflow-usage-%s-", sanitized) + steps := []string{ + " - name: Restore daily AIC usage cache\n", + " id: restore-daily-aic-cache\n", + fmt.Sprintf(" if: %s\n", maxDailyAICreditsConfiguredIfExpr), + " continue-on-error: true\n", + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/cache/restore", data)), + " with:\n", + fmt.Sprintf(" key: %s${{ github.run_id }}\n", cacheKeyPrefix), + fmt.Sprintf(" restore-keys: %s\n", cacheKeyPrefix), + " path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl\n", + } + return append(steps, c.buildDailyAICCacheFallbackSteps(data)...) +} + +func (c *Compiler) buildDailyAICCacheFallbackSteps(data *WorkflowData) []string { + return []string{ + " - name: Restore daily AIC usage cache (artifact fallback)\n", + " id: restore-daily-aic-cache-fallback\n", + fmt.Sprintf(" if: %s\n", maxDailyAICreditsConfiguredIfExpr), + " continue-on-error: true\n", + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), + " env:\n", + " GH_AW_RESTORE_DAILY_AIC_CACHE_HIT: ${{ steps.restore-daily-aic-cache.outputs.cache-hit }}\n", + " GH_AW_RESTORE_DAILY_AIC_CACHE_MATCHED_KEY: ${{ steps.restore-daily-aic-cache.outputs.cache-matched-key }}\n", + " with:\n", + fmt.Sprintf(" github-token: %s\n", c.resolveDailyAICToken(data)), + " script: |\n", + " const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n", + " setupGlobals(core, github, context, exec, io, getOctokit);\n", + " const { main } = require('" + SetupActionDestination + "/restore_aic_usage_cache_fallback.cjs');\n", + " await main();\n", + } +} + +func (c *Compiler) buildDailyAICGuardrailScriptStep(data *WorkflowData) []string { + steps := []string{ + " - name: Check daily workflow token guardrail\n", + " id: daily-effective-workflow-guardrail\n", + fmt.Sprintf(" if: %s\n", maxDailyAICreditsConfiguredIfExpr), + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), + " env:\n", + fmt.Sprintf(" GH_AW_WORKFLOW_NAME: %q\n", data.Name), + fmt.Sprintf(" GH_AW_WORKFLOW_ID: %q\n", data.WorkflowID), + " GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\n", + " GH_AW_WORKFLOW_DISPATCH_AW_CONTEXT: ${{ github.event.inputs.aw_context || '' }}\n", + fmt.Sprintf(" GH_AW_HAS_SLASH_COMMAND: %q\n", strconv.FormatBool(len(data.Command) > 0)), + fmt.Sprintf(" GH_AW_HAS_LABEL_COMMAND: %q\n", strconv.FormatBool(len(data.LabelCommand) > 0)), + fmt.Sprintf(" GH_AW_GITHUB_TOKEN: %s\n", c.resolveDailyAICToken(data)), + } + steps = append(steps, buildTemplatableIntEnvVar(maxDailyAICreditsEnvVar, data.MaxDailyAICredits)...) + return append(steps, c.buildDailyAICGuardrailScriptWith(data)...) +} + +func (c *Compiler) buildDailyAICGuardrailScriptWith(data *WorkflowData) []string { + return []string{ + " with:\n", + fmt.Sprintf(" github-token: %s\n", c.resolveDailyAICToken(data)), + " script: |\n", + " const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n", + " setupGlobals(core, github, context, exec, io, getOctokit);\n", + " const { main } = require('" + SetupActionDestination + "/check_daily_aic_workflow_guardrail.cjs');\n", + " await main();\n", + } +} + func buildDailyAICActivationJobEnv(data *WorkflowData) map[string]string { if !hasMaxDailyAICGuardrail(data) || !hasMaxDailyAICFrontmatterConfig(data) { return nil diff --git a/pkg/workflow/compiler_activation_job.go b/pkg/workflow/compiler_activation_job.go index 7c8e58886f7..57d454d740b 100644 --- a/pkg/workflow/compiler_activation_job.go +++ b/pkg/workflow/compiler_activation_job.go @@ -57,21 +57,7 @@ func (c *Compiler) buildActivationJob(data *WorkflowData, preActivationJobCreate } ctx.steps = append(ctx.steps, buildRuntimeFeaturesSummaryStep()...) - // Generate experiment selection steps when experiments are declared in the frontmatter. - // These steps run before the prompt is built so that experiments.name expressions - // can be resolved by the substitute_placeholders step. - if experimentSteps := c.generateExperimentSteps(data); len(experimentSteps) > 0 { - compilerActivationJobLog.Printf("Adding %d experiment step(s) for %d experiment(s)", len(experimentSteps), len(data.Experiments)) - ctx.steps = append(ctx.steps, experimentSteps...) - // Expose the combined experiment JSON as a job output so downstream jobs can access - // the variant assignments via needs.activation.outputs.experiments. - ctx.outputs["experiments"] = "${{ steps.pick-experiment.outputs.experiments }}" - // Also expose each experiment variant individually so downstream jobs can reference - // needs.activation.outputs. in timeout-minutes or other expressions. - for _, name := range sortedExperimentNames(data.Experiments) { - ctx.outputs[name] = "${{ steps.pick-experiment.outputs." + name + " }}" - } - } + c.addActivationExperimentSteps(ctx, data) c.configureActivationNeedsAndCondition(ctx) compilerActivationJobLog.Print("Generating prompt in activation job") @@ -104,6 +90,19 @@ func (c *Compiler) buildActivationJob(data *WorkflowData, preActivationJobCreate }, nil } +func (c *Compiler) addActivationExperimentSteps(ctx *activationJobBuildContext, data *WorkflowData) { + experimentSteps := c.generateExperimentSteps(data) + if len(experimentSteps) == 0 { + return + } + compilerActivationJobLog.Printf("Adding %d experiment step(s) for %d experiment(s)", len(experimentSteps), len(data.Experiments)) + ctx.steps = append(ctx.steps, experimentSteps...) + ctx.outputs["experiments"] = "${{ steps.pick-experiment.outputs.experiments }}" + for _, name := range sortedExperimentNames(data.Experiments) { + ctx.outputs[name] = "${{ steps.pick-experiment.outputs." + name + " }}" + } +} + func addActivationInteractionPermissions( perms *Permissions, options activationInteractionPermissionsOptions, @@ -422,18 +421,28 @@ func (c *Compiler) generateCheckoutGitHubFolderForActivation(data *WorkflowData) // In dev mode, actions/setup is referenced via a local workspace path (./actions/setup), // so it must be included in the sparse-checkout to preserve it for the post step. // In release/script/action modes, the action is in the runner cache and not the workspace. + extraPaths := c.activationSparseCheckoutExtraPaths() + + cm := NewCheckoutManager(nil) + activationToken := c.resolveActivationToken(data) + if data != nil && hasWorkflowCallTrigger(data.On) && !data.InlinedImports { + return c.generateCrossRepoActivationCheckout(cm, activationToken, extraPaths) + } + + // For activation job, sparse checkout .github, .agents, and engine-specific config directories + // (plus actions/setup in dev mode). Root instruction files are excluded as they are not needed + // during activation. sparse-checkout-cone-mode: true ensures subdirectories are recursively included. + compilerActivationJobLog.Print("Adding .github, .agents, and engine-specific dirs to sparse checkout for activation job") + return cm.GenerateGitHubFolderCheckoutStep("", "", activationToken, c.getActionPin, extraPaths...) +} + +func (c *Compiler) activationSparseCheckoutExtraPaths() []string { var extraPaths []string if c.actionMode.IsDev() { compilerActivationJobLog.Print("Dev mode: adding actions/setup to sparse-checkout to preserve local action post step") extraPaths = append(extraPaths, "actions/setup") } - - // Add engine-specific agent config directories to the sparse checkout. - // .github and .agents are already included in GenerateGitHubFolderCheckoutStep's hardcoded list. - // Root instruction files (AGENTS.md, CLAUDE.md, GEMINI.md) are excluded — they are not needed - // during activation and are omitted to keep the shallow checkout minimal. - defaultSparseCheckoutDirs := map[string]struct { - }{".github": {}, ".agents": {}} + defaultSparseCheckoutDirs := map[string]struct{}{".github": {}, ".agents": {}} registry := GetGlobalEngineRegistry() for _, folder := range registry.GetAllAgentManifestFolders() { if !setutil.Contains(defaultSparseCheckoutDirs, folder) { @@ -441,49 +450,35 @@ func (c *Compiler) generateCheckoutGitHubFolderForActivation(data *WorkflowData) } } compilerActivationJobLog.Printf("Adding %d engine-specific dirs to sparse-checkout: %v", len(extraPaths), extraPaths) + return resolveSymlinkExtraPaths(c.activationRepoRoot(), extraPaths) +} - // Detect symlinks for well-known .github sub-paths and add their resolved targets - // so that sparse checkout fetches the target directory, not just the symlink blob. - // Use c.gitRoot so detection works regardless of the process CWD. - repoRoot := c.gitRoot - if repoRoot == "" { - if cwd, err := os.Getwd(); err == nil { - repoRoot = cwd - } +func (c *Compiler) activationRepoRoot() string { + if c.gitRoot != "" { + return c.gitRoot } - extraPaths = resolveSymlinkExtraPaths(repoRoot, extraPaths) - - cm := NewCheckoutManager(nil) - activationToken := c.resolveActivationToken(data) - if data != nil && hasWorkflowCallTrigger(data.On) && !data.InlinedImports { - compilerActivationJobLog.Print("Adding cross-repo-aware .github checkout for workflow_call trigger") - cm.SetCrossRepoTargetRepo("${{ steps.resolve-host-repo.outputs.target_repo }}") - cm.SetCrossRepoTargetRef("${{ steps.resolve-host-repo.outputs.target_checkout_ref }}") - checkoutSteps := cm.GenerateGitHubFolderCheckoutStep( - cm.GetCrossRepoTargetRepo(), - cm.GetCrossRepoTargetRef(), - activationToken, - c.getActionPin, - extraPaths..., - ) - // When no custom token is configured, GITHUB_TOKEN is scoped to the calling - // repository and cannot read a private callee repository in cross-repo invocations - // (e.g. nbcnews/tvOS-App calling nbcnews/.github). Add an if: condition so the - // checkout is only attempted for same-repo invocations where GITHUB_TOKEN works. - // For cross-repo scenarios, users can enable the checkout by configuring - // activation-github-token or activation-github-app in the workflow frontmatter. - if activationToken == "${{ secrets.GITHUB_TOKEN }}" { - compilerActivationJobLog.Print("No custom activation token — restricting cross-repo checkout to same-repo invocations") - checkoutSteps = addSameRepoIfConditionToSteps(checkoutSteps) - } - return checkoutSteps + if cwd, err := os.Getwd(); err == nil { + return cwd } + return "" +} - // For activation job, sparse checkout .github, .agents, and engine-specific config directories - // (plus actions/setup in dev mode). Root instruction files are excluded as they are not needed - // during activation. sparse-checkout-cone-mode: true ensures subdirectories are recursively included. - compilerActivationJobLog.Print("Adding .github, .agents, and engine-specific dirs to sparse checkout for activation job") - return cm.GenerateGitHubFolderCheckoutStep("", "", activationToken, c.getActionPin, extraPaths...) +func (c *Compiler) generateCrossRepoActivationCheckout(cm *CheckoutManager, activationToken string, extraPaths []string) []string { + compilerActivationJobLog.Print("Adding cross-repo-aware .github checkout for workflow_call trigger") + cm.SetCrossRepoTargetRepo("${{ steps.resolve-host-repo.outputs.target_repo }}") + cm.SetCrossRepoTargetRef("${{ steps.resolve-host-repo.outputs.target_checkout_ref }}") + checkoutSteps := cm.GenerateGitHubFolderCheckoutStep( + cm.GetCrossRepoTargetRepo(), + cm.GetCrossRepoTargetRef(), + activationToken, + c.getActionPin, + extraPaths..., + ) + if activationToken == "${{ secrets.GITHUB_TOKEN }}" { + compilerActivationJobLog.Print("No custom activation token — restricting cross-repo checkout to same-repo invocations") + checkoutSteps = addSameRepoIfConditionToSteps(checkoutSteps) + } + return checkoutSteps } // addSameRepoIfConditionToSteps injects an if: condition into each step that restricts diff --git a/pkg/workflow/compiler_activation_permissions.go b/pkg/workflow/compiler_activation_permissions.go index b5599ed47bd..f2003b38185 100644 --- a/pkg/workflow/compiler_activation_permissions.go +++ b/pkg/workflow/compiler_activation_permissions.go @@ -53,38 +53,10 @@ func buildActivationAppTokenPermissions(ctx *activationJobBuildContext) *Permiss }, ) if ctx.data.CommandCentralized && (ctx.hasReaction || ctx.hasStatusComment) { - syntheticOn := buildCentralizedCommandOnSection(ctx.data.CommandEvents) - if syntheticOn != "" { - addActivationInteractionPermissions( - appPerms, - activationInteractionPermissionsOptions{ - onSection: syntheticOn, - hasReaction: ctx.hasReaction, - reactionIncludesIssues: ctx.reactionIssues, - reactionIncludesPullRequests: ctx.reactionPullRequests, - reactionIncludesDiscussions: ctx.reactionDiscussions, - hasStatusComment: ctx.hasStatusComment, - statusCommentIncludesIssues: ctx.statusCommentIssues, - statusCommentIncludesPullRequests: ctx.statusCommentPRs, - statusCommentIncludesDiscussions: ctx.statusCommentDiscussions, - }, - ) - } + addCentralizedActivationAppTokenPermissions(appPerms, ctx) } if hasWorkflowCallTrigger(ctx.data.On) && (ctx.hasReaction || ctx.hasStatusComment) { - addActivationInteractionPermissions( - appPerms, - activationInteractionPermissionsOptions{ - hasReaction: ctx.hasReaction, - reactionIncludesIssues: ctx.reactionIssues, - reactionIncludesPullRequests: ctx.reactionPullRequests, - reactionIncludesDiscussions: ctx.reactionDiscussions, - hasStatusComment: ctx.hasStatusComment, - statusCommentIncludesIssues: ctx.statusCommentIssues, - statusCommentIncludesPullRequests: ctx.statusCommentPRs, - statusCommentIncludesDiscussions: ctx.statusCommentDiscussions, - }, - ) + addActivationInteractionPermissions(appPerms, activationInteractionOptions(ctx, "")) } // Keep this aligned with addActivationLabelPermissions: app-token scopes are // computed separately from GITHUB_TOKEN scopes because app-token permissions @@ -94,12 +66,7 @@ func buildActivationAppTokenPermissions(ctx *activationJobBuildContext) *Permiss // ActivationGitHubApp == nil guard because this function runs only when // activationJobNeedsAppToken confirms app-token minting is enabled. if ctx.shouldRemoveLabel { - if slices.Contains(ctx.filteredLabelEvents, "issues") || slices.Contains(ctx.filteredLabelEvents, "pull_request") { - appPerms.Set(PermissionIssues, PermissionWrite) - } - if slices.Contains(ctx.filteredLabelEvents, "discussion") { - appPerms.Set(PermissionDiscussions, PermissionWrite) - } + addActivationAppTokenLabelPermissions(appPerms, ctx) } if ctx.needsAppTokenForAccess { appPerms.Set(PermissionContents, PermissionRead) @@ -118,6 +85,37 @@ func buildActivationAppTokenPermissions(ctx *activationJobBuildContext) *Permiss return appPerms } +func addCentralizedActivationAppTokenPermissions(appPerms *Permissions, ctx *activationJobBuildContext) { + syntheticOn := buildCentralizedCommandOnSection(ctx.data.CommandEvents) + if syntheticOn == "" { + return + } + addActivationInteractionPermissions(appPerms, activationInteractionOptions(ctx, syntheticOn)) +} + +func activationInteractionOptions(ctx *activationJobBuildContext, onSection string) activationInteractionPermissionsOptions { + return activationInteractionPermissionsOptions{ + onSection: onSection, + hasReaction: ctx.hasReaction, + reactionIncludesIssues: ctx.reactionIssues, + reactionIncludesPullRequests: ctx.reactionPullRequests, + reactionIncludesDiscussions: ctx.reactionDiscussions, + hasStatusComment: ctx.hasStatusComment, + statusCommentIncludesIssues: ctx.statusCommentIssues, + statusCommentIncludesPullRequests: ctx.statusCommentPRs, + statusCommentIncludesDiscussions: ctx.statusCommentDiscussions, + } +} + +func addActivationAppTokenLabelPermissions(appPerms *Permissions, ctx *activationJobBuildContext) { + if slices.Contains(ctx.filteredLabelEvents, "issues") || slices.Contains(ctx.filteredLabelEvents, "pull_request") { + appPerms.Set(PermissionIssues, PermissionWrite) + } + if slices.Contains(ctx.filteredLabelEvents, "discussion") { + appPerms.Set(PermissionDiscussions, PermissionWrite) + } +} + // buildActivationPermissions builds activation job permissions from workflow features and selected interactions. // Returns an error if any activation job step section contains write gh CLI commands that would require write permissions. func (c *Compiler) buildActivationPermissions(ctx *activationJobBuildContext) (string, error) { diff --git a/pkg/workflow/compiler_activation_steps.go b/pkg/workflow/compiler_activation_steps.go index c94f80fb499..677aef456ca 100644 --- a/pkg/workflow/compiler_activation_steps.go +++ b/pkg/workflow/compiler_activation_steps.go @@ -169,68 +169,81 @@ func (c *Compiler) addActivationVersionCheckStep(ctx *activationJobBuildContext) } func (c *Compiler) addActivationSkillInstallSteps(ctx *activationJobBuildContext) error { - skillRefs := append([]SkillReference(nil), ctx.data.SkillReferences...) - if len(skillRefs) == 0 && len(ctx.data.Skills) > 0 { - skillRefs = make([]SkillReference, 0, len(ctx.data.Skills)) - for _, skill := range ctx.data.Skills { - if strings.TrimSpace(skill) == "" { - continue - } - skillRefs = append(skillRefs, SkillReference{Skill: skill}) - } - } + skillRefs := activationSkillReferences(ctx.data) if len(skillRefs) == 0 { return nil } engineID := resolveActivationEngineID(ctx.data) skillDir := GetEngineSkillDir(engineID) - skillInstallAgentName := "" - if engine, err := GetGlobalEngineRegistry().GetEngine(strings.ToLower(engineID)); err == nil { - skillInstallAgentName = engine.GetGHSkillAgentName() - } + skillInstallAgentName := activationSkillInstallAgentName(engineID) ctx.steps = append(ctx.steps, " - name: Upgrade gh CLI for frontmatter skills\n") ctx.steps = append(ctx.steps, fmt.Sprintf(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/ensure_gh_cli_min_version.sh\" \"%s\"\n", constants.GhSkillsMinVersion)) for i, skillRef := range skillRefs { - tokenExpr := c.resolveActivationToken(ctx.data) - if skillRef.GitHubToken != "" { - tokenExpr = skillRef.GitHubToken - } - if skillRef.GitHubApp != nil { - stepNumber := i + 1 - stepID := fmt.Sprintf("frontmatter-skill-app-token-%d", stepNumber) - ctx.steps = append(ctx.steps, c.buildGitHubAppTokenMintStepWithMeta( - skillRef.GitHubApp, - nil, - "", - "", - fmt.Sprintf("Generate GitHub App token for frontmatter skill %d", stepNumber), - stepID, - )...) - stepTokenExpr := fmt.Sprintf("${{ steps.%s.outputs.token }}", stepID) - if skillRef.GitHubApp.shouldIgnoreMissingKey() { - tokenExpr = combineTokenExpressions(stepTokenExpr, c.resolveActivationToken(ctx.data)) - } else { - tokenExpr = stepTokenExpr - } + c.addActivationSingleSkillInstallStep(ctx, skillRef, i+1, engineID, skillInstallAgentName, skillDir) + } + + c.addActivationSkillFailureCollectionStep(ctx) + + return nil +} + +func activationSkillReferences(data *WorkflowData) []SkillReference { + skillRefs := append([]SkillReference(nil), data.SkillReferences...) + if len(skillRefs) > 0 || len(data.Skills) == 0 { + return skillRefs + } + skillRefs = make([]SkillReference, 0, len(data.Skills)) + for _, skill := range data.Skills { + if strings.TrimSpace(skill) != "" { + skillRefs = append(skillRefs, SkillReference{Skill: skill}) } - ctx.steps = append(ctx.steps, fmt.Sprintf(" - name: Install frontmatter skill %d\n", i+1)) - ctx.steps = append(ctx.steps, " env:\n") - ctx.steps = append(ctx.steps, fmt.Sprintf(" GH_TOKEN: %s\n", tokenExpr)) - ctx.steps = append(ctx.steps, formatYAMLEnv(" ", "GH_AW_INFO_ENGINE_ID", engineID)) - ctx.steps = append(ctx.steps, formatYAMLEnv(" ", "GH_AW_GH_SKILL_AGENT_NAME", skillInstallAgentName)) - ctx.steps = append(ctx.steps, formatYAMLEnv(" ", "GH_AW_SKILL_DIR", skillDir)) - ctx.steps = append(ctx.steps, formatYAMLEnv(" ", "GH_AW_FRONTMATTER_SKILLS", skillRef.Skill)) - ctx.steps = append(ctx.steps, fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", ctx.data))) - ctx.steps = append(ctx.steps, " with:\n") - ctx.steps = append(ctx.steps, " script: |\n") - ctx.steps = append(ctx.steps, generateGitHubScriptWithRequire("install_frontmatter_skills.cjs")) - } - - // Collect skill install failures written by each install step into a shared file. - // Runs with if: always() so failures are captured even if a prior step was unexpectedly hard-failed. + } + return skillRefs +} + +func activationSkillInstallAgentName(engineID string) string { + if engine, err := GetGlobalEngineRegistry().GetEngine(strings.ToLower(engineID)); err == nil { + return engine.GetGHSkillAgentName() + } + return "" +} + +func (c *Compiler) addActivationSingleSkillInstallStep(ctx *activationJobBuildContext, skillRef SkillReference, stepNumber int, engineID string, skillInstallAgentName string, skillDir string) { + tokenExpr := c.activationSkillTokenExpr(ctx, skillRef, stepNumber) + ctx.steps = append(ctx.steps, fmt.Sprintf(" - name: Install frontmatter skill %d\n", stepNumber)) + ctx.steps = append(ctx.steps, " env:\n") + ctx.steps = append(ctx.steps, fmt.Sprintf(" GH_TOKEN: %s\n", tokenExpr)) + ctx.steps = append(ctx.steps, formatYAMLEnv(" ", "GH_AW_INFO_ENGINE_ID", engineID)) + ctx.steps = append(ctx.steps, formatYAMLEnv(" ", "GH_AW_GH_SKILL_AGENT_NAME", skillInstallAgentName)) + ctx.steps = append(ctx.steps, formatYAMLEnv(" ", "GH_AW_SKILL_DIR", skillDir)) + ctx.steps = append(ctx.steps, formatYAMLEnv(" ", "GH_AW_FRONTMATTER_SKILLS", skillRef.Skill)) + ctx.steps = append(ctx.steps, fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", ctx.data))) + ctx.steps = append(ctx.steps, " with:\n") + ctx.steps = append(ctx.steps, " script: |\n") + ctx.steps = append(ctx.steps, generateGitHubScriptWithRequire("install_frontmatter_skills.cjs")) +} + +func (c *Compiler) activationSkillTokenExpr(ctx *activationJobBuildContext, skillRef SkillReference, stepNumber int) string { + tokenExpr := c.resolveActivationToken(ctx.data) + if skillRef.GitHubToken != "" { + tokenExpr = skillRef.GitHubToken + } + if skillRef.GitHubApp == nil { + return tokenExpr + } + stepID := fmt.Sprintf("frontmatter-skill-app-token-%d", stepNumber) + ctx.steps = append(ctx.steps, c.buildGitHubAppTokenMintStepWithMeta(skillRef.GitHubApp, nil, "", "", fmt.Sprintf("Generate GitHub App token for frontmatter skill %d", stepNumber), stepID)...) + stepTokenExpr := fmt.Sprintf("${{ steps.%s.outputs.token }}", stepID) + if skillRef.GitHubApp.shouldIgnoreMissingKey() { + return combineTokenExpressions(stepTokenExpr, c.resolveActivationToken(ctx.data)) + } + return stepTokenExpr +} + +func (c *Compiler) addActivationSkillFailureCollectionStep(ctx *activationJobBuildContext) { ctx.steps = append(ctx.steps, " - name: Collect skill install failures\n") ctx.steps = append(ctx.steps, " id: collect-skill-install-failures\n") ctx.steps = append(ctx.steps, " if: always()\n") @@ -238,11 +251,8 @@ func (c *Compiler) addActivationSkillInstallSteps(ctx *activationJobBuildContext ctx.steps = append(ctx.steps, " with:\n") ctx.steps = append(ctx.steps, " script: |\n") ctx.steps = append(ctx.steps, generateGitHubScriptWithRequire("collect_skill_install_failures.cjs")) - ctx.outputs["skill_install_failure_count"] = "${{ steps.collect-skill-install-failures.outputs.failure_count || '0' }}" ctx.outputs["skill_install_errors"] = "${{ steps.collect-skill-install-failures.outputs.errors || '' }}" - - return nil } func (c *Compiler) addActivationTextOutputStep(ctx *activationJobBuildContext) error { diff --git a/pkg/workflow/compiler_aw_context.go b/pkg/workflow/compiler_aw_context.go index 0e57b7f7c46..02f6d63ed9f 100644 --- a/pkg/workflow/compiler_aw_context.go +++ b/pkg/workflow/compiler_aw_context.go @@ -57,32 +57,39 @@ func injectInputIntoTrigger(onSection string, triggerName string, inputName stri awContextLog.Printf("Injecting %s input into %s trigger", inputName, triggerName) lines := strings.Split(onSection, "\n") + triggerLineIdx, triggerIndent := findBareTriggerLine(lines, triggerName) + if triggerLineIdx == -1 { + awContextLog.Printf("No bare %s: line found, skipping %s injection", triggerName, inputName) + return onSection + } + awContextLog.Printf("Found %s at line %d (indent=%d), injecting %s", triggerName, triggerLineIdx, triggerIndent, inputName) - // Find the trigger line (bare — no sub-value on same line) - triggerLineIdx := -1 - triggerIndent := 0 + inputsLineIdx := findTriggerInputsLine(lines, triggerLineIdx, triggerIndent) + if inputsLineIdx != -1 && inputAlreadyInjected(lines, inputsLineIdx, inputName) { + awContextLog.Printf("%s already injected into %s, skipping", inputName, triggerName) + return onSection + } + + inputLines := buildInputLines(triggerIndent) + result := injectInputLinesIntoTrigger(lines, triggerLineIdx, triggerIndent, inputsLineIdx, inputLines) + return strings.Join(result, "\n") +} + +func findBareTriggerLine(lines []string, triggerName string) (int, int) { for i, line := range lines { stripped := strings.TrimLeft(line, " \t") rest, found := strings.CutPrefix(stripped, triggerName+":") if found { rest = strings.TrimSpace(rest) if rest == "" || rest == "null" || rest == "~" { - triggerLineIdx = i - triggerIndent = len(line) - len(stripped) - break + return i, len(line) - len(stripped) } } } + return -1, 0 +} - if triggerLineIdx == -1 { - awContextLog.Printf("No bare %s: line found, skipping %s injection", triggerName, inputName) - return onSection - } - awContextLog.Printf("Found %s at line %d (indent=%d), injecting %s", triggerName, triggerLineIdx, triggerIndent, inputName) - - // Look for an "inputs:" key directly inside the trigger block. - // Only the first non-empty, non-comment line after the trigger matters. - inputsLineIdx := -1 +func findTriggerInputsLine(lines []string, triggerLineIdx int, triggerIndent int) int { for i := triggerLineIdx + 1; i < len(lines); i++ { stripped := strings.TrimLeft(lines[i], " \t") if stripped == "" || strings.HasPrefix(stripped, "#") { @@ -90,34 +97,35 @@ func injectInputIntoTrigger(onSection string, triggerName string, inputName stri } lineIndent := len(lines[i]) - len(stripped) if lineIndent <= triggerIndent { - break // left workflow_dispatch block entirely + break } if strings.HasPrefix(stripped, "inputs:") { - inputsLineIdx = i + return i } - break // only inspect the first substantive child key + break } + return -1 +} - if inputsLineIdx != -1 { - inputsIndent := len(lines[inputsLineIdx]) - len(strings.TrimLeft(lines[inputsLineIdx], " \t")) - for i := inputsLineIdx + 1; i < len(lines); i++ { - stripped := strings.TrimLeft(lines[i], " \t") - if stripped == "" || strings.HasPrefix(stripped, "#") { - continue - } - lineIndent := len(lines[i]) - len(stripped) - if lineIndent <= inputsIndent { - break - } - if strings.HasPrefix(stripped, inputName+":") { - awContextLog.Printf("%s already injected into %s, skipping", inputName, triggerName) - return onSection - } +func inputAlreadyInjected(lines []string, inputsLineIdx int, inputName string) bool { + inputsIndent := len(lines[inputsLineIdx]) - len(strings.TrimLeft(lines[inputsLineIdx], " \t")) + for i := inputsLineIdx + 1; i < len(lines); i++ { + stripped := strings.TrimLeft(lines[i], " \t") + if stripped == "" || strings.HasPrefix(stripped, "#") { + continue + } + lineIndent := len(lines[i]) - len(stripped) + if lineIndent <= inputsIndent { + break + } + if strings.HasPrefix(stripped, inputName+":") { + return true } } + return false +} - inputLines := buildInputLines(triggerIndent) - +func injectInputLinesIntoTrigger(lines []string, triggerLineIdx int, triggerIndent int, inputsLineIdx int, inputLines []string) []string { result := make([]string, 0, safeAllocationCapacity(len(lines), len(inputLines), 1)) for i, line := range lines { // When the trigger line contains an explicit null/~ value, @@ -137,8 +145,7 @@ func injectInputIntoTrigger(onSection string, triggerName string, inputName stri result = append(result, inputLines...) } } - - return strings.Join(result, "\n") + return result } // buildAwContextInputLines returns the indented YAML lines for the aw_context input diff --git a/pkg/workflow/compiler_custom_jobs.go b/pkg/workflow/compiler_custom_jobs.go index 83f46ec1031..027fcb62ebe 100644 --- a/pkg/workflow/compiler_custom_jobs.go +++ b/pkg/workflow/compiler_custom_jobs.go @@ -480,86 +480,79 @@ func configureCustomReusableWorkflow(job *Job, jobName string, usesStr string, c } func (c *Compiler) configureCustomJobSteps(job *Job, jobName string, configMap map[string]any, data *WorkflowData) error { - // Add basic steps if specified (only for non-reusable workflow jobs). - // `setup-steps` and `pre-steps` stay distinct so setup-steps can remain the - // first injected steps in the job, followed by compiler scaffolding, - // `pre-steps`, and the regular `steps` list. - var setupSteps []string - var preSteps []string - var regularSteps []string _, hasSetupStepsField := configMap["setup-steps"] _, hasPreStepsField := configMap["pre-steps"] _, hasStepsField := configMap["steps"] - - if hasSetupStepsField { - var err error - setupSteps, err = c.extractPinnedJobSteps("setup-steps", jobName, configMap, data) - if err != nil { - return fmt.Errorf("failed to process setup-steps for job '%s': %w", jobName, err) - } - } - if hasPreStepsField { - var err error - preSteps, err = c.extractPinnedJobSteps("pre-steps", jobName, configMap, data) - if err != nil { - return fmt.Errorf("failed to process pre-steps for job '%s': %w", jobName, err) - } - } - if hasStepsField { - var err error - regularSteps, err = c.extractPinnedJobSteps("steps", jobName, configMap, data) - if err != nil { - return fmt.Errorf("failed to process steps for job '%s': %w", jobName, err) - } + setupSteps, preSteps, regularSteps, err := c.extractCustomJobStepGroups(jobName, configMap, data) + if err != nil { + return err } - - // Parse restore-memory configuration. - // restore-memory injects read-only memory restore steps into the custom job. - // No write-back or commit steps are ever emitted for memory in custom jobs. restoreMemCfg, err := extractRestoreMemoryConfig(configMap, jobName, data) if err != nil { return err } - hasRestoreMemory := restoreMemCfg != nil - - // When cache-memory restore is requested, inject GH_AW_WORKFLOW_ID_SANITIZED so that - // restore keys match those used by the agent job. Only set it when the user has not - // already provided the variable in their job's env: block. - if hasRestoreMemory && restoreMemCfg.CacheMemory && data.WorkflowID != "" { - sanitized := SanitizeWorkflowIDForCacheKey(data.WorkflowID) - if job.Env == nil { - job.Env = make(map[string]string) - } - if _, alreadySet := job.Env["GH_AW_WORKFLOW_ID_SANITIZED"]; !alreadySet { - job.Env["GH_AW_WORKFLOW_ID_SANITIZED"] = sanitized - } - } - + ensureRestoreMemoryCacheEnv(job, restoreMemCfg, data) if hasSetupStepsField || hasPreStepsField || hasStepsField || hasRestoreMemory { job.Steps = append(job.Steps, setupSteps...) - // Prepend GH_HOST configuration step for GHES/GHEC compatibility. - // Custom frontmatter jobs run as independent GitHub Actions jobs that - // don't inherit GITHUB_ENV from the agent job, so the gh CLI won't - // know which host to target without this step. job.Steps = append(job.Steps, generateGHESHostConfigurationStep()) - - // Inject gh-aw setup + memory restore steps when restore-memory is requested. - // Setup lines come first (they install scripts needed by repo/comment memory). - // Memory lines follow immediately after (restore/clone/prepare steps). if hasRestoreMemory { - memorySetupLines, memoryRestoreLines, memErr := c.buildRestoreMemorySteps(restoreMemCfg, jobName, data) - if memErr != nil { - return memErr + if err := c.appendCustomJobRestoreMemorySteps(job, restoreMemCfg, jobName, data); err != nil { + return err } - job.Steps = append(job.Steps, memorySetupLines...) - job.Steps = append(job.Steps, memoryRestoreLines...) } - job.Steps = append(job.Steps, preSteps...) job.Steps = append(job.Steps, regularSteps...) } + return nil +} +func (c *Compiler) extractCustomJobStepGroups(jobName string, configMap map[string]any, data *WorkflowData) ([]string, []string, []string, error) { + setupSteps, err := c.extractCustomJobStepGroup("setup-steps", jobName, configMap, data) + if err != nil { + return nil, nil, nil, err + } + preSteps, err := c.extractCustomJobStepGroup("pre-steps", jobName, configMap, data) + if err != nil { + return nil, nil, nil, err + } + regularSteps, err := c.extractCustomJobStepGroup("steps", jobName, configMap, data) + if err != nil { + return nil, nil, nil, err + } + return setupSteps, preSteps, regularSteps, nil +} + +func (c *Compiler) extractCustomJobStepGroup(fieldName, jobName string, configMap map[string]any, data *WorkflowData) ([]string, error) { + if _, hasField := configMap[fieldName]; !hasField { + return nil, nil + } + steps, err := c.extractPinnedJobSteps(fieldName, jobName, configMap, data) + if err != nil { + return nil, fmt.Errorf("failed to process %s for job '%s': %w", fieldName, jobName, err) + } + return steps, nil +} + +func ensureRestoreMemoryCacheEnv(job *Job, restoreMemCfg *restoreMemoryConfig, data *WorkflowData) { + if restoreMemCfg == nil || !restoreMemCfg.CacheMemory || data.WorkflowID == "" { + return + } + if job.Env == nil { + job.Env = make(map[string]string) + } + if _, alreadySet := job.Env["GH_AW_WORKFLOW_ID_SANITIZED"]; !alreadySet { + job.Env["GH_AW_WORKFLOW_ID_SANITIZED"] = SanitizeWorkflowIDForCacheKey(data.WorkflowID) + } +} + +func (c *Compiler) appendCustomJobRestoreMemorySteps(job *Job, restoreMemCfg *restoreMemoryConfig, jobName string, data *WorkflowData) error { + memorySetupLines, memoryRestoreLines, err := c.buildRestoreMemorySteps(restoreMemCfg, jobName, data) + if err != nil { + return err + } + job.Steps = append(job.Steps, memorySetupLines...) + job.Steps = append(job.Steps, memoryRestoreLines...) return nil } @@ -682,7 +675,6 @@ func (c *Compiler) applyBuiltinJobNeedsAugmentations(data *WorkflowData) error { if data == nil || data.Jobs == nil { return nil } - allJobs := c.jobManager.GetAllJobs() for configuredJobName, rawConfig := range data.Jobs { targetJobName := normalizeBuiltinJobAlias(configuredJobName) @@ -702,45 +694,46 @@ func (c *Compiler) applyBuiltinJobNeedsAugmentations(data *WorkflowData) error { if len(augmentedNeeds) == 0 { continue } - targetJob, exists := c.jobManager.GetJob(targetJobName) if !exists { return fmt.Errorf("jobs.%s.needs: cannot augment %q because this workflow does not generate that job", configuredJobName, targetJobName) } - - normalizedNeeds := make([]string, 0, len(augmentedNeeds)) - for _, rawNeed := range augmentedNeeds { - need := normalizeBuiltinJobAlias(rawNeed) - if need == targetJobName { - return fmt.Errorf("jobs.%s.needs: %q cannot depend on itself", configuredJobName, rawNeed) - } - if _, known := allJobs[need]; !known { - return fmt.Errorf("jobs.%s.needs: unknown job %q", configuredJobName, rawNeed) - } - normalizedNeeds = append(normalizedNeeds, need) + normalizedNeeds, err := normalizeBuiltinJobNeeds(configuredJobName, targetJobName, augmentedNeeds, allJobs) + if err != nil { + return err } + targetJob.Needs = mergeBuiltinJobNeeds(targetJob.Needs, normalizedNeeds) + compilerJobsLog.Printf("Applied jobs.%s.needs augmentation to %q: %v", configuredJobName, targetJobName, normalizedNeeds) + } + return nil +} - seen := make(map[string]struct{}, len(targetJob.Needs)+len(normalizedNeeds)) - mergedNeeds := make([]string, 0, len(targetJob.Needs)+len(normalizedNeeds)) - for _, need := range targetJob.Needs { - if _, alreadySeen := seen[need]; alreadySeen { - continue - } - seen[need] = struct{}{} - mergedNeeds = append(mergedNeeds, need) +func normalizeBuiltinJobNeeds(configuredJobName, targetJobName string, augmentedNeeds []string, allJobs map[string]*Job) ([]string, error) { + normalizedNeeds := make([]string, 0, len(augmentedNeeds)) + for _, rawNeed := range augmentedNeeds { + need := normalizeBuiltinJobAlias(rawNeed) + if need == targetJobName { + return nil, fmt.Errorf("jobs.%s.needs: %q cannot depend on itself", configuredJobName, rawNeed) } - for _, need := range normalizedNeeds { - if _, alreadySeen := seen[need]; alreadySeen { - continue - } - seen[need] = struct{}{} - mergedNeeds = append(mergedNeeds, need) + if _, known := allJobs[need]; !known { + return nil, fmt.Errorf("jobs.%s.needs: unknown job %q", configuredJobName, rawNeed) } - targetJob.Needs = mergedNeeds - compilerJobsLog.Printf("Applied jobs.%s.needs augmentation to %q: %v", configuredJobName, targetJobName, normalizedNeeds) + normalizedNeeds = append(normalizedNeeds, need) } + return normalizedNeeds, nil +} - return nil +func mergeBuiltinJobNeeds(existingNeeds, normalizedNeeds []string) []string { + seen := make(map[string]struct{}, len(existingNeeds)+len(normalizedNeeds)) + mergedNeeds := make([]string, 0, len(existingNeeds)+len(normalizedNeeds)) + for _, need := range append(existingNeeds, normalizedNeeds...) { + if _, alreadySeen := seen[need]; alreadySeen { + continue + } + seen[need] = struct{}{} + mergedNeeds = append(mergedNeeds, need) + } + return mergedNeeds } func validateRestrictedBuiltinSetupSteps(jobName string, hasSetupSteps bool) error { @@ -777,42 +770,43 @@ func insertPreStepsAtEarliestBoundary(steps []string, preSteps []string) []strin if len(preSteps) == 0 { return steps } + insertIdx := findPreStepsInsertIndex(steps) + result := make([]string, 0, safeAllocationCapacity(len(steps), len(preSteps))) + result = append(result, steps[:insertIdx]...) + result = append(result, preSteps...) + result = append(result, steps[insertIdx:]...) + return result +} +func findPreStepsInsertIndex(steps []string) int { firstCheckoutIdx := -1 firstTokenMintIdx := -1 lastSetupIdx := -1 for i, step := range steps { if firstCheckoutIdx == -1 && strings.Contains(step, "uses: actions/checkout@") { - firstCheckoutIdx = i - // Walk backward to the checkout step's list-item boundary ("- "). - // If no boundary is found, keep the current index so insertion still - // occurs before the checkout uses-line. - for j := i; j >= 0; j-- { - trimmed := strings.TrimLeft(steps[j], " ") - if strings.HasPrefix(trimmed, "- ") { - firstCheckoutIdx = j - break - } - } + firstCheckoutIdx = findStepBoundaryBefore(steps, i) } if firstTokenMintIdx == -1 && strings.Contains(step, "uses: actions/create-github-app-token@") { - firstTokenMintIdx = i - // Walk backward to the token-mint step's list-item boundary ("- "). - // If no boundary is found, keep the current index so insertion still - // occurs before the token-mint uses-line. - for j := i; j >= 0; j-- { - trimmed := strings.TrimLeft(steps[j], " ") - if strings.HasPrefix(trimmed, "- ") { - firstTokenMintIdx = j - break - } - } + firstTokenMintIdx = findStepBoundaryBefore(steps, i) } if exactSetupStepIDPattern.MatchString(step) { lastSetupIdx = i } } + return choosePreStepsInsertIndex(steps, firstCheckoutIdx, firstTokenMintIdx, lastSetupIdx) +} +func findStepBoundaryBefore(steps []string, index int) int { + for j := index; j >= 0; j-- { + trimmed := strings.TrimLeft(steps[j], " ") + if strings.HasPrefix(trimmed, "- ") { + return j + } + } + return index +} + +func choosePreStepsInsertIndex(steps []string, firstCheckoutIdx, firstTokenMintIdx, lastSetupIdx int) int { insertIdx := len(steps) if lastSetupIdx >= 0 { for i := lastSetupIdx + 1; i < len(steps); i++ { @@ -838,12 +832,7 @@ func insertPreStepsAtEarliestBoundary(steps []string, preSteps []string) []strin if insertIdx > len(steps) { insertIdx = len(steps) } - - result := make([]string, 0, safeAllocationCapacity(len(steps), len(preSteps))) - result = append(result, steps[:insertIdx]...) - result = append(result, preSteps...) - result = append(result, steps[insertIdx:]...) - return result + return insertIdx } func (c *Compiler) extractPinnedJobSteps(fieldName string, jobName string, configMap map[string]any, data *WorkflowData) ([]string, error) { diff --git a/pkg/workflow/compiler_difc_proxy.go b/pkg/workflow/compiler_difc_proxy.go index ceb42ba3480..05249310337 100644 --- a/pkg/workflow/compiler_difc_proxy.go +++ b/pkg/workflow/compiler_difc_proxy.go @@ -364,22 +364,7 @@ func injectProxyEnvIntoCustomSteps(customSteps string) string { return customSteps } - // Extract version comments from uses lines before unmarshaling. - // YAML treats "# comment" as a comment and strips it during Unmarshal, so we - // must capture them here and re-apply after processing to preserve annotations - // like "uses: actions/upload-artifact@sha # v7" in the compiled lock file. - // Without this, gh-aw-manifest falls back to recording the SHA as the version. - versionComments := make(map[string]string) // key: action@sha, value: " # vX" - for line := range strings.SplitSeq(customSteps, "\n") { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "uses:") && strings.Contains(trimmed, " # ") { - parts := strings.SplitN(trimmed, " # ", 2) - if len(parts) == 2 { - usesValue := strings.TrimSpace(strings.TrimPrefix(parts[0], "uses:")) - versionComments[usesValue] = " # " + parts[1] - } - } - } + versionComments := extractUsesVersionComments(customSteps) var parsed struct { Steps []map[string]any `yaml:"steps"` @@ -395,24 +380,7 @@ func injectProxyEnvIntoCustomSteps(customSteps string) string { // name/uses stay ahead of env for stable diffs, then merge proxy env vars. orderedSteps := make([]yaml.MapSlice, len(parsed.Steps)) for i, step := range parsed.Steps { - envMap, ok := step["env"].(map[string]any) - if !ok { - envMap = make(map[string]any) - } - for k, v := range proxyEnv { - envMap[k] = v - } - step["env"] = envMap - - // Re-apply version comment to uses value so the comment survives re-serialization. - if usesVal, hasUses := step["uses"]; hasUses { - if usesStr, ok := usesVal.(string); ok { - if comment, hasComment := versionComments[usesStr]; hasComment { - step["uses"] = usesStr + comment - } - } - } - + injectProxyEnvIntoStep(step, proxyEnv, versionComments) orderedSteps[i] = OrderMapFields(step, constants.PriorityStepFields) } @@ -431,6 +399,39 @@ func injectProxyEnvIntoCustomSteps(customSteps string) string { return unquoteUsesWithComments(strings.TrimRight(string(resultBytes), "\n")) } +func extractUsesVersionComments(customSteps string) map[string]string { + versionComments := make(map[string]string) + for line := range strings.SplitSeq(customSteps, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "uses:") && strings.Contains(trimmed, " # ") { + parts := strings.SplitN(trimmed, " # ", 2) + if len(parts) == 2 { + usesValue := strings.TrimSpace(strings.TrimPrefix(parts[0], "uses:")) + versionComments[usesValue] = " # " + parts[1] + } + } + } + return versionComments +} + +func injectProxyEnvIntoStep(step map[string]any, proxyEnv map[string]string, versionComments map[string]string) { + envMap, ok := step["env"].(map[string]any) + if !ok { + envMap = make(map[string]any) + } + for k, v := range proxyEnv { + envMap[k] = v + } + step["env"] = envMap + if usesVal, hasUses := step["uses"]; hasUses { + if usesStr, ok := usesVal.(string); ok { + if comment, hasComment := versionComments[usesStr]; hasComment { + step["uses"] = usesStr + comment + } + } + } +} + // generateStopDIFCProxyStep generates a step that stops the DIFC proxy container // before the MCP gateway starts. The proxy must be stopped first to avoid // double-filtering: the gateway uses the same guard policy for the agent phase. diff --git a/pkg/workflow/compiler_experiments.go b/pkg/workflow/compiler_experiments.go index d7bab265999..622a333080a 100644 --- a/pkg/workflow/compiler_experiments.go +++ b/pkg/workflow/compiler_experiments.go @@ -147,81 +147,105 @@ func extractOneExperimentConfig(name string, val any) *ExperimentConfig { return &ExperimentConfig{Variants: variants} } case map[string]any: - // New object form: extract variants and optional metadata fields. - cfg := &ExperimentConfig{} - varRaw, ok := v["variants"] - if !ok { - experimentsLog.Printf("Skipping experiment %q: object form requires 'variants' field", name) - return nil - } - switch vv := varRaw.(type) { - case []string: - cfg.Variants = vv - case []any: - for _, item := range vv { - if s, ok := item.(string); ok { - cfg.Variants = append(cfg.Variants, s) - } - } - } - if len(cfg.Variants) < 2 { - experimentsLog.Printf("Skipping experiment %q: must have at least 2 variants", name) - return nil - } - if d, ok := v["description"].(string); ok { - cfg.Description = d - } - if m, ok := v["metric"].(string); ok { - cfg.Metric = m - } - if sd, ok := v["start_date"].(string); ok { - cfg.StartDate = sd - } - if ed, ok := v["end_date"].(string); ok { - cfg.EndDate = ed - } - if n, ok := extractIntField(v["issue"]); ok { - cfg.Issue = n - } - if weightRaw, ok := v["weight"]; ok { - cfg.Weight = extractIntSlice(weightRaw) - } - if h, ok := v["hypothesis"].(string); ok { - cfg.Hypothesis = h - } - if smRaw, ok := v["secondary_metrics"]; ok { - cfg.SecondaryMetrics = parseStringSliceAny(smRaw, nil) - } - if gmRaw, ok := v["guardrail_metrics"]; ok { - cfg.GuardrailMetrics = extractGuardrailMetrics(gmRaw) - } - if n, ok := extractIntField(v["min_samples"]); ok { - cfg.MinSamples = n - } - if at, ok := v["analysis_type"].(string); ok { - cfg.AnalysisType = at - } - if tagsRaw, ok := v["tags"]; ok { - cfg.Tags = parseStringSliceAny(tagsRaw, nil) - } - if notifyRaw, ok := v["notify"]; ok { - if notifyMap, ok := notifyRaw.(map[string]any); ok { - notify := &ExperimentNotify{} - hasNotify := false - if n, ok := extractIntField(notifyMap["discussion"]); ok { - notify.Discussion = n - hasNotify = true - } - if n, ok := extractIntField(notifyMap["issue"]); ok { - notify.Issue = n - hasNotify = true - } - if hasNotify { - cfg.Notify = notify - } + return extractExperimentObjectConfig(name, v) + } + return nil +} + +func extractExperimentObjectConfig(name string, raw map[string]any) *ExperimentConfig { + cfg := &ExperimentConfig{} + varRaw, ok := raw["variants"] + if !ok { + experimentsLog.Printf("Skipping experiment %q: object form requires 'variants' field", name) + return nil + } + cfg.Variants = extractExperimentVariants(varRaw) + if len(cfg.Variants) < 2 { + experimentsLog.Printf("Skipping experiment %q: must have at least 2 variants", name) + return nil + } + applyExperimentMetadata(cfg, raw) + return cfg +} + +func extractExperimentVariants(varRaw any) []string { + switch vv := varRaw.(type) { + case []string: + return vv + case []any: + var variants []string + for _, item := range vv { + if s, ok := item.(string); ok { + variants = append(variants, s) } } - return cfg + return variants + default: + return nil + } +} + +func applyExperimentMetadata(cfg *ExperimentConfig, raw map[string]any) { + if d, ok := raw["description"].(string); ok { + cfg.Description = d + } + if m, ok := raw["metric"].(string); ok { + cfg.Metric = m + } + if sd, ok := raw["start_date"].(string); ok { + cfg.StartDate = sd + } + if ed, ok := raw["end_date"].(string); ok { + cfg.EndDate = ed + } + if n, ok := extractIntField(raw["issue"]); ok { + cfg.Issue = n + } + applyExperimentExtendedMetadata(cfg, raw) +} + +func applyExperimentExtendedMetadata(cfg *ExperimentConfig, raw map[string]any) { + if weightRaw, ok := raw["weight"]; ok { + cfg.Weight = extractIntSlice(weightRaw) + } + if h, ok := raw["hypothesis"].(string); ok { + cfg.Hypothesis = h + } + if smRaw, ok := raw["secondary_metrics"]; ok { + cfg.SecondaryMetrics = parseStringSliceAny(smRaw, nil) + } + if gmRaw, ok := raw["guardrail_metrics"]; ok { + cfg.GuardrailMetrics = extractGuardrailMetrics(gmRaw) + } + if n, ok := extractIntField(raw["min_samples"]); ok { + cfg.MinSamples = n + } + if at, ok := raw["analysis_type"].(string); ok { + cfg.AnalysisType = at + } + if tagsRaw, ok := raw["tags"]; ok { + cfg.Tags = parseStringSliceAny(tagsRaw, nil) + } + cfg.Notify = extractExperimentNotify(raw["notify"]) +} + +func extractExperimentNotify(raw any) *ExperimentNotify { + notifyMap, ok := raw.(map[string]any) + if !ok { + return nil + } + notify := &ExperimentNotify{} + hasNotify := false + if n, ok := extractIntField(notifyMap["discussion"]); ok { + notify.Discussion = n + hasNotify = true + } + if n, ok := extractIntField(notifyMap["issue"]); ok { + notify.Issue = n + hasNotify = true + } + if hasNotify { + return notify } return nil } @@ -654,9 +678,26 @@ func (c *Compiler) buildPushExperimentsStateJob(data *WorkflowData) (*Job, error experimentsLog.Printf("Building push_experiments_state job (branch=%s)", experimentsBranchName(data.WorkflowID)) - var steps []string + steps := c.buildPushExperimentsSetupSteps(data) + steps = append(steps, buildPushExperimentsCheckoutStep()) + steps = append(steps, c.generateGitConfigurationSteps()...) + steps = append(steps, c.buildPushExperimentsDownloadStep(data)) + steps = append(steps, buildPushExperimentsStateStep(data)) + if c.actionMode.IsDev() { + steps = append(steps, c.generateRestoreActionsSetupStep()) + } + return &Job{ + Name: pushExperimentsStateJobName, + RunsOn: c.formatFrameworkJobRunsOn(data), + If: buildPushExperimentsStateCondition(), + Permissions: "permissions:\n contents: write", + Needs: []string{string(constants.ActivationJobName)}, + Steps: steps, + }, nil +} - // Setup step so the push_experiment_state.cjs script is available. +func (c *Compiler) buildPushExperimentsSetupSteps(data *WorkflowData) []string { + var steps []string setupActionRef := c.resolveActionReference("./actions/setup", data) if setupActionRef != "" || c.actionMode.IsScript() { steps = append(steps, c.generateCheckoutActionsFolder(data)...) @@ -664,34 +705,31 @@ func (c *Compiler) buildPushExperimentsStateJob(data *WorkflowData) (*Job, error parentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) steps = append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, traceID, parentSpanID)...) } + return steps +} - // Checkout step – configure git credentials without downloading workspace files. +func buildPushExperimentsCheckoutStep() string { var checkoutStep strings.Builder checkoutStep.WriteString(" - name: Checkout repository\n") fmt.Fprintf(&checkoutStep, " uses: %s\n", getActionPin("actions/checkout")) checkoutStep.WriteString(" with:\n") checkoutStep.WriteString(" persist-credentials: false\n") checkoutStep.WriteString(" sparse-checkout: .\n") - steps = append(steps, checkoutStep.String()) - - // Git configuration (author, email). - steps = append(steps, c.generateGitConfigurationSteps()...) + return checkoutStep.String() +} - // Download the experiment artifact uploaded by the activation job. - artifactName := experimentArtifactDownloadName(data) +func (c *Compiler) buildPushExperimentsDownloadStep(data *WorkflowData) string { var downloadStep strings.Builder downloadStep.WriteString(" - name: Download experiment artifact\n") fmt.Fprintf(&downloadStep, " uses: %s\n", c.getActionPin("actions/download-artifact")) downloadStep.WriteString(" continue-on-error: true\n") downloadStep.WriteString(" with:\n") - fmt.Fprintf(&downloadStep, " name: %s\n", artifactName) + fmt.Fprintf(&downloadStep, " name: %s\n", experimentArtifactDownloadName(data)) fmt.Fprintf(&downloadStep, " path: %s\n", experimentsCacheDir) - steps = append(steps, downloadStep.String()) - - // Push experiment state to the git branch via push_experiment_state.cjs. - // This helper uses pushSignedCommits to create verified (signed) commits. - branchName := experimentsBranchName(data.WorkflowID) + return downloadStep.String() +} +func buildPushExperimentsStateStep(data *WorkflowData) string { var pushStep strings.Builder pushStep.WriteString(" - name: Push experiment state to git\n") pushStep.WriteString(" id: push_experiments_state\n") @@ -702,37 +740,21 @@ func (c *Compiler) buildPushExperimentsStateJob(data *WorkflowData) (*Job, error pushStep.WriteString(" GITHUB_RUN_ID: ${{ github.run_id }}\n") pushStep.WriteString(" GITHUB_SERVER_URL: ${{ github.server_url }}\n") fmt.Fprintf(&pushStep, " GH_AW_EXPERIMENT_STATE_DIR: %s\n", experimentsCacheDir) - fmt.Fprintf(&pushStep, " GH_AW_EXPERIMENT_BRANCH: %s\n", branchName) + fmt.Fprintf(&pushStep, " GH_AW_EXPERIMENT_BRANCH: %s\n", experimentsBranchName(data.WorkflowID)) pushStep.WriteString(" with:\n") pushStep.WriteString(" script: |\n") pushStep.WriteString(" const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n") pushStep.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") pushStep.WriteString(" const { main } = require('" + SetupActionDestination + "/push_experiment_state.cjs');\n") pushStep.WriteString(" await main();\n") - steps = append(steps, pushStep.String()) - - // Restore the checkout in dev mode (same reason as push_repo_memory). - if c.actionMode.IsDev() { - steps = append(steps, c.generateRestoreActionsSetupStep()) - } + return pushStep.String() +} - // The push_experiments_state job runs after the activation job succeeds. - // It does not depend on the agent job because experiment state was fully resolved in activation. +func buildPushExperimentsStateCondition() string { activationSucceeded := BuildEquals( BuildPropertyAccess(fmt.Sprintf("needs.%s.result", constants.ActivationJobName)), BuildStringLiteral("success"), ) notCancelled := &NotNode{Child: BuildFunctionCall("cancelled")} - jobCondition := RenderCondition(BuildAnd(BuildAnd(BuildFunctionCall("always"), notCancelled), activationSucceeded)) - - job := &Job{ - Name: pushExperimentsStateJobName, - RunsOn: c.formatFrameworkJobRunsOn(data), - If: jobCondition, - Permissions: "permissions:\n contents: write", - Needs: []string{string(constants.ActivationJobName)}, - Steps: steps, - } - - return job, nil + return RenderCondition(BuildAnd(BuildAnd(BuildFunctionCall("always"), notCancelled), activationSucceeded)) } diff --git a/pkg/workflow/compiler_github_actions_steps.go b/pkg/workflow/compiler_github_actions_steps.go index 05cf47f3c26..d310c6a2b44 100644 --- a/pkg/workflow/compiler_github_actions_steps.go +++ b/pkg/workflow/compiler_github_actions_steps.go @@ -70,35 +70,42 @@ func generatePlaceholderSubstitutionStep(yaml *strings.Builder, expressionMappin yaml.WriteString(indent + " env:\n") yaml.WriteString(indent + " GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt\n") - // Add all environment variables - // For static values (wrapped in quotes), output them directly without ${{ }} - // For GitHub expressions, wrap them in ${{ }} + writePlaceholderSubstitutionEnv(yaml, expressionMappings, indent) + + yaml.WriteString(indent + " with:\n") + yaml.WriteString(indent + " script: |\n") + writePlaceholderSubstitutionScript(yaml, expressionMappings, indent) +} + +func writePlaceholderSubstitutionEnv(yaml *strings.Builder, expressionMappings []*ExpressionMapping, indent string) { for _, mapping := range expressionMappings { content := mapping.Content - // Check if this is a static quoted value (starts and ends with quotes) - if (strings.HasPrefix(content, "'") && strings.HasSuffix(content, "'")) || - (strings.HasPrefix(content, "\"") && strings.HasSuffix(content, "\"")) { - // Static value - output directly without ${{ }} wrapper - // Check if inner value is multi-line; if so use a YAML double-quoted scalar - // with escaped newlines to avoid invalid YAML. - innerValue := content[1 : len(content)-1] - if strings.Contains(innerValue, "\n") { - escaped := strings.ReplaceAll(innerValue, `\`, `\\`) - escaped = strings.ReplaceAll(escaped, `"`, `\"`) - escaped = strings.ReplaceAll(escaped, "\n", `\n`) - fmt.Fprintf(yaml, indent+" %s: \"%s\"\n", mapping.EnvVar, escaped) - } else { - fmt.Fprintf(yaml, indent+" %s: %s\n", mapping.EnvVar, content) - } + if isStaticQuotedPlaceholderValue(content) { + writeStaticPlaceholderEnv(yaml, mapping.EnvVar, content, indent) } else { - // GitHub expression - wrap in ${{ }} fmt.Fprintf(yaml, indent+" %s: ${{ %s }}\n", mapping.EnvVar, content) } } +} - yaml.WriteString(indent + " with:\n") - yaml.WriteString(indent + " script: |\n") +func isStaticQuotedPlaceholderValue(content string) bool { + return (strings.HasPrefix(content, "'") && strings.HasSuffix(content, "'")) || + (strings.HasPrefix(content, "\"") && strings.HasSuffix(content, "\"")) +} + +func writeStaticPlaceholderEnv(yaml *strings.Builder, envVar, content, indent string) { + innerValue := content[1 : len(content)-1] + if strings.Contains(innerValue, "\n") { + escaped := strings.ReplaceAll(innerValue, `\`, `\\`) + escaped = strings.ReplaceAll(escaped, `"`, `\"`) + escaped = strings.ReplaceAll(escaped, "\n", `\n`) + fmt.Fprintf(yaml, indent+" %s: \"%s\"\n", envVar, escaped) + return + } + fmt.Fprintf(yaml, indent+" %s: %s\n", envVar, content) +} +func writePlaceholderSubstitutionScript(yaml *strings.Builder, expressionMappings []*ExpressionMapping, indent string) { // Use setup_globals helper to make GitHub Actions objects available globally yaml.WriteString(indent + " const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n") yaml.WriteString(indent + " setupGlobals(core, github, context, exec, io, getOctokit);\n") diff --git a/pkg/workflow/compiler_github_mcp_steps.go b/pkg/workflow/compiler_github_mcp_steps.go index 605f10d9326..2ed80851b76 100644 --- a/pkg/workflow/compiler_github_mcp_steps.go +++ b/pkg/workflow/compiler_github_mcp_steps.go @@ -36,19 +36,43 @@ func (c *Compiler) generateGitHubMCPLockdownDetectionStep(yaml *strings.Builder, // at runtime with undefined step output errors. githubConfigLog.Print("Generating automatic guard policy determination step for GitHub MCP server") - // Resolve the latest version of actions/github-script + pinnedAction := resolveGitHubScriptActionPinForMCP(data) + configuredMinIntegrity, configuredRepos, privateToPublicFlowsAllow := githubMCPGuardPolicyEnvValues(githubTool) + + // Generate the step using the determine_automatic_lockdown.cjs action + yaml.WriteString(" - name: Determine automatic lockdown mode for GitHub MCP Server\n") + yaml.WriteString(" id: determine-automatic-lockdown\n") + fmt.Fprintf(yaml, " uses: %s\n", pinnedAction) + yaml.WriteString(" env:\n") + yaml.WriteString(" GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }}\n") + yaml.WriteString(" GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }}\n") + if configuredMinIntegrity != "" { + fmt.Fprintf(yaml, " GH_AW_GITHUB_MIN_INTEGRITY: %s\n", quoteYAMLEnvValue(configuredMinIntegrity)) + } + if configuredRepos != "" { + fmt.Fprintf(yaml, " GH_AW_GITHUB_REPOS: %s\n", quoteYAMLEnvValue(configuredRepos)) + } + if privateToPublicFlowsAllow { + yaml.WriteString(" GH_AW_PRIVATE_TO_PUBLIC_FLOWS: " + quoteYAMLEnvValue("allow") + "\n") + } + yaml.WriteString(" with:\n") + yaml.WriteString(" script: |\n") + yaml.WriteString(" const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs');\n") + yaml.WriteString(" await determineAutomaticLockdown(github, context, core);\n") +} + +func resolveGitHubScriptActionPinForMCP(data *WorkflowData) string { actionRepo := "actions/github-script" actionVersion := string(constants.DefaultGitHubScriptVersion) pinnedAction, err := getActionPinWithData(actionRepo, actionVersion, data) if err != nil { githubConfigLog.Printf("Failed to resolve %s@%s: %v", actionRepo, actionVersion, err) - // In strict mode, this error would have been returned by getActionPinWithData - // In normal mode, we fall back to using the version tag without pinning - pinnedAction = fmt.Sprintf("%s@%s", actionRepo, actionVersion) + return fmt.Sprintf("%s@%s", actionRepo, actionVersion) } + return pinnedAction +} - // Extract current guard policy configuration to pass as env vars so the step can - // detect whether each field is already configured and avoid overriding it. +func githubMCPGuardPolicyEnvValues(githubTool any) (string, string, bool) { configuredMinIntegrity := "" configuredRepos := "" privateToPublicFlowsAllow := false @@ -56,41 +80,16 @@ func (c *Compiler) generateGitHubMCPLockdownDetectionStep(yaml *strings.Builder, if v, exists := toolConfig["min-integrity"]; exists { configuredMinIntegrity = serializeEnvStringValue(v) } - // Support both 'allowed-repos' (preferred) and deprecated 'repos' if v, exists := toolConfig["allowed-repos"]; exists { configuredRepos = serializeEnvStringValue(v) } else if v, exists := toolConfig["repos"]; exists { configuredRepos = serializeEnvStringValue(v) } - // Detect private-to-public-flows: allow to inform the default repos value. - // When set to "allow", the user has explicitly opted in to cross-visibility data - // flows, so the repos default should be "all" rather than "public" even for - // public repositories. if ptpFlows, _ := toolConfig["private-to-public-flows"].(string); ptpFlows == "allow" { privateToPublicFlowsAllow = true } } - - // Generate the step using the determine_automatic_lockdown.cjs action - yaml.WriteString(" - name: Determine automatic lockdown mode for GitHub MCP Server\n") - yaml.WriteString(" id: determine-automatic-lockdown\n") - fmt.Fprintf(yaml, " uses: %s\n", pinnedAction) - yaml.WriteString(" env:\n") - yaml.WriteString(" GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }}\n") - yaml.WriteString(" GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }}\n") - if configuredMinIntegrity != "" { - fmt.Fprintf(yaml, " GH_AW_GITHUB_MIN_INTEGRITY: %s\n", quoteYAMLEnvValue(configuredMinIntegrity)) - } - if configuredRepos != "" { - fmt.Fprintf(yaml, " GH_AW_GITHUB_REPOS: %s\n", quoteYAMLEnvValue(configuredRepos)) - } - if privateToPublicFlowsAllow { - yaml.WriteString(" GH_AW_PRIVATE_TO_PUBLIC_FLOWS: " + quoteYAMLEnvValue("allow") + "\n") - } - yaml.WriteString(" with:\n") - yaml.WriteString(" script: |\n") - yaml.WriteString(" const determineAutomaticLockdown = require('${{ runner.temp }}/gh-aw/actions/determine_automatic_lockdown.cjs');\n") - yaml.WriteString(" await determineAutomaticLockdown(github, context, core);\n") + return configuredMinIntegrity, configuredRepos, privateToPublicFlowsAllow } // serializeEnvStringValue converts a workflow config value to a string suitable for a @@ -214,33 +213,7 @@ func (c *Compiler) generateParseGuardVarsStep(yaml *strings.Builder, data *Workf githubConfigLog.Print("Generating parse-guard-vars step for blocked-users, trusted-users and approval-labels") - // Determine the compile-time static values (or user expression) for each field. - // These come from the parsed tools config so we don't lose data from the raw map. - var blockedUsersExtra, trustedUsersExtra, approvalLabelsExtra string - - if data.ParsedTools != nil && data.ParsedTools.GitHub != nil { - gh := data.ParsedTools.GitHub - switch { - case len(gh.BlockedUsers) > 0: - // Static list from frontmatter — join as comma-separated for the env var. - blockedUsersExtra = strings.Join(gh.BlockedUsers, ",") - case gh.BlockedUsersExpr != "": - // User-provided GitHub Actions expression — passed verbatim; GHA evaluates it. - blockedUsersExtra = gh.BlockedUsersExpr - } - switch { - case len(gh.TrustedUsers) > 0: - trustedUsersExtra = strings.Join(gh.TrustedUsers, ",") - case gh.TrustedUsersExpr != "": - trustedUsersExtra = gh.TrustedUsersExpr - } - switch { - case len(gh.ApprovalLabels) > 0: - approvalLabelsExtra = strings.Join(gh.ApprovalLabels, ",") - case gh.ApprovalLabelsExpr != "": - approvalLabelsExtra = gh.ApprovalLabelsExpr - } - } + blockedUsersExtra, trustedUsersExtra, approvalLabelsExtra := githubMCPGuardListExtraEnvValues(data) yaml.WriteString(" - name: Parse integrity filter lists\n") yaml.WriteString(" id: parse-guard-vars\n") @@ -263,3 +236,24 @@ func (c *Compiler) generateParseGuardVarsStep(yaml *strings.Builder, data *Workf yaml.WriteString(" run: bash \"${RUNNER_TEMP}/gh-aw/actions/parse_guard_list.sh\"\n") } + +func githubMCPGuardListExtraEnvValues(data *WorkflowData) (string, string, string) { + if data.ParsedTools == nil || data.ParsedTools.GitHub == nil { + return "", "", "" + } + gh := data.ParsedTools.GitHub + return guardListExtraEnvValue(gh.BlockedUsers, gh.BlockedUsersExpr), + guardListExtraEnvValue(gh.TrustedUsers, gh.TrustedUsersExpr), + guardListExtraEnvValue(gh.ApprovalLabels, gh.ApprovalLabelsExpr) +} + +func guardListExtraEnvValue(values []string, expr string) string { + switch { + case len(values) > 0: + return strings.Join(values, ",") + case expr != "": + return expr + default: + return "" + } +} diff --git a/pkg/workflow/compiler_jobs.go b/pkg/workflow/compiler_jobs.go index a844b10bb2d..8df1a93420b 100644 --- a/pkg/workflow/compiler_jobs.go +++ b/pkg/workflow/compiler_jobs.go @@ -177,83 +177,78 @@ func (c *Compiler) getCustomJobsReferencedInPromptWithNoActivationDep(data *Work func (c *Compiler) buildJobs(data *WorkflowData, markdownPath string) error { compilerJobsLog.Printf("Building jobs for workflow: %s", markdownPath) - // Use the already-parsed frontmatter from WorkflowData (populated by ParseWorkflowFile / - // ParseWorkflowString) instead of re-reading and re-parsing the file on every compilation. - // Note: RawFrontmatter has already been through preprocessScheduleFields, so shorthand - // triggers (e.g. "on: daily") are already expanded into their structured form. - // The consumers (needsRoleCheck, hasWorkflowRunTrigger) only inspect event keys in the - // "on" field, which is exactly what we need here. frontmatter := data.RawFrontmatter - - // Extract lock filename for timestamp check lockFilename := filepath.Base(stringutil.MarkdownToLockFile(markdownPath)) - // Resolve custom safe-output actions early so that tool schemas (derived from action.yml) - // are available when buildMainJobWrapper → generateMCPSetup → generateToolsMetaJSON → - // generateDynamicTools runs. Without this early resolution the dynamic_tools entry for - // each action tool would have an empty schema because Inputs/ActionDescription are nil. - if data.SafeOutputs != nil && len(data.SafeOutputs.Actions) > 0 { - c.resolveAllActions(data, markdownPath) - } + c.resolveSafeOutputActionsForJobBuild(data, markdownPath) - // Build pre-activation and activation jobs _, activationJobCreated, err := c.buildPreActivationAndActivationJobs(data, frontmatter, lockFilename) if err != nil { return err } + if err := c.buildCoreWorkflowJobs(data, activationJobCreated, markdownPath); err != nil { + return err + } + if err := c.applyPostJobBuildAugmentations(data); err != nil { + return err + } - // Build main workflow job + compilerJobsLog.Print("Successfully built all jobs for workflow") + return nil +} + +func (c *Compiler) buildCoreWorkflowJobs(data *WorkflowData, activationJobCreated bool, markdownPath string) error { if err := c.buildMainJobWrapper(data, activationJobCreated); err != nil { return err } - - // Build safe outputs jobs if configured if err := c.buildSafeOutputsJobs(data, string(constants.AgentJobName), markdownPath); err != nil { return fmt.Errorf("failed to build safe outputs jobs: %w", err) } - - // Build BinEval evals job if evals are declared in frontmatter. - if evalsJob, err := c.buildEvalsJob(data); err != nil { - return fmt.Errorf("failed to build evals job: %w", err) - } else if evalsJob != nil { - if err := c.jobManager.AddJob(evalsJob); err != nil { - return fmt.Errorf("failed to add evals job: %w", err) - } + if err := c.addEvalsJobIfConfigured(data); err != nil { + return err } - - // Apply jobs..pre-steps customizations to already-created built-in jobs - // before processing non-built-in custom jobs. if err := c.applyBuiltinJobPreSteps(data); err != nil { return fmt.Errorf("failed to apply built-in job pre-steps: %w", err) } - - // Build additional custom jobs from frontmatter jobs section if len(data.Jobs) > 0 { compilerJobsLog.Printf("Building %d custom jobs from frontmatter", len(data.Jobs)) } if err := c.buildCustomJobs(data, activationJobCreated); err != nil { return fmt.Errorf("failed to build custom jobs: %w", err) } - - // Build memory management jobs (repo-memory and cache-memory) if err := c.buildMemoryManagementJobs(data); err != nil { return err } + return nil +} - // Apply additive jobs..needs augmentations once all jobs are created, - // so referenced custom/imported jobs can be validated against the final job set. +func (c *Compiler) applyPostJobBuildAugmentations(data *WorkflowData) error { if err := c.applyBuiltinJobNeedsAugmentations(data); err != nil { return fmt.Errorf("failed to apply built-in job needs augmentations: %w", err) } - - // Final pass: ensure conclusion job depends on ALL remaining workflow jobs. - // This guarantees conclusion always runs last, even for custom user-defined jobs - // (e.g. post-issue, super_linter) that were not explicitly added to its needs. if err := c.ensureConclusionIsLastJob(); err != nil { return err } + return nil +} - compilerJobsLog.Print("Successfully built all jobs for workflow") +func (c *Compiler) resolveSafeOutputActionsForJobBuild(data *WorkflowData, markdownPath string) { + if data.SafeOutputs != nil && len(data.SafeOutputs.Actions) > 0 { + c.resolveAllActions(data, markdownPath) + } +} + +func (c *Compiler) addEvalsJobIfConfigured(data *WorkflowData) error { + evalsJob, err := c.buildEvalsJob(data) + if err != nil { + return fmt.Errorf("failed to build evals job: %w", err) + } + if evalsJob == nil { + return nil + } + if err := c.jobManager.AddJob(evalsJob); err != nil { + return fmt.Errorf("failed to add evals job: %w", err) + } return nil } diff --git a/pkg/workflow/compiler_orchestrator_engine.go b/pkg/workflow/compiler_orchestrator_engine.go index e2f05c77b5d..a35f757dde2 100644 --- a/pkg/workflow/compiler_orchestrator_engine.go +++ b/pkg/workflow/compiler_orchestrator_engine.go @@ -317,18 +317,32 @@ func (c *Compiler) applyEngineImportDefaults( if engineConfig == nil { engineConfig = &EngineConfig{ID: engineSetting} } - if preservedMaxTurns != "" { - engineConfig.MaxTurns = preservedMaxTurns + applyPreservedEngineLimits(engineConfig, preservedMaxTurns, preservedMaxAICredits, preservedMaxRuns, preservedMaxTurnCacheMisses) + applyImportedEngineLimits(engineConfig, importsResult) + applyImportedEngineMCPDefaults(engineConfig, importsResult) + if engineConfig.Model == "" && importsResult.MergedEngineModel != "" { + engineConfig.Model = importsResult.MergedEngineModel + orchestratorEngineLog.Printf("Applied engine.model preference from import: %s", engineConfig.Model) + } + return engineConfig +} + +func applyPreservedEngineLimits(engineConfig *EngineConfig, maxTurns string, maxAICredits int64, maxRuns, maxTurnCacheMisses int) { + if maxTurns != "" { + engineConfig.MaxTurns = maxTurns } - if preservedMaxAICredits != 0 { - engineConfig.MaxAICredits = preservedMaxAICredits + if maxAICredits != 0 { + engineConfig.MaxAICredits = maxAICredits } - if preservedMaxRuns > 0 { - engineConfig.MaxRuns = preservedMaxRuns + if maxRuns > 0 { + engineConfig.MaxRuns = maxRuns } - if preservedMaxTurnCacheMisses > 0 { - engineConfig.MaxTurnCacheMisses = preservedMaxTurnCacheMisses + if maxTurnCacheMisses > 0 { + engineConfig.MaxTurnCacheMisses = maxTurnCacheMisses } +} + +func applyImportedEngineLimits(engineConfig *EngineConfig, importsResult *parser.ImportsResult) { if engineConfig.MaxTurns == "" && importsResult.MergedMaxTurns != "" { var importedMaxTurns any if err := json.Unmarshal([]byte(importsResult.MergedMaxTurns), &importedMaxTurns); err == nil { @@ -374,6 +388,9 @@ func (c *Compiler) applyEngineImportDefaults( } } } +} + +func applyImportedEngineMCPDefaults(engineConfig *EngineConfig, importsResult *parser.ImportsResult) { if engineConfig.MCPToolTimeout == "" && importsResult.MergedEngineMCPToolTimeout != "" { engineConfig.MCPToolTimeout = importsResult.MergedEngineMCPToolTimeout orchestratorEngineLog.Printf("Applied engine.mcp.tool-timeout from import: %s", engineConfig.MCPToolTimeout) @@ -382,11 +399,6 @@ func (c *Compiler) applyEngineImportDefaults( engineConfig.MCPSessionTimeout = importsResult.MergedEngineMCPSessionTimeout orchestratorEngineLog.Printf("Applied engine.mcp.session-timeout from import: %s", engineConfig.MCPSessionTimeout) } - if engineConfig.Model == "" && importsResult.MergedEngineModel != "" { - engineConfig.Model = importsResult.MergedEngineModel - orchestratorEngineLog.Printf("Applied engine.model preference from import: %s", engineConfig.Model) - } - return engineConfig } func (c *Compiler) resolveEngineRuntimeConfig(engineSetting string, engineConfig *EngineConfig) (CodingAgentEngine, []map[string]any, error) { diff --git a/pkg/workflow/compiler_orchestrator_frontmatter.go b/pkg/workflow/compiler_orchestrator_frontmatter.go index 5fe77deb281..33d97fd6770 100644 --- a/pkg/workflow/compiler_orchestrator_frontmatter.go +++ b/pkg/workflow/compiler_orchestrator_frontmatter.go @@ -84,98 +84,38 @@ func (c *Compiler) parseFrontmatterSection(markdownPath string) (*frontmatterPar orchestratorFrontmatterLog.Printf("Starting frontmatter parsing: %s", markdownPath) workflowLog.Printf("Reading file: %s", markdownPath) - // Clean the path to prevent path traversal issues (gosec G304) - // filepath.Clean removes ".." and other problematic path elements cleanPath := filepath.Clean(markdownPath) - - // Read the file - content, err := os.ReadFile(cleanPath) + content, err := readFrontmatterFile(cleanPath) if err != nil { - orchestratorFrontmatterLog.Printf("Failed to read file: %s, error: %v", cleanPath, err) - // Keep the user-facing message while avoiding exposure of os.PathError internals. - return nil, fmt.Errorf("failed to read file: %w", frontmatterReadError{message: err.Error()}) + return nil, err } contentString := string(content) - workflowLog.Printf("File size: %d bytes", len(content)) - // Parse frontmatter and markdown orchestratorFrontmatterLog.Printf("Parsing frontmatter from file: %s", cleanPath) result, err := parser.ExtractFrontmatterFromContent(contentString) if err != nil { - orchestratorFrontmatterLog.Printf("Frontmatter extraction failed: %v", err) - // Use FrontmatterStart from result if available, otherwise default to line 2 (after opening ---) - frontmatterStart := 2 - if result != nil && result.FrontmatterStart > 0 { - frontmatterStart = result.FrontmatterStart - } - return nil, c.createFrontmatterError(cleanPath, contentString, err, frontmatterStart) + return nil, c.handleFrontmatterExtractionError(cleanPath, contentString, result, err) } - if len(result.Frontmatter) == 0 { orchestratorFrontmatterLog.Print("No frontmatter found in file") return nil, errors.New("no frontmatter found") } - - // Preprocess schedule fields to convert human-friendly format to cron expressions if err := c.preprocessScheduleFields(result.Frontmatter, cleanPath, contentString); err != nil { orchestratorFrontmatterLog.Printf("Schedule preprocessing failed: %v", err) return nil, err } - // Create a copy of frontmatter without internal markers for schema validation - // Keep the original frontmatter with markers for YAML generation frontmatterForValidation := c.copyFrontmatterWithoutInternalMarkers(result.Frontmatter) - - // Check if user accidentally used "triggers:" instead of the correct "on:" keyword if _, hasTriggers := frontmatterForValidation["triggers"]; hasTriggers { return nil, fmt.Errorf("%s: invalid frontmatter key 'triggers:' — use 'on:' to define workflow triggers", cleanPath) } - // Check if "on" field is missing - if so, treat as a shared/imported workflow _, hasOnField := frontmatterForValidation["on"] if !hasOnField { - // Check if this is a redirect-only placeholder (has a redirect field but no 'on' trigger). - // Redirect-only files are distinct from regular shared workflows: they are placeholders - // that point to a workflow's new canonical location and are not intended to be imported. - // They occur when `gh aw add` downloads a workflow that has been moved but the redirect - // was not resolved to the full content during download. - if redirectVal, hasRedirect := frontmatterForValidation["redirect"]; hasRedirect { - if redirectStr, ok := redirectVal.(string); ok { - if redirectTarget := strings.TrimSpace(redirectStr); redirectTarget != "" { - detectionLog.Printf("Redirect-only workflow detected: redirect=%s", redirectTarget) - return &frontmatterParseResult{ - cleanPath: cleanPath, - content: content, - frontmatterResult: result, - frontmatterForValidation: frontmatterForValidation, - markdownDir: filepath.Dir(cleanPath), - isRedirectOnly: true, - redirectTarget: redirectTarget, - }, nil - } - } - } - - detectionLog.Printf("No 'on' field detected - treating as shared agentic workflow") - - // Validate as an included/shared workflow (uses main_workflow_schema with forbidden field checks) - if err := parser.ValidateIncludedFileFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { - orchestratorFrontmatterLog.Printf("Shared workflow validation failed: %v", err) - return nil, err - } - - return &frontmatterParseResult{ - cleanPath: cleanPath, - content: content, - frontmatterResult: result, - frontmatterForValidation: frontmatterForValidation, - markdownDir: filepath.Dir(cleanPath), - isSharedWorkflow: true, - }, nil + return handleSharedOrRedirectFrontmatter(cleanPath, content, result, frontmatterForValidation) } - // For main workflows (with 'on' field), markdown content is required if result.Markdown == "" { orchestratorFrontmatterLog.Print("No markdown content found for main workflow") return nil, errors.New("no markdown content found") @@ -186,84 +126,141 @@ func (c *Compiler) parseFrontmatterSection(markdownPath string) (*frontmatterPar return nil, err } - // Validate main workflow frontmatter contains only expected entries + if err := c.validateMainFrontmatterSection(cleanPath, result, frontmatterForValidation); err != nil { + return nil, err + } + + workflowLog.Printf("Frontmatter: %d chars, Markdown: %d chars", len(result.Frontmatter), len(result.Markdown)) + + return newFrontmatterParseResult(cleanPath, content, result, frontmatterForValidation, false, false, ""), nil +} + +func readFrontmatterFile(cleanPath string) ([]byte, error) { + content, err := os.ReadFile(cleanPath) + if err != nil { + orchestratorFrontmatterLog.Printf("Failed to read file: %s, error: %v", cleanPath, err) + return nil, fmt.Errorf("failed to read file: %w", frontmatterReadError{message: err.Error()}) + } + return content, nil +} + +func (c *Compiler) handleFrontmatterExtractionError(cleanPath, contentString string, result *parser.FrontmatterResult, err error) error { + orchestratorFrontmatterLog.Printf("Frontmatter extraction failed: %v", err) + frontmatterStart := 2 + if result != nil && result.FrontmatterStart > 0 { + frontmatterStart = result.FrontmatterStart + } + return c.createFrontmatterError(cleanPath, contentString, err, frontmatterStart) +} + +func handleSharedOrRedirectFrontmatter(cleanPath string, content []byte, result *parser.FrontmatterResult, frontmatterForValidation map[string]any) (*frontmatterParseResult, error) { + if redirectTarget := redirectOnlyTarget(frontmatterForValidation); redirectTarget != "" { + detectionLog.Printf("Redirect-only workflow detected: redirect=%s", redirectTarget) + return newFrontmatterParseResult(cleanPath, content, result, frontmatterForValidation, true, false, redirectTarget), nil + } + detectionLog.Printf("No 'on' field detected - treating as shared agentic workflow") + if err := parser.ValidateIncludedFileFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { + orchestratorFrontmatterLog.Printf("Shared workflow validation failed: %v", err) + return nil, err + } + return newFrontmatterParseResult(cleanPath, content, result, frontmatterForValidation, false, true, ""), nil +} + +func redirectOnlyTarget(frontmatterForValidation map[string]any) string { + if redirectVal, hasRedirect := frontmatterForValidation["redirect"]; hasRedirect { + if redirectStr, ok := redirectVal.(string); ok { + return strings.TrimSpace(redirectStr) + } + } + return "" +} + +func newFrontmatterParseResult(cleanPath string, content []byte, result *parser.FrontmatterResult, frontmatterForValidation map[string]any, isRedirectOnly, isSharedWorkflow bool, redirectTarget string) *frontmatterParseResult { + return &frontmatterParseResult{ + cleanPath: cleanPath, + content: content, + frontmatterResult: result, + frontmatterForValidation: frontmatterForValidation, + markdownDir: filepath.Dir(cleanPath), + isRedirectOnly: isRedirectOnly, + isSharedWorkflow: isSharedWorkflow, + redirectTarget: redirectTarget, + } +} + +func (c *Compiler) validateMainFrontmatterSection(cleanPath string, result *parser.FrontmatterResult, frontmatterForValidation map[string]any) error { + if err := validateMainFrontmatterSchema(cleanPath, frontmatterForValidation); err != nil { + return err + } + if err := c.validateMainFrontmatterEvents(frontmatterForValidation); err != nil { + return err + } + if err := validateRunsOn(frontmatterForValidation, cleanPath); err != nil { + orchestratorFrontmatterLog.Printf("runs-on validation failed: %v", err) + return err + } + return c.validateMainFrontmatterMarkdown(result.Markdown) +} + +func validateMainFrontmatterSchema(cleanPath string, frontmatterForValidation map[string]any) error { orchestratorFrontmatterLog.Printf("Validating main workflow frontmatter schema") if err := parser.ValidateMainWorkflowFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { orchestratorFrontmatterLog.Printf("Main workflow frontmatter validation failed: %v", err) - return nil, err + return err } if err := validateFrontmatterSkills(frontmatterForValidation); err != nil { orchestratorFrontmatterLog.Printf("Skills frontmatter validation failed: %v", err) - return nil, err + return err } + return nil +} - // Validate event filter mutual exclusivity (branches/branches-ignore, paths/paths-ignore) +func (c *Compiler) validateMainFrontmatterEvents(frontmatterForValidation map[string]any) error { if err := ValidateEventFilters(frontmatterForValidation); err != nil { orchestratorFrontmatterLog.Printf("Event filter validation failed: %v", err) - return nil, err + return err } - - // Validate that push triggers are scoped to specific branches or tags to prevent fan-out. - // In strict mode this is an error; in non-strict mode it is downgraded to a warning. - if err := ValidatePushBranchScope(frontmatterForValidation); err != nil { - if c.effectiveStrictMode(frontmatterForValidation) { - orchestratorFrontmatterLog.Printf("Push branch/tag scope validation failed: %v", err) - return nil, err - } - orchestratorFrontmatterLog.Printf("Push branch/tag scope warning (non-strict mode): %v", err) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(err.Error())) - c.IncrementWarningCount() + if err := c.validatePushBranchScope(frontmatterForValidation); err != nil { + return err } - - // Validate event type names in the 'on:' section for potential typos if err := ValidateEventTypes(frontmatterForValidation); err != nil { orchestratorFrontmatterLog.Printf("Event type validation failed: %v", err) - return nil, err + return err } - - // Validate glob pattern syntax in event filters (branches, tags, paths, etc.) if err := ValidateGlobPatterns(frontmatterForValidation); err != nil { orchestratorFrontmatterLog.Printf("Glob pattern validation failed: %v", err) - return nil, err + return err } + return nil +} - // Validate that the runs-on field does not specify unsupported runner types (e.g. macOS) - if err := validateRunsOn(frontmatterForValidation, cleanPath); err != nil { - orchestratorFrontmatterLog.Printf("runs-on validation failed: %v", err) - return nil, err +func (c *Compiler) validatePushBranchScope(frontmatterForValidation map[string]any) error { + if err := ValidatePushBranchScope(frontmatterForValidation); err != nil { + if c.effectiveStrictMode(frontmatterForValidation) { + orchestratorFrontmatterLog.Printf("Push branch/tag scope validation failed: %v", err) + return err + } + orchestratorFrontmatterLog.Printf("Push branch/tag scope warning (non-strict mode): %v", err) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(err.Error())) + c.IncrementWarningCount() } + return nil +} - // Validate that @include/@import directives are not used inside template regions - if err := validateNoIncludesInTemplateRegions(result.Markdown); err != nil { +func (c *Compiler) validateMainFrontmatterMarkdown(markdown string) error { + if err := validateNoIncludesInTemplateRegions(markdown); err != nil { orchestratorFrontmatterLog.Printf("Template region validation failed: %v", err) - return nil, fmt.Errorf("template region validation failed: %w", err) + return fmt.Errorf("template region validation failed: %w", err) } - - // Validate that pre-expanded __GH_AW_EXPERIMENTS_*__ placeholders are not used in template conditions - if err := validateNoPreExpandedExperimentPlaceholders(result.Markdown); err != nil { + if err := validateNoPreExpandedExperimentPlaceholders(markdown); err != nil { orchestratorFrontmatterLog.Printf("Pre-expanded experiment placeholder validation failed: %v", err) - return nil, fmt.Errorf("template condition validation failed: %w", err) + return fmt.Errorf("template condition validation failed: %w", err) } - - // Warn when experiment comparison expressions use double-quoted string literals. - // GitHub Actions expression syntax only supports single-quoted string literals, so - // the compiler converts double quotes to single quotes automatically — but authors - // should fix the source to use single quotes to keep it consistent with the output. - for _, w := range detectDoubleQuotedExperimentComparisons(result.Markdown) { + for _, w := range detectDoubleQuotedExperimentComparisons(markdown) { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(w)) c.IncrementWarningCount() } - - workflowLog.Printf("Frontmatter: %d chars, Markdown: %d chars", len(result.Frontmatter), len(result.Markdown)) - - return &frontmatterParseResult{ - cleanPath: cleanPath, - content: content, - frontmatterResult: result, - frontmatterForValidation: frontmatterForValidation, - markdownDir: filepath.Dir(cleanPath), - isSharedWorkflow: false, - }, nil + return nil } // copyFrontmatterWithoutInternalMarkers creates a copy of frontmatter without internal marker fields. diff --git a/pkg/workflow/compiler_orchestrator_workflow.go b/pkg/workflow/compiler_orchestrator_workflow.go index b8b942a97f3..a51face8e8d 100644 --- a/pkg/workflow/compiler_orchestrator_workflow.go +++ b/pkg/workflow/compiler_orchestrator_workflow.go @@ -400,35 +400,46 @@ func (c *Compiler) extractAdditionalConfigurations( safeOutputs *SafeOutputsConfig, ) error { orchestratorWorkflowLog.Print("Extracting additional configurations") - - // Extract cache-memory config and check for errors - cacheMemoryConfig, err := c.extractCacheMemoryConfigFromMap(tools) + toolsConfig, err := ParseToolsConfig(tools) if err != nil { return err } - workflowData.CacheMemoryConfig = cacheMemoryConfig + if err := c.extractMemoryConfigurations(tools, toolsConfig, workflowData); err != nil { + return err + } + c.extractCommandAndTriggerConfigurations(frontmatter, workflowData, importsResult) + workflowData.SafeOutputs = safeOutputs + allSafeOutputsConfigs, err := c.extractSafeOutputConfigurations(frontmatter, markdownDir, markdown, toolsConfig, workflowData, importsResult) + if err != nil { + return err + } + applyDefaultThreatDetectionFromImports(safeOutputs, workflowData, allSafeOutputsConfigs) + applyDefaultCreateIssue(workflowData) + applyTopLevelGitHubAppFallbacks(workflowData) + return c.extractExperimentAndEvalConfigurations(frontmatter, workflowData) +} - // Extract repo-memory config and check for errors - toolsConfig, err := ParseToolsConfig(tools) +func (c *Compiler) extractMemoryConfigurations(tools map[string]any, toolsConfig *ToolsConfig, workflowData *WorkflowData) error { + cacheMemoryConfig, err := c.extractCacheMemoryConfigFromMap(tools) if err != nil { return err } + workflowData.CacheMemoryConfig = cacheMemoryConfig repoMemoryConfig, err := c.extractRepoMemoryConfig(toolsConfig, workflowData.WorkflowID) if err != nil { return err } workflowData.RepoMemoryConfig = repoMemoryConfig + return nil +} - // Extract and process mcp-scripts and safe-outputs +func (c *Compiler) extractCommandAndTriggerConfigurations(frontmatter map[string]any, workflowData *WorkflowData, importsResult *parser.ImportsResult) { workflowData.Command, workflowData.CommandEvents, workflowData.CommandCentralized, workflowData.CommandPlaceholder = c.extractCommandConfig(frontmatter) workflowData.LabelCommand, workflowData.LabelCommandEvents, workflowData.LabelCommandDecentralized, workflowData.LabelCommandRemoveLabel = c.extractLabelCommandConfig(frontmatter) workflowData.Jobs = c.extractJobsFromFrontmatter(frontmatter) - - // Merge jobs from imported YAML workflows if importsResult.MergedJobs != "" && importsResult.MergedJobs != "{}" { workflowData.Jobs = c.mergeJobsFromYAMLImports(workflowData.Jobs, importsResult.MergedJobs) } - workflowData.Roles = c.extractRoles(frontmatter) workflowData.Bots = expandBotNames(mergeBots(c.extractBots(frontmatter), importsResult.MergedBots)) workflowData.LabelNames = c.extractLabelNames(frontmatter) @@ -440,111 +451,76 @@ func (c *Compiler) extractAdditionalConfigurations( workflowData.ActivationGitHubToken = c.resolveActivationGitHubToken(frontmatter, importsResult) workflowData.ActivationGitHubApp = c.resolveActivationGitHubApp(frontmatter, importsResult) workflowData.TopLevelGitHubApp = resolveTopLevelGitHubApp(frontmatter, importsResult) +} - // Use the already extracted output configuration - workflowData.SafeOutputs = safeOutputs - - // Extract comment-memory from tools and attach to safe-outputs configuration. - // comment-memory now belongs under tools: next to cache-memory and repo-memory. - commentMemoryConfig := c.extractCommentMemoryConfig(toolsConfig) - if commentMemoryConfig != nil { - if workflowData.SafeOutputs == nil { - workflowData.SafeOutputs = &SafeOutputsConfig{} - } - workflowData.SafeOutputs.CommentMemory = commentMemoryConfig - } - - // Extract mcp-scripts configuration +func (c *Compiler) extractSafeOutputConfigurations(frontmatter map[string]any, markdownDir, markdown string, toolsConfig *ToolsConfig, workflowData *WorkflowData, importsResult *parser.ImportsResult) ([]string, error) { + attachCommentMemoryConfig(c.extractCommentMemoryConfig(toolsConfig), workflowData) workflowData.MCPScripts = c.extractMCPScriptsConfig(frontmatter) - - // Merge mcp-scripts from imports if len(importsResult.MergedMCPScripts) > 0 { workflowData.MCPScripts = c.mergeMCPScripts(workflowData.MCPScripts, importsResult.MergedMCPScripts) } - - // Extract safe-jobs from safe-outputs.jobs location - topSafeJobs := extractSafeJobsFromFrontmatter(frontmatter) - - // Process @include directives to extract additional safe-outputs configurations includedSafeOutputsConfigs, err := parser.ExpandIncludesForSafeOutputs(markdown, markdownDir) if err != nil { - return fmt.Errorf("failed to expand includes for safe-outputs: %w", err) + return nil, fmt.Errorf("failed to expand includes for safe-outputs: %w", err) } + allSafeOutputsConfigs := append([]string{}, importsResult.MergedSafeOutputs...) + allSafeOutputsConfigs = append(allSafeOutputsConfigs, includedSafeOutputsConfigs...) + if err := c.applyIncludedSafeOutputConfigurations(frontmatter, workflowData, allSafeOutputsConfigs); err != nil { + return nil, err + } + return allSafeOutputsConfigs, nil +} - // Combine imported safe-outputs with included safe-outputs - var allSafeOutputsConfigs []string - if len(importsResult.MergedSafeOutputs) > 0 { - allSafeOutputsConfigs = append(allSafeOutputsConfigs, importsResult.MergedSafeOutputs...) +func attachCommentMemoryConfig(commentMemoryConfig *CommentMemoryConfig, workflowData *WorkflowData) { + if commentMemoryConfig == nil { + return } - if len(includedSafeOutputsConfigs) > 0 { - allSafeOutputsConfigs = append(allSafeOutputsConfigs, includedSafeOutputsConfigs...) + if workflowData.SafeOutputs == nil { + workflowData.SafeOutputs = &SafeOutputsConfig{} } + workflowData.SafeOutputs.CommentMemory = commentMemoryConfig +} - // Merge safe-jobs from all safe-outputs configurations (imported and included) - includedSafeJobs, err := c.mergeSafeJobsFromIncludedConfigs(topSafeJobs, allSafeOutputsConfigs) +func (c *Compiler) applyIncludedSafeOutputConfigurations(frontmatter map[string]any, workflowData *WorkflowData, allSafeOutputsConfigs []string) error { + includedSafeJobs, err := c.mergeSafeJobsFromIncludedConfigs(extractSafeJobsFromFrontmatter(frontmatter), allSafeOutputsConfigs) if err != nil { return fmt.Errorf("failed to merge safe-jobs from includes: %w", err) } - - // Merge app configuration from included safe-outputs configurations includedApp, err := c.mergeAppFromIncludedConfigs(workflowData.SafeOutputs, allSafeOutputsConfigs) if err != nil { return fmt.Errorf("failed to merge app from includes: %w", err) } - - // Ensure SafeOutputs exists and populate the Jobs field with merged jobs if workflowData.SafeOutputs == nil && len(includedSafeJobs) > 0 { workflowData.SafeOutputs = &SafeOutputsConfig{} } - // Always use the merged includedSafeJobs as it contains both main and imported jobs if workflowData.SafeOutputs != nil && len(includedSafeJobs) > 0 { workflowData.SafeOutputs.Jobs = includedSafeJobs } - - // Populate the App field if it's not set in the top-level workflow but is in an included config if workflowData.SafeOutputs != nil && workflowData.SafeOutputs.GitHubApp == nil && includedApp != nil { workflowData.SafeOutputs.GitHubApp = includedApp } - - // Merge safe-outputs types from imports. - // Pass the raw safe-outputs map from frontmatter so MergeSafeOutputs can distinguish - // between types the user explicitly configured and types that were auto-defaulted by - // extractSafeOutputsConfig. Without this, auto-defaults (e.g. threat-detection) would - // prevent imported configurations for those types from being merged. rawSafeOutputsMap, _ := frontmatter["safe-outputs"].(map[string]any) mergedSafeOutputs, err := c.MergeSafeOutputs(workflowData.SafeOutputs, allSafeOutputsConfigs, rawSafeOutputsMap) if err != nil { return fmt.Errorf("failed to merge safe-outputs from imports: %w", err) } workflowData.SafeOutputs = mergedSafeOutputs + return nil +} - // Apply default threat detection when safe-outputs came entirely from imports/includes - // (i.e. the main frontmatter has no safe-outputs: section). In this case the merge - // produces a non-nil SafeOutputs but leaves ThreatDetection nil, which would suppress - // the detection gate on the safe_outputs job. Mirroring the behaviour of - // extractSafeOutputsConfig for direct frontmatter declarations, we enable detection by - // default unless any imported config explicitly sets threat-detection: false. +func applyDefaultThreatDetectionFromImports(safeOutputs *SafeOutputsConfig, workflowData *WorkflowData, allSafeOutputsConfigs []string) { if safeOutputs == nil && workflowData.SafeOutputs != nil && workflowData.SafeOutputs.ThreatDetection == nil { if !isThreatDetectionExplicitlyDisabledInConfigs(allSafeOutputsConfigs) { orchestratorWorkflowLog.Print("Applying default threat-detection for safe-outputs assembled from imports/includes") workflowData.SafeOutputs.ThreatDetection = &ThreatDetectionConfig{} } } +} - // Auto-inject create-issues if safe-outputs is configured but has no non-builtin outputs. - // This ensures every workflow with safe-outputs has at least one meaningful action handler. - applyDefaultCreateIssue(workflowData) - - // Apply the top-level github-app as a fallback for all nested github-app token minting operations. - // This runs last so that all section-specific configurations have been resolved first. - applyTopLevelGitHubAppFallbacks(workflowData) - - // Extract experiments configuration once; derive the simple variants map from the configs. +func (c *Compiler) extractExperimentAndEvalConfigurations(frontmatter map[string]any, workflowData *WorkflowData) error { workflowData.ExperimentConfigs = extractExperimentConfigsFromFrontmatter(frontmatter) workflowData.Experiments = experimentVariantsFromConfigs(workflowData.ExperimentConfigs) workflowData.ExperimentsStorage = extractExperimentsStorageFromFrontmatter(frontmatter) - - // Extract BinEval evals configuration. evalsConfig, err := c.parseEvalsFromFrontmatter(frontmatter) if err != nil { return fmt.Errorf("invalid evals configuration: %w", err) @@ -553,7 +529,6 @@ func (c *Compiler) extractAdditionalConfigurations( if err := validateExperimentMetricReferences(workflowData.ExperimentConfigs, workflowData.Evals); err != nil { return fmt.Errorf("invalid experiments configuration: %w", err) } - return nil } @@ -624,95 +599,82 @@ func (c *Compiler) processOnSectionAndFilters( cleanPath string, ) error { orchestratorWorkflowLog.Print("Processing on section and filters") - - // Process stop-after configuration from the on: section - if err := c.processStopAfterConfiguration(frontmatter, workflowData, cleanPath); err != nil { + if err := c.processOnFilterConfigurations(frontmatter, workflowData, cleanPath); err != nil { return err } - - // Process skip-if-match configuration from the on: section - if err := c.processSkipIfMatchConfiguration(frontmatter, workflowData); err != nil { + if err := c.parseOnSection(frontmatter, workflowData, cleanPath); err != nil { return err } - - // Process skip-if-no-match configuration from the on: section - if err := c.processSkipIfNoMatchConfiguration(frontmatter, workflowData); err != nil { + if err := c.applyDefaults(workflowData, cleanPath); err != nil { return err } - - // Process skip-if-check-failing configuration from the on: section - if err := c.processSkipIfCheckFailingConfiguration(frontmatter, workflowData); err != nil { + c.applyPullRequestDraftFilter(workflowData, frontmatter) + c.applyPullRequestForkFilter(workflowData, frontmatter) + c.applyLabelFilter(workflowData, frontmatter) + if err := c.extractOnWorkflowControls(frontmatter, workflowData); err != nil { return err } + return nil +} - // Process manual-approval configuration from the on: section - if err := c.processManualApprovalConfiguration(frontmatter, workflowData); err != nil { +func (c *Compiler) processOnFilterConfigurations(frontmatter map[string]any, workflowData *WorkflowData, cleanPath string) error { + if err := c.processStopAfterConfiguration(frontmatter, workflowData, cleanPath); err != nil { return err } - - // Parse the "on" section for command triggers, reactions, and other events - if err := c.parseOnSection(frontmatter, workflowData, cleanPath); err != nil { + if err := c.processSkipIfMatchConfiguration(frontmatter, workflowData); err != nil { return err } - - // Apply defaults - if err := c.applyDefaults(workflowData, cleanPath); err != nil { + if err := c.processSkipIfNoMatchConfiguration(frontmatter, workflowData); err != nil { return err } + if err := c.processSkipIfCheckFailingConfiguration(frontmatter, workflowData); err != nil { + return err + } + return c.processManualApprovalConfiguration(frontmatter, workflowData) +} - // Apply pull request draft filter if specified - c.applyPullRequestDraftFilter(workflowData, frontmatter) - - // Apply pull request fork filter if specified - c.applyPullRequestForkFilter(workflowData, frontmatter) - - // Apply label filter if specified - c.applyLabelFilter(workflowData, frontmatter) - - // Extract on.steps for pre-activation step injection +func (c *Compiler) extractOnWorkflowControls(frontmatter map[string]any, workflowData *WorkflowData) error { onSteps, err := extractOnSteps(frontmatter) if err != nil { return err } - - // Apply action pinning to on.steps - if len(onSteps) > 0 { - anySteps := make([]any, len(onSteps)) - for i, s := range onSteps { - anySteps[i] = s - } - typedSteps, convErr := SliceToSteps(anySteps) - if convErr == nil { - typedSteps, convErr = applyActionPinsToTypedSteps(typedSteps, workflowData) - if convErr != nil { - return fmt.Errorf("on.steps: %w", convErr) - } - for i, s := range typedSteps { - onSteps[i] = s.ToMap() - } - } else { - orchestratorWorkflowLog.Printf("Failed to convert on.steps to typed steps for action pinning: %v", convErr) - } + if err := applyActionPinsToOnSteps(onSteps, workflowData); err != nil { + return err } - workflowData.OnSteps = onSteps - - // Extract on.permissions for pre-activation job permissions workflowData.OnPermissions = extractOnPermissions(frontmatter) - - // Extract on.needs for pre-activation/activation job dependencies onNeeds, err := extractOnNeeds(frontmatter) if err != nil { return err } workflowData.OnNeeds = onNeeds - - // Extract on.restore-memory to opt in to pre-activation memory restore for on.steps. onRestoreMemory, err := extractOnRestoreMemory(frontmatter) if err != nil { return err } workflowData.OnRestoreMemory = onRestoreMemory + return nil +} +func applyActionPinsToOnSteps(onSteps []map[string]any, workflowData *WorkflowData) error { + if len(onSteps) == 0 { + return nil + } + anySteps := make([]any, len(onSteps)) + for i, s := range onSteps { + anySteps[i] = s + } + typedSteps, convErr := SliceToSteps(anySteps) + if convErr != nil { + orchestratorWorkflowLog.Printf("Failed to convert on.steps to typed steps for action pinning: %v", convErr) + return nil + } + typedSteps, convErr = applyActionPinsToTypedSteps(typedSteps, workflowData) + if convErr != nil { + return fmt.Errorf("on.steps: %w", convErr) + } + for i, s := range typedSteps { + onSteps[i] = s.ToMap() + } return nil } diff --git a/pkg/workflow/compiler_safe_output_jobs.go b/pkg/workflow/compiler_safe_output_jobs.go index bf71421a987..b99277a94d0 100644 --- a/pkg/workflow/compiler_safe_output_jobs.go +++ b/pkg/workflow/compiler_safe_output_jobs.go @@ -20,144 +20,152 @@ func (c *Compiler) buildSafeOutputsJobs(data *WorkflowData, jobName, markdownPat } compilerSafeOutputJobsLog.Print("Building safe outputs jobs") - // Detection is always enabled for safe-outputs workflows unless threat-detection is explicitly - // disabled (threat-detection: false) or the engine is disabled with no custom steps - // (threat-detection: { engine: false } with no steps). ThreatDetection is nil only when - // explicitly disabled. When engine is false with no custom steps, the detection job has - // nothing to run so it is skipped entirely. threatDetectionEnabled := IsDetectionJobEnabled(data.SafeOutputs) + if err := c.addDetectionSafeOutputJob(data, threatDetectionEnabled); err != nil { + return err + } - // Build the separate detection job. Detection runs by default for all safe-outputs workflows - // and is only skipped when ThreatDetection is nil (i.e. threat-detection: false was set). - // The detection job runs after the agent job, downloads the agent artifact, - // and outputs detection_success and detection_conclusion for downstream jobs. - if threatDetectionEnabled { - detectionJob, err := c.buildDetectionJob(data) - if err != nil { - return fmt.Errorf("failed to build detection job: %w", err) - } - if detectionJob != nil { - if err := c.jobManager.AddJob(detectionJob); err != nil { - return fmt.Errorf("failed to add detection job: %w", err) - } - compilerSafeOutputJobsLog.Print("Added separate detection job") - } + safeOutputJobNames, err := c.buildPrimarySafeOutputJobNames(data, jobName, markdownPath, threatDetectionEnabled) + if err != nil { + return err } - // Track safe output job names to establish dependencies for conclusion job - var safeOutputJobNames []string + unlockJob, err := c.addUnlockSafeOutputJob(data, threatDetectionEnabled) + if err != nil { + return err + } + + return c.addConclusionSafeOutputJob(data, jobName, safeOutputJobNames, unlockJob) +} - // Build consolidated safe outputs job containing all safe output operations as steps +func (c *Compiler) addDetectionSafeOutputJob(data *WorkflowData, threatDetectionEnabled bool) error { + if !threatDetectionEnabled { + return nil + } + detectionJob, err := c.buildDetectionJob(data) + if err != nil { + return fmt.Errorf("failed to build detection job: %w", err) + } + if detectionJob == nil { + return nil + } + if err := c.jobManager.AddJob(detectionJob); err != nil { + return fmt.Errorf("failed to add detection job: %w", err) + } + compilerSafeOutputJobsLog.Print("Added separate detection job") + return nil +} + +func (c *Compiler) buildPrimarySafeOutputJobNames(data *WorkflowData, jobName, markdownPath string, threatDetectionEnabled bool) ([]string, error) { + var safeOutputJobNames []string consolidatedJob, consolidatedStepNames, err := c.buildConsolidatedSafeOutputsJob(data, jobName, markdownPath) if err != nil { - return fmt.Errorf("failed to build consolidated safe outputs job: %w", err) + return nil, fmt.Errorf("failed to build consolidated safe outputs job: %w", err) } if consolidatedJob != nil { if err := c.jobManager.AddJob(consolidatedJob); err != nil { - return fmt.Errorf("failed to add consolidated safe outputs job: %w", err) + return nil, fmt.Errorf("failed to add consolidated safe outputs job: %w", err) } safeOutputJobNames = append(safeOutputJobNames, consolidatedJob.Name) compilerSafeOutputJobsLog.Printf("Added consolidated safe outputs job with %d steps: %v", len(consolidatedStepNames), consolidatedStepNames) } - // Build safe-jobs if configured - // Safe-jobs should depend on agent job (always) AND detection job (if threat detection is enabled) - // These custom safe-jobs should also be included in the conclusion job's dependencies safeJobNames, err := c.buildSafeJobs(data, threatDetectionEnabled) if err != nil { - return fmt.Errorf("failed to build safe-jobs: %w", err) + return nil, fmt.Errorf("failed to build safe-jobs: %w", err) } - // Add custom safe-job names to the list of safe output jobs safeOutputJobNames = append(safeOutputJobNames, safeJobNames...) compilerSafeOutputJobsLog.Printf("Added %d custom safe-job names to conclusion dependencies", len(safeJobNames)) - // Build upload_assets job as a separate job if configured - // This needs to be separate from the consolidated safe_outputs job because it requires: - // 1. Git configuration for pushing to orphaned branches - // 2. Checkout with proper credentials - // 3. Different permissions (contents: write) + safeOutputJobNames, err = c.appendUploadAssetsJobName(data, jobName, threatDetectionEnabled, safeOutputJobNames) + if err != nil { + return nil, err + } + safeOutputJobNames, err = c.appendCodeScanningUploadJobName(data, safeOutputJobNames) + if err != nil { + return nil, err + } + callWorkflowJobNames, err := c.buildCallWorkflowJobs(data, markdownPath) + if err != nil { + return nil, fmt.Errorf("failed to build call-workflow fan-out jobs: %w", err) + } + safeOutputJobNames = append(safeOutputJobNames, callWorkflowJobNames...) + compilerSafeOutputJobsLog.Printf("Added %d call-workflow fan-out jobs", len(callWorkflowJobNames)) + return safeOutputJobNames, nil +} + +func (c *Compiler) appendUploadAssetsJobName(data *WorkflowData, jobName string, threatDetectionEnabled bool, safeOutputJobNames []string) ([]string, error) { if data.SafeOutputs != nil && data.SafeOutputs.UploadAssets != nil { compilerSafeOutputJobsLog.Print("Building separate upload_assets job") uploadAssetsJob, err := c.buildUploadAssetsJob(data, jobName, threatDetectionEnabled) if err != nil { - return fmt.Errorf("failed to build upload_assets job: %w", err) + return nil, fmt.Errorf("failed to build upload_assets job: %w", err) } if err := c.jobManager.AddJob(uploadAssetsJob); err != nil { - return fmt.Errorf("failed to add upload_assets job: %w", err) + return nil, fmt.Errorf("failed to add upload_assets job: %w", err) } safeOutputJobNames = append(safeOutputJobNames, uploadAssetsJob.Name) compilerSafeOutputJobsLog.Printf("Added separate upload_assets job") } + return safeOutputJobNames, nil +} - // Build upload_code_scanning_sarif job as a separate job if create-code-scanning-alert is configured. - // This job runs after safe_outputs and only when the safe_outputs job exported a SARIF file. - // It is separate to avoid the checkout step (needed to restore HEAD to github.sha) from - // interfering with other safe-output operations in the consolidated safe_outputs job. +func (c *Compiler) appendCodeScanningUploadJobName(data *WorkflowData, safeOutputJobNames []string) ([]string, error) { if data.SafeOutputs != nil && data.SafeOutputs.CreateCodeScanningAlerts != nil && !isHandlerStaged(templatableBoolIsTrue(data.SafeOutputs.Staged), data.SafeOutputs.CreateCodeScanningAlerts.Staged) { compilerSafeOutputJobsLog.Print("Building separate upload_code_scanning_sarif job") codeScanningJob, err := c.buildCodeScanningUploadJob(data) if err != nil { - return fmt.Errorf("failed to build upload_code_scanning_sarif job: %w", err) + return nil, fmt.Errorf("failed to build upload_code_scanning_sarif job: %w", err) } if err := c.jobManager.AddJob(codeScanningJob); err != nil { - return fmt.Errorf("failed to add upload_code_scanning_sarif job: %w", err) + return nil, fmt.Errorf("failed to add upload_code_scanning_sarif job: %w", err) } safeOutputJobNames = append(safeOutputJobNames, codeScanningJob.Name) compilerSafeOutputJobsLog.Printf("Added separate upload_code_scanning_sarif job") } + return safeOutputJobNames, nil +} - // Build conditional call-workflow fan-out jobs if configured. - // Each allowed worker gets its own `uses:` job with an `if:` condition that - // checks whether safe_outputs selected it. Only one runs per execution. - callWorkflowJobNames, err := c.buildCallWorkflowJobs(data, markdownPath) - if err != nil { - return fmt.Errorf("failed to build call-workflow fan-out jobs: %w", err) - } - safeOutputJobNames = append(safeOutputJobNames, callWorkflowJobNames...) - compilerSafeOutputJobsLog.Printf("Added %d call-workflow fan-out jobs", len(callWorkflowJobNames)) - - // Build dedicated unlock job if lock-for-agent is enabled - // This job is separate from conclusion to ensure it always runs, even if other jobs fail - // It depends on agent and detection (if enabled) to run after workflow execution completes +func (c *Compiler) addUnlockSafeOutputJob(data *WorkflowData, threatDetectionEnabled bool) (*Job, error) { unlockJob, err := c.buildUnlockJob(data, threatDetectionEnabled) if err != nil { - return fmt.Errorf("failed to build unlock job: %w", err) + return nil, fmt.Errorf("failed to build unlock job: %w", err) } if unlockJob != nil { if err := c.jobManager.AddJob(unlockJob); err != nil { - return fmt.Errorf("failed to add unlock job: %w", err) + return nil, fmt.Errorf("failed to add unlock job: %w", err) } compilerSafeOutputJobsLog.Print("Added dedicated unlock job") } + return unlockJob, nil +} - // Build conclusion job if add-comment is configured OR if command trigger is configured with reactions - // This job runs last, after all safe output jobs (and push_repo_memory if configured), to update the activation comment on failure - // The buildConclusionJob function itself will decide whether to create the job based on the configuration +func (c *Compiler) addConclusionSafeOutputJob(data *WorkflowData, jobName string, safeOutputJobNames []string, unlockJob *Job) error { conclusionJob, err := c.buildConclusionJob(data, jobName, safeOutputJobNames) if err != nil { return fmt.Errorf("failed to build conclusion job: %w", err) } if conclusionJob != nil { - // If unlock job exists, conclusion should depend on it to run after unlock completes - if unlockJob != nil { - conclusionJob.Needs = append(conclusionJob.Needs, "unlock") - compilerSafeOutputJobsLog.Printf("Added unlock job dependency to conclusion job") - } - // If push_repo_memory job exists, conclusion should depend on it - // Check if the job was already created (it's created in buildJobs) - if _, exists := c.jobManager.GetJob("push_repo_memory"); exists { - conclusionJob.Needs = append(conclusionJob.Needs, "push_repo_memory") - compilerSafeOutputJobsLog.Printf("Added push_repo_memory dependency to conclusion job") - } + c.addConclusionJobDependencies(conclusionJob, unlockJob) if err := c.jobManager.AddJob(conclusionJob); err != nil { return fmt.Errorf("failed to add conclusion job: %w", err) } } - return nil } +func (c *Compiler) addConclusionJobDependencies(conclusionJob *Job, unlockJob *Job) { + if unlockJob != nil { + conclusionJob.Needs = append(conclusionJob.Needs, "unlock") + compilerSafeOutputJobsLog.Printf("Added unlock job dependency to conclusion job") + } + if _, exists := c.jobManager.GetJob("push_repo_memory"); exists { + conclusionJob.Needs = append(conclusionJob.Needs, "push_repo_memory") + compilerSafeOutputJobsLog.Printf("Added push_repo_memory dependency to conclusion job") + } +} + // buildCallWorkflowJobs generates one conditional `uses:` job per workflow in the // call-workflow allowlist. Each job: // - depends on safe_outputs @@ -190,163 +198,154 @@ func (c *Compiler) buildCallWorkflowJobs(data *WorkflowData, markdownPath string var jobNames []string for _, workflowName := range config.Workflows { - // Build the job name: "call-{sanitized-workflow-name}" - // sanitizeJobName normalizes underscores to hyphens (NormalizeSafeOutputIdentifier + dash conversion) - sanitizedName := sanitizeJobName(workflowName) - jobName := "call-" + sanitizedName - - // Determine the relative path to the worker workflow file - workflowPath, ok := config.WorkflowFiles[workflowName] - if !ok || workflowPath == "" { - // Fallback: construct path from name - workflowPath = fmt.Sprintf("./.github/workflows/%s.lock.yml", workflowName) + jobName, err := c.addCallWorkflowJob(data, config, workflowName, markdownPath) + if err != nil { + return nil, err } + jobNames = append(jobNames, jobName) + } - // Build the with: block. Forward one entry per declared workflow_call input - // on the worker, derived from the payload, so that worker steps can reference - // inputs. directly without parsing JSON. The canonical `payload` - // envelope is only forwarded when the worker explicitly declares a `payload` - // input; GitHub Actions rejects a `uses:` step that passes an input the - // called workflow does not declare, so it must not be added unconditionally. - jobNeeds := []string{"safe_outputs"} - with := map[string]any{} - - if markdownPath != "" { - fileResult, findErr := findWorkflowFile(workflowName, markdownPath) - if findErr != nil { - compilerSafeOutputJobsLog.Printf("Warning: could not find worker workflow file for '%s': %v. "+ - "Typed inputs will not be forwarded in the with: block.", workflowName, findErr) - } else { - var workflowInputs map[string]any - var inputErr error - switch { - case fileResult.lockExists: - workflowInputs, inputErr = extractWorkflowCallInputs(fileResult.lockPath) - case fileResult.ymlExists: - workflowInputs, inputErr = extractWorkflowCallInputs(fileResult.ymlPath) - case fileResult.mdExists: - workflowInputs, inputErr = extractMDWorkflowCallInputs(fileResult.mdPath) - default: - compilerSafeOutputJobsLog.Printf("Warning: no worker file found for '%s'; "+ - "typed inputs will not be forwarded in the with: block.", workflowName) - } - if inputErr != nil { - compilerSafeOutputJobsLog.Printf("Warning: could not extract workflow_call inputs for '%s': %v. "+ - "Typed inputs will not be forwarded in the with: block.", workflowName, inputErr) - } else if workflowInputs != nil { - typedInputCount := 0 - for inputName := range workflowInputs { - if inputName == "payload" { - // The worker explicitly declares the canonical payload - // envelope input; forward the raw transport rather than a - // fromJSON expression. - with["payload"] = "${{ needs.safe_outputs.outputs.call_workflow_payload }}" - continue - } - with[inputName] = buildCallWorkflowInputExpression(inputName) - typedInputCount++ - } - compilerSafeOutputJobsLog.Printf("Forwarding %d typed inputs for call-workflow job '%s'", typedInputCount, jobName) - } - - } - } + return jobNames, nil +} - callJob := &Job{ - Name: jobName, - Needs: jobNeeds, - If: fmt.Sprintf("needs.safe_outputs.outputs.call_workflow_name == '%s'", workflowName), - Uses: workflowPath, - With: with, - } +func (c *Compiler) addCallWorkflowJob(data *WorkflowData, config *CallWorkflowConfig, workflowName, markdownPath string) (string, error) { + jobName := "call-" + sanitizeJobName(workflowName) + workflowPath := callWorkflowPath(config, workflowName) + callJob := &Job{ + Name: jobName, + Needs: []string{"safe_outputs"}, + If: fmt.Sprintf("needs.safe_outputs.outputs.call_workflow_name == '%s'", workflowName), + Uses: workflowPath, + With: buildCallWorkflowWith(workflowName, jobName, markdownPath), + } + configureCallWorkflowSecrets(callJob, workflowName, jobName, markdownPath) + configureCallWorkflowPermissions(callJob, data, workflowName, jobName, markdownPath) + if err := c.jobManager.AddJob(callJob); err != nil { + return "", fmt.Errorf("failed to add call-workflow job '%s': %w", jobName, err) + } + compilerSafeOutputJobsLog.Printf("Added call-workflow job: %s (uses: %s)", jobName, workflowPath) + return jobName, nil +} - // Infer the minimal set of secrets required by the worker workflow so we can - // pass them explicitly instead of using secrets: inherit. This requires the - // worker to have been compiled with on.workflow_call.secrets declarations. - // If the worker has not yet been compiled (no .lock.yml/.yml), or declares no - // secrets, fall back to secrets: inherit for backward compatibility. - if markdownPath != "" { - workerSecrets, secretsErr := extractCallWorkflowSecrets(workflowName, markdownPath) - if secretsErr != nil { - compilerSafeOutputJobsLog.Printf("Warning: could not extract secrets for call-workflow job '%s': %v. "+ - "Falling back to secrets: inherit.", jobName, secretsErr) - callJob.SecretsInherit = true - } else if len(workerSecrets) == 0 { - // No secrets were extracted from the worker. This can mean either the - // worker declares no workflow_call secrets or its compiled file was not - // found yet. Fall back to secrets: inherit for backward compatibility. - compilerSafeOutputJobsLog.Printf("No workflow_call secrets could be extracted for worker '%s' "+ - "(worker may declare none or its compiled file may not exist yet); using secrets: inherit", workflowName) - callJob.SecretsInherit = true - } else { - // Map each declared secret explicitly. - callJob.Secrets = make(map[string]string, len(workerSecrets)) - for _, s := range workerSecrets { - callJob.Secrets[s] = fmt.Sprintf("${{ secrets.%s }}", s) - } - compilerSafeOutputJobsLog.Printf("Mapped %d explicit secrets for call-workflow job '%s'", len(workerSecrets), jobName) - } - } else { - callJob.SecretsInherit = true - } +func callWorkflowPath(config *CallWorkflowConfig, workflowName string) string { + if workflowPath, ok := config.WorkflowFiles[workflowName]; ok && workflowPath != "" { + return workflowPath + } + return fmt.Sprintf("./.github/workflows/%s.lock.yml", workflowName) +} - // Compute the call- job's permission envelope as the union of: - // 1. The caller's own declared permissions (the base scope the caller controls). - // 2. The worker's job-level permissions (the minimum the worker needs to run). - // GitHub validates reusable workflow calls against the caller job's declared - // permissions and rejects the run at startup when the caller grants less than - // the worker requires. Taking the union ensures the call job always holds a - // sufficient grant without requiring the caller's markdown to enumerate every - // permission the worker needs. - callerPerms := data.CachedPermissions - if callerPerms == nil { - callerPerms = NewPermissionsParser(data.Permissions).ToPermissions() +func buildCallWorkflowWith(workflowName, jobName, markdownPath string) map[string]any { + with := map[string]any{} + if markdownPath == "" { + return with + } + workflowInputs := extractCallWorkflowInputsForJob(workflowName, markdownPath) + if workflowInputs == nil { + return with + } + typedInputCount := 0 + for inputName := range workflowInputs { + if inputName == "payload" { + with["payload"] = "${{ needs.safe_outputs.outputs.call_workflow_payload }}" + continue } + with[inputName] = buildCallWorkflowInputExpression(inputName) + typedInputCount++ + } + compilerSafeOutputJobsLog.Printf("Forwarding %d typed inputs for call-workflow job '%s'", typedInputCount, jobName) + return with +} - effectivePerms := callerPerms - var importedPerms *callWorkflowPermissionImport - var permErr error - if markdownPath != "" { - importedPerms, permErr = extractCallWorkflowPermissionImport(workflowName, markdownPath) - if permErr != nil { - // Non-fatal: log and continue. The worker file may not exist yet (it may be - // compiled in the same batch), in which case we fall back to the caller's - // own declared permissions. - compilerSafeOutputJobsLog.Printf("Could not extract worker permissions for call-workflow job '%s' (falling back to caller-only permissions): %v", jobName, permErr) - } else if importedPerms != nil && importedPerms.permissions != nil { - // Compute the union by merging caller and worker permissions into a - // fresh map-based Permissions. Starting from a blank slate (rather - // than a clone of callerPerms) ensures shorthand values like - // "read-all" are correctly expanded before the worker's explicit - // scopes are merged on top — cloning a shorthand Permissions and then - // merging a map into it would clear the shorthand field without first - // expanding it, silently dropping the caller's baseline grant. - merged := NewPermissions() - merged.Merge(callerPerms) - merged.Merge(importedPerms.permissions) - effectivePerms = merged - compilerSafeOutputJobsLog.Printf("Merged caller and worker permissions for call-workflow job '%s'", jobName) - } - } +func extractCallWorkflowInputsForJob(workflowName, markdownPath string) map[string]any { + fileResult, findErr := findWorkflowFile(workflowName, markdownPath) + if findErr != nil { + compilerSafeOutputJobsLog.Printf("Warning: could not find worker workflow file for '%s': %v. "+ + "Typed inputs will not be forwarded in the with: block.", workflowName, findErr) + return nil + } + workflowInputs, inputErr := extractCallWorkflowInputsFromFileResult(fileResult, workflowName) + if inputErr != nil { + compilerSafeOutputJobsLog.Printf("Warning: could not extract workflow_call inputs for '%s': %v. "+ + "Typed inputs will not be forwarded in the with: block.", workflowName, inputErr) + return nil + } + return workflowInputs +} - if effectivePerms != nil { - rendered := effectivePerms.RenderToYAML() - if rendered != "" { - callJob.PermissionsComment = buildCallWorkflowPermissionsComment(workflowName, importedPerms) - callJob.Permissions = rendered - compilerSafeOutputJobsLog.Printf("Set permissions on call-workflow job '%s': %s", jobName, rendered) - } - } +func extractCallWorkflowInputsFromFileResult(fileResult *findWorkflowFileResult, workflowName string) (map[string]any, error) { + switch { + case fileResult.lockExists: + return extractWorkflowCallInputs(fileResult.lockPath) + case fileResult.ymlExists: + return extractWorkflowCallInputs(fileResult.ymlPath) + case fileResult.mdExists: + return extractMDWorkflowCallInputs(fileResult.mdPath) + default: + compilerSafeOutputJobsLog.Printf("Warning: no worker file found for '%s'; typed inputs will not be forwarded in the with: block.", workflowName) + return nil, nil + } +} - if err := c.jobManager.AddJob(callJob); err != nil { - return nil, fmt.Errorf("failed to add call-workflow job '%s': %w", jobName, err) - } +func configureCallWorkflowSecrets(callJob *Job, workflowName, jobName, markdownPath string) { + if markdownPath == "" { + callJob.SecretsInherit = true + return + } + workerSecrets, secretsErr := extractCallWorkflowSecrets(workflowName, markdownPath) + if secretsErr != nil { + compilerSafeOutputJobsLog.Printf("Warning: could not extract secrets for call-workflow job '%s': %v. Falling back to secrets: inherit.", jobName, secretsErr) + callJob.SecretsInherit = true + return + } + if len(workerSecrets) == 0 { + compilerSafeOutputJobsLog.Printf("No workflow_call secrets could be extracted for worker '%s' (worker may declare none or its compiled file may not exist yet); using secrets: inherit", workflowName) + callJob.SecretsInherit = true + return + } + callJob.Secrets = make(map[string]string, len(workerSecrets)) + for _, s := range workerSecrets { + callJob.Secrets[s] = fmt.Sprintf("${{ secrets.%s }}", s) + } + compilerSafeOutputJobsLog.Printf("Mapped %d explicit secrets for call-workflow job '%s'", len(workerSecrets), jobName) +} - jobNames = append(jobNames, jobName) - compilerSafeOutputJobsLog.Printf("Added call-workflow job: %s (uses: %s)", jobName, workflowPath) +func configureCallWorkflowPermissions(callJob *Job, data *WorkflowData, workflowName, jobName, markdownPath string) { + callerPerms := data.CachedPermissions + if callerPerms == nil { + callerPerms = NewPermissionsParser(data.Permissions).ToPermissions() + } + effectivePerms, importedPerms := callWorkflowEffectivePermissions(callerPerms, workflowName, jobName, markdownPath) + if effectivePerms == nil { + return } + rendered := effectivePerms.RenderToYAML() + if rendered == "" { + return + } + callJob.PermissionsComment = buildCallWorkflowPermissionsComment(workflowName, importedPerms) + callJob.Permissions = rendered + compilerSafeOutputJobsLog.Printf("Set permissions on call-workflow job '%s': %s", jobName, rendered) +} - return jobNames, nil +func callWorkflowEffectivePermissions(callerPerms *Permissions, workflowName, jobName, markdownPath string) (*Permissions, *callWorkflowPermissionImport) { + effectivePerms := callerPerms + var importedPerms *callWorkflowPermissionImport + if markdownPath == "" { + return effectivePerms, importedPerms + } + importedPerms, permErr := extractCallWorkflowPermissionImport(workflowName, markdownPath) + if permErr != nil { + compilerSafeOutputJobsLog.Printf("Could not extract worker permissions for call-workflow job '%s' (falling back to caller-only permissions): %v", jobName, permErr) + return effectivePerms, importedPerms + } + if importedPerms != nil && importedPerms.permissions != nil { + merged := NewPermissions() + merged.Merge(callerPerms) + merged.Merge(importedPerms.permissions) + effectivePerms = merged + compilerSafeOutputJobsLog.Printf("Merged caller and worker permissions for call-workflow job '%s'", jobName) + } + return effectivePerms, importedPerms } func buildCallWorkflowInputExpression(inputName string) string { diff --git a/pkg/workflow/compiler_safe_outputs_job.go b/pkg/workflow/compiler_safe_outputs_job.go index 362917c7161..66c4ba5ad37 100644 --- a/pkg/workflow/compiler_safe_outputs_job.go +++ b/pkg/workflow/compiler_safe_outputs_job.go @@ -172,7 +172,15 @@ func (c *Compiler) buildConsolidatedSafeOutputsJob(data *WorkflowData, mainJobNa func (c *Compiler) buildSafeOutputsSetupAndDownloadSteps(data *WorkflowData, agentArtifactPrefix string) ([]string, error) { var steps []string - // Add setup action to copy JavaScript files + steps = c.appendSafeOutputsSetupActionSteps(steps, data) + steps = appendSafeOutputsTelemetryMaskSteps(steps, data) + steps = append(steps, buildAgentOutputDownloadSteps(agentArtifactPrefix, c.getActionPin)...) + steps = c.appendSafeOutputsPatchCheckoutSteps(steps, data, agentArtifactPrefix) + steps = append(steps, generateGHESHostConfigurationStep()) + return c.appendUserProvidedSafeOutputSteps(steps, data) +} + +func (c *Compiler) appendSafeOutputsSetupActionSteps(steps []string, data *WorkflowData) []string { setupActionRef := c.resolveActionReference("./actions/setup", data) if setupActionRef != "" || c.actionMode.IsScript() { // For dev mode (local action path), checkout the actions folder first @@ -186,7 +194,10 @@ func (c *Compiler) buildSafeOutputsSetupAndDownloadSteps(data *WorkflowData, age safeOutputsParentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) steps = append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, enableArtifactClient, safeOutputsTraceID, safeOutputsParentSpanID)...) } + return steps +} +func appendSafeOutputsTelemetryMaskSteps(steps []string, data *WorkflowData) []string { // Mask OTLP telemetry headers immediately after setup so authentication tokens cannot // leak into runner debug logs for any subsequent step in the safe outputs job. if isOTLPHeadersPresent(data) { @@ -196,11 +207,10 @@ func (c *Compiler) buildSafeOutputsSetupAndDownloadSteps(data *WorkflowData, age if isOTLPAttributesPresent(data) { steps = append(steps, generateOTLPAttributesMaskStep()) } + return steps +} - // Add artifact download steps after setup. - // In workflow_call context, use the per-invocation prefix to avoid artifact name clashes. - steps = append(steps, buildAgentOutputDownloadSteps(agentArtifactPrefix, c.getActionPin)...) - +func (c *Compiler) appendSafeOutputsPatchCheckoutSteps(steps []string, data *WorkflowData, agentArtifactPrefix string) []string { // Add patch artifact download if create-pull-request or push-to-pull-request-branch is enabled // Both of these safe outputs require the patch file to apply changes // Download from unified agent artifact (prefixed in workflow_call context) @@ -221,14 +231,10 @@ func (c *Compiler) buildSafeOutputsSetupAndDownloadSteps(data *WorkflowData, age checkoutSteps := c.buildSharedPRCheckoutSteps(data) steps = append(steps, checkoutSteps...) } + return steps +} - // Configure GH_HOST for GHES/GHEC compatibility. - // The safe-outputs job runs as an independent GitHub Actions job and does not - // inherit GITHUB_ENV from the agent job. User-provided steps (below) and future - // safe-output handlers that invoke the gh CLI need GH_HOST to target the - // correct enterprise instance. - steps = append(steps, generateGHESHostConfigurationStep()) - +func (c *Compiler) appendUserProvidedSafeOutputSteps(steps []string, data *WorkflowData) ([]string, error) { // Add user-provided steps after checkout/setup, before safe-output code if len(data.SafeOutputs.Steps) > 0 { consolidatedSafeOutputsJobLog.Printf("Adding %d user-provided steps to safe-outputs job", len(data.SafeOutputs.Steps)) @@ -266,222 +272,198 @@ func (c *Compiler) buildSafeOutputsHandlerOutputsAndActionSteps(data *WorkflowDa outputs := make(map[string]string) var safeOutputStepNames []string - // Note: Unlock step has been moved to dedicated unlock job - // The safe_outputs job now depends on the unlock job, so the issue - // will already be unlocked when this job runs - - // === Build safe output steps === - // - // IMPORTANT: Step order matters for safe outputs that depend on each other. - // The execution order ensures dependencies are satisfied: - // 1. Handler Manager - processes create_issue, update_issue, add_comment, etc. - // 2. Assign To Agent - assigns issue to agent (after handler managers complete) - // 3. Create Agent Session - creates agent session (after assignment) - // - // Note: All project-related operations (create_project, update_project, create_project_status_update) - // are now handled by the unified handler in the handler manager step. - - // Check if any handler-manager-supported types are enabled - hasHandlerManagerTypes := data.SafeOutputs.CreateIssues != nil || - data.SafeOutputs.AddComments != nil || - data.SafeOutputs.CreateDiscussions != nil || - data.SafeOutputs.CloseIssues != nil || - data.SafeOutputs.CloseDiscussions != nil || - data.SafeOutputs.AddLabels != nil || - data.SafeOutputs.RemoveLabels != nil || - data.SafeOutputs.UpdateIssues != nil || - data.SafeOutputs.UpdateDiscussions != nil || - data.SafeOutputs.LinkSubIssue != nil || - data.SafeOutputs.UpdateRelease != nil || - data.SafeOutputs.CreatePullRequestReviewComments != nil || - data.SafeOutputs.SubmitPullRequestReview != nil || - data.SafeOutputs.ReplyToPullRequestReviewComment != nil || - data.SafeOutputs.ResolvePullRequestReviewThread != nil || - data.SafeOutputs.CreatePullRequests != nil || - data.SafeOutputs.PushToPullRequestBranch != nil || - data.SafeOutputs.UpdatePullRequests != nil || - data.SafeOutputs.ClosePullRequests != nil || - data.SafeOutputs.MarkPullRequestAsReadyForReview != nil || - data.SafeOutputs.HideComment != nil || - data.SafeOutputs.SetIssueType != nil || - data.SafeOutputs.SetIssueField != nil || - data.SafeOutputs.DispatchWorkflow != nil || - data.SafeOutputs.CallWorkflow != nil || - data.SafeOutputs.CreateCodeScanningAlerts != nil || - data.SafeOutputs.AutofixCodeScanningAlert != nil || - data.SafeOutputs.CreateCheckRun != nil || - data.SafeOutputs.MissingTool != nil || - data.SafeOutputs.MissingData != nil || - data.SafeOutputs.AssignToAgent != nil || // assign_to_agent is now handled by the handler manager - data.SafeOutputs.CreateAgentSessions != nil || // create_agent_session is now handled by the handler manager - data.SafeOutputs.UploadArtifact != nil || // upload_artifact is handled inline in the handler loop - len(data.SafeOutputs.Scripts) > 0 || // Custom scripts run in the handler loop - len(data.SafeOutputs.Actions) > 0 // Custom actions need handler to export their payloads - - // Note: All project-related operations are now handled by the unified handler. - // The project handler manager has been removed. - - // Add custom script files step (writes inline scripts to the actions folder) - // This must run before the handler manager step so the files are available for require() - if len(data.SafeOutputs.Scripts) > 0 { - consolidatedSafeOutputsJobLog.Printf("Adding setup step for %d custom safe-output script(s)", len(data.SafeOutputs.Scripts)) - scriptSetupSteps, err := buildCustomScriptFilesStep(data.SafeOutputs.Scripts) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to build custom script files step: %w", err) - } - steps = append(steps, scriptSetupSteps...) + var err error + steps, err = c.appendCustomSafeOutputScriptSteps(steps, data) + if err != nil { + return nil, nil, nil, err } + steps = c.appendUploadArtifactStagingDownloadStep(steps, data, agentArtifactPrefix) + steps, safeOutputStepNames, err = c.appendHandlerManagerSafeOutputSteps(steps, data, outputs, safeOutputStepNames) + if err != nil { + return nil, nil, nil, err + } + steps = c.appendSarifArtifactSafeOutputSteps(steps, data, outputs, agentArtifactPrefix) + steps, safeOutputStepNames = c.appendCustomActionSafeOutputSteps(steps, data, markdownPath, safeOutputStepNames) + addNamedSafeOutputJobOutputs(data, outputs) - // Download the upload-artifact staging artifact before the handler manager runs so that - // the upload_artifact handler (which runs inline in the handler loop) can access the files. - if data.SafeOutputs.UploadArtifact != nil { - consolidatedSafeOutputsJobLog.Print("Adding upload-artifact staging download step") - stagingArtifactName := agentArtifactPrefix + SafeOutputsUploadArtifactStagingArtifactName - steps = append(steps, - " - name: Download upload-artifact staging\n", - " continue-on-error: true\n", - fmt.Sprintf(" uses: %s\n", c.getActionPin("actions/download-artifact")), - " with:\n", - fmt.Sprintf(" name: %s\n", stagingArtifactName), - fmt.Sprintf(" path: %s\n", artifactStagingDirExpr), - ) + return steps, outputs, safeOutputStepNames, nil +} + +func hasHandlerManagerSafeOutputTypes(safeOutputs *SafeOutputsConfig) bool { + return safeOutputs.CreateIssues != nil || + safeOutputs.AddComments != nil || + safeOutputs.CreateDiscussions != nil || + safeOutputs.CloseIssues != nil || + safeOutputs.CloseDiscussions != nil || + safeOutputs.AddLabels != nil || + safeOutputs.RemoveLabels != nil || + safeOutputs.UpdateIssues != nil || + safeOutputs.UpdateDiscussions != nil || + safeOutputs.LinkSubIssue != nil || + safeOutputs.UpdateRelease != nil || + safeOutputs.CreatePullRequestReviewComments != nil || + safeOutputs.SubmitPullRequestReview != nil || + safeOutputs.ReplyToPullRequestReviewComment != nil || + safeOutputs.ResolvePullRequestReviewThread != nil || + safeOutputs.CreatePullRequests != nil || + safeOutputs.PushToPullRequestBranch != nil || + safeOutputs.UpdatePullRequests != nil || + safeOutputs.ClosePullRequests != nil || + safeOutputs.MarkPullRequestAsReadyForReview != nil || + safeOutputs.HideComment != nil || + safeOutputs.SetIssueType != nil || + safeOutputs.SetIssueField != nil || + safeOutputs.DispatchWorkflow != nil || + safeOutputs.CallWorkflow != nil || + safeOutputs.CreateCodeScanningAlerts != nil || + safeOutputs.AutofixCodeScanningAlert != nil || + safeOutputs.CreateCheckRun != nil || + safeOutputs.MissingTool != nil || + safeOutputs.MissingData != nil || + safeOutputs.AssignToAgent != nil || + safeOutputs.CreateAgentSessions != nil || + safeOutputs.UploadArtifact != nil || + len(safeOutputs.Scripts) > 0 || + len(safeOutputs.Actions) > 0 +} + +func (c *Compiler) appendCustomSafeOutputScriptSteps(steps []string, data *WorkflowData) ([]string, error) { + if len(data.SafeOutputs.Scripts) == 0 { + return steps, nil + } + consolidatedSafeOutputsJobLog.Printf("Adding setup step for %d custom safe-output script(s)", len(data.SafeOutputs.Scripts)) + scriptSetupSteps, err := buildCustomScriptFilesStep(data.SafeOutputs.Scripts) + if err != nil { + return nil, fmt.Errorf("failed to build custom script files step: %w", err) } + return append(steps, scriptSetupSteps...), nil +} - // 1. Handler Manager step (processes create_issue, update_issue, add_comment, assign_to_agent, - // upload_artifact, etc.) - // This processes all safe output types that are handled by the unified handler - // Critical for workflows that create projects and then add issues/PRs to those projects - if hasHandlerManagerTypes { - consolidatedSafeOutputsJobLog.Print("Using handler manager for safe outputs") - handlerManagerSteps, err := c.buildHandlerManagerStep(data) - if err != nil { - return nil, nil, nil, err - } - steps = append(steps, handlerManagerSteps...) - safeOutputStepNames = append(safeOutputStepNames, "process_safe_outputs") - - // Add outputs from handler manager - outputs["process_safe_outputs_temporary_id_map"] = "${{ steps.process_safe_outputs.outputs.temporary_id_map }}" - outputs["process_safe_outputs_processed_count"] = "${{ steps.process_safe_outputs.outputs.processed_count }}" - outputs["create_discussion_errors"] = "${{ steps.process_safe_outputs.outputs.create_discussion_errors }}" - outputs["create_discussion_error_count"] = "${{ steps.process_safe_outputs.outputs.create_discussion_error_count }}" - outputs["code_push_failure_errors"] = "${{ steps.process_safe_outputs.outputs.code_push_failure_errors }}" - outputs["code_push_failure_count"] = "${{ steps.process_safe_outputs.outputs.code_push_failure_count }}" - - // Note: Permissions are now computed centrally by ComputePermissionsForSafeOutputs() - // at the start of this function to ensure consistent permission calculation - - // Export assign_to_agent outputs from the handler manager step - if data.SafeOutputs.AssignToAgent != nil { - consolidatedSafeOutputsJobLog.Print("Exposing assign_to_agent outputs from handler manager") - outputs["assign_to_agent_assigned"] = "${{ steps.process_safe_outputs.outputs.assign_to_agent_assigned }}" - outputs["assign_to_agent_assignment_errors"] = "${{ steps.process_safe_outputs.outputs.assign_to_agent_assignment_errors }}" - outputs["assign_to_agent_assignment_error_count"] = "${{ steps.process_safe_outputs.outputs.assign_to_agent_assignment_error_count }}" - } +func (c *Compiler) appendUploadArtifactStagingDownloadStep(steps []string, data *WorkflowData, agentArtifactPrefix string) []string { + if data.SafeOutputs.UploadArtifact == nil { + return steps + } + consolidatedSafeOutputsJobLog.Print("Adding upload-artifact staging download step") + stagingArtifactName := agentArtifactPrefix + SafeOutputsUploadArtifactStagingArtifactName + return append(steps, + " - name: Download upload-artifact staging\n", + " continue-on-error: true\n", + fmt.Sprintf(" uses: %s\n", c.getActionPin("actions/download-artifact")), + " with:\n", + fmt.Sprintf(" name: %s\n", stagingArtifactName), + fmt.Sprintf(" path: %s\n", artifactStagingDirExpr), + ) +} - // Export create_agent_session outputs from the handler manager step - if data.SafeOutputs.CreateAgentSessions != nil { - consolidatedSafeOutputsJobLog.Print("Exposing create_agent_session outputs from handler manager") - outputs["create_agent_session_session_number"] = "${{ steps.process_safe_outputs.outputs.session_number }}" - outputs["create_agent_session_session_url"] = "${{ steps.process_safe_outputs.outputs.session_url }}" - } +func (c *Compiler) appendHandlerManagerSafeOutputSteps(steps []string, data *WorkflowData, outputs map[string]string, safeOutputStepNames []string) ([]string, []string, error) { + if !hasHandlerManagerSafeOutputTypes(data.SafeOutputs) { + return steps, safeOutputStepNames, nil + } + consolidatedSafeOutputsJobLog.Print("Using handler manager for safe outputs") + handlerManagerSteps, err := c.buildHandlerManagerStep(data) + if err != nil { + return nil, nil, err + } + steps = append(steps, handlerManagerSteps...) + safeOutputStepNames = append(safeOutputStepNames, "process_safe_outputs") + addHandlerManagerSafeOutputJobOutputs(data, outputs) + return steps, safeOutputStepNames, nil +} - // Export upload_artifact outputs. - // The handler sets slot_N_* outputs on the process_safe_outputs step; we expose - // them as upload_artifact_slot_N_* job outputs for external consumers. - // The actual artifact uploads are performed directly by the JS handler via - // @actions/artifact REST API — no additional YAML steps are required. - if data.SafeOutputs.UploadArtifact != nil { - consolidatedSafeOutputsJobLog.Print("Exposing upload_artifact outputs from handler manager") - cfg := data.SafeOutputs.UploadArtifact - outputs["upload_artifact_count"] = "${{ steps.process_safe_outputs.outputs.upload_artifact_count }}" - for i := range cfg.MaxUploads { - outputs[fmt.Sprintf("upload_artifact_slot_%d_tmp_id", i)] = fmt.Sprintf("${{ steps.process_safe_outputs.outputs.slot_%d_tmp_id }}", i) - } - } +func addHandlerManagerSafeOutputJobOutputs(data *WorkflowData, outputs map[string]string) { + outputs["process_safe_outputs_temporary_id_map"] = "${{ steps.process_safe_outputs.outputs.temporary_id_map }}" + outputs["process_safe_outputs_processed_count"] = "${{ steps.process_safe_outputs.outputs.processed_count }}" + outputs["create_discussion_errors"] = "${{ steps.process_safe_outputs.outputs.create_discussion_errors }}" + outputs["create_discussion_error_count"] = "${{ steps.process_safe_outputs.outputs.create_discussion_error_count }}" + outputs["code_push_failure_errors"] = "${{ steps.process_safe_outputs.outputs.code_push_failure_errors }}" + outputs["code_push_failure_count"] = "${{ steps.process_safe_outputs.outputs.code_push_failure_count }}" + addAssignmentSafeOutputJobOutputs(data, outputs) + addUploadArtifactSafeOutputJobOutputs(data, outputs) +} +func addAssignmentSafeOutputJobOutputs(data *WorkflowData, outputs map[string]string) { + if data.SafeOutputs.AssignToAgent != nil { + consolidatedSafeOutputsJobLog.Print("Exposing assign_to_agent outputs from handler manager") + outputs["assign_to_agent_assigned"] = "${{ steps.process_safe_outputs.outputs.assign_to_agent_assigned }}" + outputs["assign_to_agent_assignment_errors"] = "${{ steps.process_safe_outputs.outputs.assign_to_agent_assignment_errors }}" + outputs["assign_to_agent_assignment_error_count"] = "${{ steps.process_safe_outputs.outputs.assign_to_agent_assignment_error_count }}" } + if data.SafeOutputs.CreateAgentSessions != nil { + consolidatedSafeOutputsJobLog.Print("Exposing create_agent_session outputs from handler manager") + outputs["create_agent_session_session_number"] = "${{ steps.process_safe_outputs.outputs.session_number }}" + outputs["create_agent_session_session_url"] = "${{ steps.process_safe_outputs.outputs.session_url }}" + } +} - // 2. SARIF output — expose sarif_file from the handler so the dedicated - // upload_code_scanning_sarif job (built in buildCodeScanningUploadJob) can access it - // via needs.safe_outputs.outputs.sarif_file and decide whether to run. - // Additionally, upload the SARIF file as a GitHub Actions artifact so the upload job - // can retrieve the actual file (job outputs only carry the path string; the file itself - // only exists in the safe_outputs job workspace). - // NOTE: We do NOT export checkout_token as a job output. GitHub Actions masks output - // values that contain secret references, so the downstream job would receive an empty - // string. The upload job computes the token directly from static secret references. - if data.SafeOutputs.CreateCodeScanningAlerts != nil && !isHandlerStaged(c.trialMode || templatableBoolIsTrue(data.SafeOutputs.Staged), data.SafeOutputs.CreateCodeScanningAlerts.Staged) { - consolidatedSafeOutputsJobLog.Print("Exposing sarif_file output for upload_code_scanning_sarif job") - outputs["sarif_file"] = "${{ steps.process_safe_outputs.outputs.sarif_file }}" - - // Upload the SARIF file as an artifact so the upload_code_scanning_sarif job - // (which runs in a separate, fresh workspace) can download and process it. - steps = append(steps, buildSarifArtifactUploadStep(agentArtifactPrefix, c.getActionPin)...) - } - - // 3. Custom action steps — compiler-generated steps for each configured safe-output action. - // These steps run after the handler manager, which processes the agent payload and exports - // a JSON payload output for each action tool call. Each step is guarded by an `if:` condition - // that checks whether the handler manager exported a payload for this action. - if len(data.SafeOutputs.Actions) > 0 { - // resolveAllActions was already called early in buildJobs (before generateToolsMetaJSON) - // so action configs already have Inputs/ActionDescription populated. We only call it - // again here as a safety net in case compileSafeOutputsJob is called independently. - c.resolveAllActions(data, markdownPath) - - actionStepYAML := c.buildActionSteps(data) - steps = append(steps, actionStepYAML...) - - // Register each action as having a handler manager output - for actionName := range data.SafeOutputs.Actions { - normalizedName := stringutil.NormalizeSafeOutputIdentifier(actionName) - safeOutputStepNames = append(safeOutputStepNames, "action_"+normalizedName) - } +func addUploadArtifactSafeOutputJobOutputs(data *WorkflowData, outputs map[string]string) { + if data.SafeOutputs.UploadArtifact == nil { + return + } + consolidatedSafeOutputsJobLog.Print("Exposing upload_artifact outputs from handler manager") + cfg := data.SafeOutputs.UploadArtifact + outputs["upload_artifact_count"] = "${{ steps.process_safe_outputs.outputs.upload_artifact_count }}" + for i := range cfg.MaxUploads { + outputs[fmt.Sprintf("upload_artifact_slot_%d_tmp_id", i)] = fmt.Sprintf("${{ steps.process_safe_outputs.outputs.slot_%d_tmp_id }}", i) + } +} + +func (c *Compiler) appendSarifArtifactSafeOutputSteps(steps []string, data *WorkflowData, outputs map[string]string, agentArtifactPrefix string) []string { + if data.SafeOutputs.CreateCodeScanningAlerts == nil || + isHandlerStaged(c.trialMode || templatableBoolIsTrue(data.SafeOutputs.Staged), data.SafeOutputs.CreateCodeScanningAlerts.Staged) { + return steps } + consolidatedSafeOutputsJobLog.Print("Exposing sarif_file output for upload_code_scanning_sarif job") + outputs["sarif_file"] = "${{ steps.process_safe_outputs.outputs.sarif_file }}" + return append(steps, buildSarifArtifactUploadStep(agentArtifactPrefix, c.getActionPin)...) +} - // The outputs and permissions are configured in the handler manager section above +func (c *Compiler) appendCustomActionSafeOutputSteps(steps []string, data *WorkflowData, markdownPath string, safeOutputStepNames []string) ([]string, []string) { + if len(data.SafeOutputs.Actions) == 0 { + return steps, safeOutputStepNames + } + c.resolveAllActions(data, markdownPath) + steps = append(steps, c.buildActionSteps(data)...) + for actionName := range data.SafeOutputs.Actions { + normalizedName := stringutil.NormalizeSafeOutputIdentifier(actionName) + safeOutputStepNames = append(safeOutputStepNames, "action_"+normalizedName) + } + return steps, safeOutputStepNames +} + +func addNamedSafeOutputJobOutputs(data *WorkflowData, outputs map[string]string) { if data.SafeOutputs.AddReviewer != nil { outputs["add_reviewer_reviewers_added"] = "${{ steps.process_safe_outputs.outputs.reviewers_added }}" } - - // The outputs and permissions are configured in the handler manager section above if data.SafeOutputs.AssignMilestone != nil { outputs["assign_milestone_milestone_assigned"] = "${{ steps.process_safe_outputs.outputs.milestone_assigned }}" } - - // The outputs and permissions are configured in the handler manager section above if data.SafeOutputs.AssignToUser != nil { outputs["assign_to_user_assigned"] = "${{ steps.process_safe_outputs.outputs.assigned }}" } + addCreatedItemSafeOutputJobOutputs(data, outputs) +} - // Individual named outputs for first-created items (enables workflow_call consumers to access results) +func addCreatedItemSafeOutputJobOutputs(data *WorkflowData, outputs map[string]string) { if data.SafeOutputs.CreateIssues != nil { outputs["created_issue_number"] = "${{ steps.process_safe_outputs.outputs.created_issue_number }}" outputs["created_issue_url"] = "${{ steps.process_safe_outputs.outputs.created_issue_url }}" } - if data.SafeOutputs.CreatePullRequests != nil { outputs["created_pr_number"] = "${{ steps.process_safe_outputs.outputs.created_pr_number }}" outputs["created_pr_url"] = "${{ steps.process_safe_outputs.outputs.created_pr_url }}" } - if data.SafeOutputs.AddComments != nil { outputs["comment_id"] = "${{ steps.process_safe_outputs.outputs.comment_id }}" outputs["comment_url"] = "${{ steps.process_safe_outputs.outputs.comment_url }}" } - if data.SafeOutputs.PushToPullRequestBranch != nil { outputs["push_commit_sha"] = "${{ steps.process_safe_outputs.outputs.push_commit_sha }}" outputs["push_commit_url"] = "${{ steps.process_safe_outputs.outputs.push_commit_url }}" } - if data.SafeOutputs.CallWorkflow != nil { outputs["call_workflow_name"] = "${{ steps.process_safe_outputs.outputs.call_workflow_name }}" outputs["call_workflow_payload"] = "${{ steps.process_safe_outputs.outputs.call_workflow_payload }}" } - - return steps, outputs, safeOutputStepNames, nil } // buildSafeOutputsJobFromParts finalizes the step list (app-token insertion, token invalidation, @@ -503,27 +485,27 @@ func (c *Compiler) buildSafeOutputsJobFromParts( opts buildSafeOutputsJobFromPartsOptions, ) (*Job, []string, error) { data := opts.data - mainJobName := opts.mainJobName - markdownPath := opts.markdownPath - agentArtifactPrefix := opts.agentArtifactPrefix steps := opts.steps outputs := opts.outputs - safeOutputStepNames := opts.safeOutputStepNames - permissions := opts.permissions - threatDetectionEnabled := opts.threatDetectionEnabled - // Build all preamble token minting steps to insert before checkout/safe-output steps. - var preambleTokenSteps []string + preambleTokenSteps := c.buildSafeOutputsPreambleTokenSteps(data, opts.permissions, outputs) + steps = c.insertSafeOutputsPreambleTokenSteps(data, opts.agentArtifactPrefix, steps, preambleTokenSteps) + steps = c.appendSafeOutputsFinalSteps(data, opts.agentArtifactPrefix, steps) + + jobCondition := buildSafeOutputsJobCondition(data, opts.threatDetectionEnabled) + needs := c.buildSafeOutputsNeeds(data, opts.mainJobName, opts.threatDetectionEnabled) + workflowID := GetWorkflowIDFromPath(opts.markdownPath) + job := c.buildSafeOutputsJob(data, opts.permissions, outputs, steps, needs, jobCondition, workflowID) - // Add main GitHub App token minting step if configured. + consolidatedSafeOutputsJobLog.Printf("Built consolidated safe outputs job with %d steps", len(opts.safeOutputStepNames)) + + return job, opts.safeOutputStepNames, nil +} + +func (c *Compiler) buildSafeOutputsPreambleTokenSteps(data *WorkflowData, permissions *Permissions, outputs map[string]string) []string { + var preambleTokenSteps []string if data.SafeOutputs.GitHubApp != nil { - // Track whether the app token minting succeeded so the conclusion job can surface - // authentication errors in the failure issue. outputs["app_token_minting_failed"] = "${{ steps.safe-outputs-app-token.outcome == 'failure' }}" - - // For workflow_call relay workflows, scope the token to the platform repo name only - // (not the full slug) because actions/create-github-app-token expects repo names - // without the owner prefix when `owner` is also set. var appTokenFallbackRepo string if hasWorkflowCallTrigger(data.On) { appTokenFallbackRepo = "${{ needs.activation.outputs.target_repo_name }}" @@ -535,126 +517,87 @@ func (c *Compiler) buildSafeOutputsJobFromParts( inferSingleCheckoutRepositoryForGitHubAppOwner(data), )...) } - - // Add head GitHub App token minting step if any PR handler has head-github-app configured. - // The minted token is passed as head-github-token in the handler config so fork branch - // operations can authenticate to the configured automation fork without requiring the - // main token to have upstream contents: write. if headApp := getSafeOutputsHeadApp(data.SafeOutputs); headApp != nil { headRepoSlug := getSafeOutputsHeadRepoSlug(data.SafeOutputs) - headFallbackRepo := headRepoNameFromSlug(headRepoSlug) preambleTokenSteps = append(preambleTokenSteps, c.buildGitHubAppTokenMintStepWithMeta( headApp, - nil, // head token is scoped to fork operations; no job-level permissions filter - headFallbackRepo, - headRepoSlug, // used to derive the owner when HeadGitHubApp.Owner is not set + nil, + headRepoNameFromSlug(headRepoSlug), + headRepoSlug, "Generate GitHub App head token", "safe-outputs-head-app-token", )...) } + return preambleTokenSteps +} - // Insert all preamble token steps before checkout and safe-output handler steps. - if len(preambleTokenSteps) > 0 { - // Calculate insertion index: after setup action (if present) and artifact downloads, - // but before checkout and safe output steps. - insertIndex := 0 - - // Count setup action steps (checkout + setup if in dev mode without action-tag, or just setup) - setupActionRef := c.resolveActionReference("./actions/setup", data) - if setupActionRef != "" { - insertIndex += len(c.generateCheckoutActionsFolder(data)) - // Use the same traceID as the real call so the line count matches exactly - countTraceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) - countParentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) - insertIndex += len(c.generateSetupStep(data, setupActionRef, SetupActionDestination, data.SafeOutputs != nil && data.SafeOutputs.UploadArtifact != nil, countTraceID, countParentSpanID)) - } - // Keep insertion index aligned with setup steps that may be injected between setup - // and artifact downloads. Count the entries each mask helper appends so the index - // stays self-consistent if either helper ever emits more than one step. - if isOTLPHeadersPresent(data) { - insertIndex += strings.Count(generateOTLPHeadersMaskStep(), stepNameLinePrefix) - } - if isOTLPAttributesPresent(data) { - insertIndex += strings.Count(generateOTLPAttributesMaskStep(), stepNameLinePrefix) - } - - // Add artifact download steps count - insertIndex += len(buildAgentOutputDownloadSteps(agentArtifactPrefix, c.getActionPin)) - - // Add upload-artifact staging download step count. - // The step has 6 YAML string entries: name, continue-on-error, uses, with:, name: , path: - if data.SafeOutputs.UploadArtifact != nil { - insertIndex += 6 - } - - // Add patch download steps if present - // Download from unified agent artifact (prefixed in workflow_call context) - if usesPatchesAndCheckouts(data.SafeOutputs) { - patchDownloadSteps := buildArtifactDownloadSteps(ArtifactDownloadConfig{ - ArtifactName: agentArtifactPrefix + constants.AgentArtifactName, - DownloadPath: constants.TmpGhAwDirSlash, - SetupEnvStep: false, - StepName: "Download patch artifact", - }, c.getActionPin) - insertIndex += len(patchDownloadSteps) - } - - // Note: App token steps must be inserted BEFORE shared checkout steps - // because those steps reference steps.safe-outputs-app-token.outputs.token - // - // The insertion index is line-oriented; if it lands in the middle of a - // multi-line run/with block, move it to the next step boundary. - for insertIndex < len(steps) && !strings.HasPrefix(steps[insertIndex], stepNameLinePrefix) { - insertIndex++ - } - if insertIndex == len(steps) { - consolidatedSafeOutputsJobLog.Printf( - "WARN: preamble-token insertion reached end of steps slice (len=%d); step ordering may be incorrect", - len(steps), - ) - } +func (c *Compiler) insertSafeOutputsPreambleTokenSteps(data *WorkflowData, agentArtifactPrefix string, steps, preambleTokenSteps []string) []string { + if len(preambleTokenSteps) == 0 { + return steps + } + insertIndex := c.calculateSafeOutputsPreambleInsertionIndex(data, agentArtifactPrefix) + for insertIndex < len(steps) && !strings.HasPrefix(steps[insertIndex], stepNameLinePrefix) { + insertIndex++ + } + if insertIndex == len(steps) { + consolidatedSafeOutputsJobLog.Printf( + "WARN: preamble-token insertion reached end of steps slice (len=%d); step ordering may be incorrect", + len(steps), + ) + } + newSteps := append([]string{}, steps[:insertIndex]...) + newSteps = append(newSteps, preambleTokenSteps...) + return append(newSteps, steps[insertIndex:]...) +} - // Insert all preamble token steps at the computed index. - var newSteps []string - newSteps = append(newSteps, steps[:insertIndex]...) - newSteps = append(newSteps, preambleTokenSteps...) - newSteps = append(newSteps, steps[insertIndex:]...) - steps = newSteps +func (c *Compiler) calculateSafeOutputsPreambleInsertionIndex(data *WorkflowData, agentArtifactPrefix string) int { + insertIndex := 0 + setupActionRef := c.resolveActionReference("./actions/setup", data) + if setupActionRef != "" { + insertIndex += len(c.generateCheckoutActionsFolder(data)) + countTraceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) + countParentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) + enableArtifactClient := data.SafeOutputs != nil && data.SafeOutputs.UploadArtifact != nil + insertIndex += len(c.generateSetupStep(data, setupActionRef, SetupActionDestination, enableArtifactClient, countTraceID, countParentSpanID)) + } + if isOTLPHeadersPresent(data) { + insertIndex += strings.Count(generateOTLPHeadersMaskStep(), stepNameLinePrefix) + } + if isOTLPAttributesPresent(data) { + insertIndex += strings.Count(generateOTLPAttributesMaskStep(), stepNameLinePrefix) + } + insertIndex += len(buildAgentOutputDownloadSteps(agentArtifactPrefix, c.getActionPin)) + if data.SafeOutputs.UploadArtifact != nil { + insertIndex += 6 } + if usesPatchesAndCheckouts(data.SafeOutputs) { + patchDownloadSteps := buildArtifactDownloadSteps(ArtifactDownloadConfig{ + ArtifactName: agentArtifactPrefix + constants.AgentArtifactName, + DownloadPath: constants.TmpGhAwDirSlash, + SetupEnvStep: false, + StepName: "Download patch artifact", + }, c.getActionPin) + insertIndex += len(patchDownloadSteps) + } + return insertIndex +} - // Upload the safe output items manifest as an artifact (non-staged mode only). - // This step runs even if previous steps fail, ensuring the audit trail - // is always available for the audit command to display. - // In staged mode, no items are actually created in GitHub so there is nothing to record. +func (c *Compiler) appendSafeOutputsFinalSteps(data *WorkflowData, agentArtifactPrefix string, steps []string) []string { isStaged := c.trialMode || templatableBoolIsTrue(data.SafeOutputs.Staged) if !isStaged { steps = append(steps, buildSafeOutputItemsManifestUploadStep(agentArtifactPrefix, c.getActionPin)...) } - - // Append OTLP conclusion span step (no-op when endpoint is not configured). - // Note: this step is now handled by the action post step (post.js) so no - // injected step is needed here. - - // In dev mode the setup action is referenced via a local path (./actions/setup), so its files - // live in the workspace. When the safe_outputs job contains a checkout step for - // create_pull_request or push_to_pull_request_branch, the workspace is replaced with the - // target repository content, removing the actions/setup directory. - // Without restoring it, the runner's post-step for Setup Scripts would fail with - // "Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under .../actions/setup". - // We add a restore checkout step (if: always()) as the last step so the post-step - // can always find action.yml and complete its /tmp/gh-aw cleanup. if c.actionMode.IsDev() && usesPatchesAndCheckouts(data.SafeOutputs) { steps = append(steps, c.generateRestoreActionsSetupStep()) consolidatedSafeOutputsJobLog.Print("Added restore actions folder step to safe_outputs job (dev mode with checkout)") } - - // In script mode, explicitly add a cleanup step (mirrors post.js in dev/release/action mode). if c.actionMode.IsScript() { steps = append(steps, c.generateScriptModeCleanupStep()) } + return steps +} - // Build the job condition - // The job should run if agent job completed (not skipped) AND detection passed (if enabled) +func buildSafeOutputsJobCondition(data *WorkflowData, threatDetectionEnabled bool) ConditionNode { agentNotSkipped := BuildAnd( &NotNode{Child: BuildFunctionCall("cancelled")}, BuildNotEquals( @@ -662,117 +605,98 @@ func (c *Compiler) buildSafeOutputsJobFromParts( BuildStringLiteral("skipped"), ), ) - - jobCondition := agentNotSkipped if IsConditionalDetection(data.SafeOutputs) { - // When detection is expression-controlled, the detection job may be skipped at runtime - // (expression evaluated to false). Use always() to prevent safe_outputs from being - // skipped due to a skipped dependency, and accept both success and skipped results. - jobCondition = BuildAnd( + return BuildAnd( BuildAnd(BuildFunctionCall("always"), agentNotSkipped), buildDetectionPassedCondition(), ) - } else if threatDetectionEnabled { - jobCondition = BuildAnd(agentNotSkipped, buildDetectionSuccessCondition()) } + if threatDetectionEnabled { + return BuildAnd(agentNotSkipped, buildDetectionSuccessCondition()) + } + return agentNotSkipped +} - // Build dependencies — safe_outputs depends on agent; when threat detection is enabled it also - // depends on the detection job (so that detection_success is available). +func (c *Compiler) buildSafeOutputsNeeds(data *WorkflowData, mainJobName string, threatDetectionEnabled bool) []string { needs := []string{mainJobName} if threatDetectionEnabled { needs = append(needs, string(constants.DetectionJobName)) consolidatedSafeOutputsJobLog.Print("Added detection job dependency to safe_outputs job") } - // Always add activation job dependency to get the trace-id for OTLP correlation, - // and also when needed for other reasons: - // - create_pull_request or push_to_pull_request_branch (need the activation artifact) - // - lock-for-agent (need the activation lock) - // - workflow_call trigger (need needs.activation.outputs.target_repo for cross-repo token/dispatch) needs = append(needs, string(constants.ActivationJobName)) - // Add unlock job dependency if lock-for-agent is enabled - // This ensures the issue is unlocked before safe outputs run if data.LockForAgent { needs = append(needs, "unlock") consolidatedSafeOutputsJobLog.Print("Added unlock job dependency to safe_outputs job") } - seenNeeds := make(map[string]struct { - }, len(needs)) + seenNeeds := make(map[string]struct{}, len(needs)) for _, need := range needs { - seenNeeds[need] = struct { - }{} - } - if data.SafeOutputs != nil { - for _, need := range data.SafeOutputs.Needs { - if setutil.Contains(seenNeeds, need) { - continue - } - needs = append(needs, need) - seenNeeds[need] = struct { - }{} - consolidatedSafeOutputsJobLog.Printf("Added explicit safe-outputs needs dependency to safe_outputs job: %s", need) - } + seenNeeds[need] = struct{}{} } + needs = addExplicitSafeOutputsNeeds(data, needs, seenNeeds) + return c.addPreActivationSafeOutputsNeed(data, needs, seenNeeds) +} - // If any message template references needs.pre_activation.outputs.*, add pre_activation - // as a dependency so that GitHub Actions can resolve the expression at runtime. - if data.SafeOutputs != nil && messagesContainPreActivationRef(data.SafeOutputs.Messages) { - if _, exists := c.jobManager.GetJob(string(constants.PreActivationJobName)); exists { - preActName := string(constants.PreActivationJobName) - if !setutil.Contains(seenNeeds, preActName) { - needs = append(needs, preActName) - seenNeeds[preActName] = struct{}{} - consolidatedSafeOutputsJobLog.Print("Added pre_activation dependency to safe_outputs job (messages reference pre_activation outputs)") - } +func addExplicitSafeOutputsNeeds(data *WorkflowData, needs []string, seenNeeds map[string]struct{}) []string { + if data.SafeOutputs == nil { + return needs + } + for _, need := range data.SafeOutputs.Needs { + if setutil.Contains(seenNeeds, need) { + continue } + needs = append(needs, need) + seenNeeds[need] = struct{}{} + consolidatedSafeOutputsJobLog.Printf("Added explicit safe-outputs needs dependency to safe_outputs job: %s", need) } + return needs +} - // Extract workflow ID from markdown path for GH_AW_WORKFLOW_ID - workflowID := GetWorkflowIDFromPath(markdownPath) - - // Build job-level environment variables that are common to all safe output steps - jobEnv := c.buildJobLevelSafeOutputEnvVars(data, workflowID) - - // Build concurrency config for the safe-outputs job if a concurrency-group is configured - var concurrency string - if data.SafeOutputs.ConcurrencyGroup != "" { - concurrency = c.indentYAMLLines(fmt.Sprintf("concurrency:\n group: %q\n cancel-in-progress: false", data.SafeOutputs.ConcurrencyGroup), " ") - consolidatedSafeOutputsJobLog.Printf("Configuring safe_outputs job concurrency group: %s", data.SafeOutputs.ConcurrencyGroup) +func (c *Compiler) addPreActivationSafeOutputsNeed(data *WorkflowData, needs []string, seenNeeds map[string]struct{}) []string { + if data.SafeOutputs == nil || !messagesContainPreActivationRef(data.SafeOutputs.Messages) { + return needs } - - // Determine the environment for the safe-outputs job. - // If safe-outputs.environment is explicitly set, use that override. - // Otherwise, propagate the top-level environment: field so that environment-scoped - // secrets (e.g. for GitHub App token minting) are accessible in this job. - safeOutputsEnvironment := resolveSafeOutputsEnvironment(data) - - // Use the configured timeout or fall back to the default of 45 minutes. - // The previous default was 15 minutes, which is insufficient for workflows - // with many sequential safe output operations (e.g. push_to_pull_request_branch - // against large monorepos). 45 minutes is the new default; users can override - // via safe-outputs.timeout-minutes in frontmatter. - const defaultSafeOutputsTimeoutMinutes = 45 - timeoutMinutes := defaultSafeOutputsTimeoutMinutes - if data.SafeOutputs.TimeoutMinutes > 0 { - timeoutMinutes = data.SafeOutputs.TimeoutMinutes + if _, exists := c.jobManager.GetJob(string(constants.PreActivationJobName)); !exists { + return needs + } + preActName := string(constants.PreActivationJobName) + if !setutil.Contains(seenNeeds, preActName) { + needs = append(needs, preActName) + seenNeeds[preActName] = struct{}{} + consolidatedSafeOutputsJobLog.Print("Added pre_activation dependency to safe_outputs job (messages reference pre_activation outputs)") } + return needs +} - job := &Job{ +func (c *Compiler) buildSafeOutputsJob(data *WorkflowData, permissions *Permissions, outputs map[string]string, steps, needs []string, jobCondition ConditionNode, workflowID string) *Job { + return &Job{ Name: "safe_outputs", If: RenderCondition(jobCondition), RunsOn: c.formatFrameworkJobRunsOn(data), - Environment: c.indentYAMLLines(safeOutputsEnvironment, " "), + Environment: c.indentYAMLLines(resolveSafeOutputsEnvironment(data), " "), Permissions: permissions.RenderToYAML(), - TimeoutMinutes: timeoutMinutes, - Concurrency: concurrency, - Env: jobEnv, + TimeoutMinutes: safeOutputsJobTimeoutMinutes(data), + Concurrency: c.buildSafeOutputsConcurrency(data), + Env: c.buildJobLevelSafeOutputEnvVars(data, workflowID), Steps: steps, Outputs: outputs, Needs: needs, } +} - consolidatedSafeOutputsJobLog.Printf("Built consolidated safe outputs job with %d steps", len(safeOutputStepNames)) +func (c *Compiler) buildSafeOutputsConcurrency(data *WorkflowData) string { + if data.SafeOutputs.ConcurrencyGroup == "" { + return "" + } + consolidatedSafeOutputsJobLog.Printf("Configuring safe_outputs job concurrency group: %s", data.SafeOutputs.ConcurrencyGroup) + return c.indentYAMLLines(fmt.Sprintf("concurrency:\n group: %q\n cancel-in-progress: false", data.SafeOutputs.ConcurrencyGroup), " ") +} - return job, safeOutputStepNames, nil +func safeOutputsJobTimeoutMinutes(data *WorkflowData) int { + const defaultSafeOutputsTimeoutMinutes = 45 + if data.SafeOutputs.TimeoutMinutes > 0 { + return data.SafeOutputs.TimeoutMinutes + } + return defaultSafeOutputsTimeoutMinutes } // buildJobLevelSafeOutputEnvVars builds environment variables that should be set at the job level @@ -780,25 +704,24 @@ func (c *Compiler) buildSafeOutputsJobFromParts( func (c *Compiler) buildJobLevelSafeOutputEnvVars(data *WorkflowData, workflowID string) map[string]string { envVars := make(map[string]string) - // Set GH_AW_WORKFLOW_ID to the workflow ID (filename without extension) - // This is used for branch naming in create_pull_request and other operations - envVars["GH_AW_WORKFLOW_ID"] = fmt.Sprintf("%q", workflowID) + c.addWorkflowSafeOutputEnvVars(envVars, data, workflowID) + c.addEngineSafeOutputEnvVars(envVars, data) + addAgentOutputSafeOutputEnvVars(envVars) + addCommandSafeOutputEnvVars(envVars, data) + c.addSafeOutputModeEnvVars(envVars, data) + c.addMessagesSafeOutputEnvVars(envVars, data) + addDetectionSafeOutputEnvVars(envVars, data) - // Set GH_AW_CALLER_WORKFLOW_ID to uniquely identify the calling workflow at runtime. - // When a reusable workflow is called via workflow_call, multiple callers share the - // same GH_AW_WORKFLOW_ID (derived from the reusable file). This separate value - // combines the runtime repository (to identify the caller repo) with the compile-time - // workflow ID (filename without extension), producing a stable "owner/repo/workflow-id" - // form used for close-older-issues disambiguation. - envVars["GH_AW_CALLER_WORKFLOW_ID"] = fmt.Sprintf(`"${{ github.repository }}/%s"`, workflowID) + return envVars +} - // Add workflow metadata that's common to all steps +func (c *Compiler) addWorkflowSafeOutputEnvVars(envVars map[string]string, data *WorkflowData, workflowID string) { + envVars["GH_AW_WORKFLOW_ID"] = fmt.Sprintf("%q", workflowID) + envVars["GH_AW_CALLER_WORKFLOW_ID"] = fmt.Sprintf(`"${{ github.repository }}/%s"`, workflowID) envVars["GH_AW_WORKFLOW_NAME"] = fmt.Sprintf("%q", data.Name) - if data.FrontmatterEmoji != "" { envVars["GH_AW_WORKFLOW_EMOJI"] = fmt.Sprintf("%q", data.FrontmatterEmoji) } - if data.Source != "" { envVars["GH_AW_WORKFLOW_SOURCE"] = fmt.Sprintf("%q", data.Source) sourceURL := buildSourceURL(data.Source) @@ -806,47 +729,41 @@ func (c *Compiler) buildJobLevelSafeOutputEnvVars(data *WorkflowData, workflowID envVars["GH_AW_WORKFLOW_SOURCE_URL"] = fmt.Sprintf("%q", sourceURL) } } else if localURL := buildLocalWorkflowSourceURL(c.markdownPath); localURL != "" { - // For local workflows (no external source), point to the markdown file in the repo - // so that failure issue links resolve to the workflow source rather than "#". envVars["GH_AW_WORKFLOW_SOURCE_URL"] = fmt.Sprintf("%q", localURL) } - if data.TrackerID != "" { envVars["GH_AW_TRACKER_ID"] = fmt.Sprintf("%q", data.TrackerID) } - - // Bake the repository project UTC offset (from aw.json) into safe-outputs job env - // so runtime JavaScript helpers do not need to read aw.json on the runner. if utcOffset := c.getCompiledProjectUTCOffset(); utcOffset != "" { envVars["GH_AW_PROJECT_UTC"] = fmt.Sprintf("%q", utcOffset) } +} - // Add engine metadata that's common to all steps - if data.EngineConfig != nil { - if data.EngineConfig.ID != "" { - envVars["GH_AW_ENGINE_ID"] = fmt.Sprintf("%q", data.EngineConfig.ID) - } - if data.EngineConfig.Version != "" { - envVars["GH_AW_ENGINE_VERSION"] = fmt.Sprintf("%q", data.EngineConfig.Version) - } - // Prefer explicit compile-time model; fall back to the runtime model captured by the - // activation job so footers always show the actual model used for auditability. - if data.EngineConfig.Model != "" { - envVars["GH_AW_ENGINE_MODEL"] = fmt.Sprintf("%q", data.EngineConfig.Model) - } else { - envVars["GH_AW_ENGINE_MODEL"] = fmt.Sprintf("${{ needs.%s.outputs.model }}", constants.AgentJobName) - } +func (c *Compiler) addEngineSafeOutputEnvVars(envVars map[string]string, data *WorkflowData) { + if data.EngineConfig == nil { + return + } + if data.EngineConfig.ID != "" { + envVars["GH_AW_ENGINE_ID"] = fmt.Sprintf("%q", data.EngineConfig.ID) } + if data.EngineConfig.Version != "" { + envVars["GH_AW_ENGINE_VERSION"] = fmt.Sprintf("%q", data.EngineConfig.Version) + } + if data.EngineConfig.Model != "" { + envVars["GH_AW_ENGINE_MODEL"] = fmt.Sprintf("%q", data.EngineConfig.Model) + } else { + envVars["GH_AW_ENGINE_MODEL"] = fmt.Sprintf("${{ needs.%s.outputs.model }}", constants.AgentJobName) + } +} - // Pass effective tokens from the agent job so footer templates can use {effective_tokens_suffix}. - // The value is set by parse_mcp_gateway_log.cjs in the agent job and exposed as a job output. - // An empty/missing value is handled gracefully by getEffectiveTokensFromEnv() in messages_footer.cjs. +func addAgentOutputSafeOutputEnvVars(envVars map[string]string) { envVars["GH_AW_EFFECTIVE_TOKENS"] = fmt.Sprintf("${{ needs.%s.outputs.effective_tokens }}", constants.AgentJobName) envVars["GH_AW_AIC"] = fmt.Sprintf("${{ needs.%s.outputs.aic }}", constants.AgentJobName) envVars["GH_AW_AMBIENT_CONTEXT"] = fmt.Sprintf("${{ needs.%s.outputs.ambient_context }}", constants.AgentJobName) envVars["GH_AW_AGENT_AIC"] = fmt.Sprintf("${{ needs.%s.outputs.aic }}", constants.AgentJobName) +} - // Add slash command metadata so safe output handlers can render run-again footer hints. +func addCommandSafeOutputEnvVars(envVars map[string]string, data *WorkflowData) { if len(data.Command) > 0 { if commandsJSON, err := json.Marshal(data.Command); err == nil { envVars["GH_AW_COMMANDS"] = fmt.Sprintf("%q", string(commandsJSON)) @@ -855,14 +772,14 @@ func (c *Compiler) buildJobLevelSafeOutputEnvVars(data *WorkflowData, workflowID envVars["GH_AW_COMMAND_PLACEHOLDER"] = fmt.Sprintf("%q", data.CommandPlaceholder) } } - // Add label command metadata so safe output handlers can render run-again footer hints. if len(data.LabelCommand) > 0 { if labelCommandsJSON, err := json.Marshal(data.LabelCommand); err == nil { envVars["GH_AW_LABEL_COMMANDS"] = fmt.Sprintf("%q", string(labelCommandsJSON)) } } +} - // Add safe output job environment variables (staged/target repo) +func (c *Compiler) addSafeOutputModeEnvVars(envVars map[string]string, data *WorkflowData) { if data.SafeOutputs != nil { if value := resolveSafeOutputsStagedValue(c.trialMode, data.SafeOutputs.Staged); value != nil { if isExpression(*value) { @@ -872,39 +789,30 @@ func (c *Compiler) buildJobLevelSafeOutputEnvVars(data *WorkflowData, workflowID } } } - - // Set GH_AW_TARGET_REPO_SLUG - prefer trial target repo (applies to all steps) - // Note: Individual steps with target-repo config will override this in their step-level env if c.trialMode && c.trialLogicalRepoSlug != "" { envVars["GH_AW_TARGET_REPO_SLUG"] = fmt.Sprintf("%q", c.trialLogicalRepoSlug) } +} - // Add messages config if present (applies to all steps) - if data.SafeOutputs != nil && data.SafeOutputs.Messages != nil { - messagesJSON, err := serializeMessagesConfig(data.SafeOutputs.Messages) - if err != nil { - consolidatedSafeOutputsJobLog.Printf("Warning: failed to serialize messages config: %v", err) - } else if messagesJSON != "" { - envVars["GH_AW_SAFE_OUTPUT_MESSAGES"] = fmt.Sprintf("%q", messagesJSON) - } +func (c *Compiler) addMessagesSafeOutputEnvVars(envVars map[string]string, data *WorkflowData) { + if data.SafeOutputs == nil || data.SafeOutputs.Messages == nil { + return } - - // Note: GH_AW_CI_TRIGGER_TOKEN is added at the step level (in buildHandlerManagerStep) - // rather than job level, since only the Process Safe Outputs step needs it, - // and only when create-pull-request or push-to-pull-request-branch is configured. - - // Note: Asset upload configuration is not needed here because upload_assets - // is now handled as a separate job (see buildUploadAssetsJob) - - // Pass detection conclusion and reason to safe outputs when threat detection is enabled. - // This allows handlers (e.g., push-to-pull-request-branch) to adjust behavior on warnings. - if IsDetectionJobEnabled(data.SafeOutputs) { - envVars["GH_AW_DETECTION_CONCLUSION"] = fmt.Sprintf("${{ needs.%s.outputs.detection_conclusion }}", constants.DetectionJobName) - envVars["GH_AW_DETECTION_REASON"] = fmt.Sprintf("${{ needs.%s.outputs.detection_reason }}", constants.DetectionJobName) - envVars["GH_AW_THREAT_DETECTION_AIC"] = fmt.Sprintf("${{ needs.%s.outputs.aic }}", constants.DetectionJobName) + messagesJSON, err := serializeMessagesConfig(data.SafeOutputs.Messages) + if err != nil { + consolidatedSafeOutputsJobLog.Printf("Warning: failed to serialize messages config: %v", err) + } else if messagesJSON != "" { + envVars["GH_AW_SAFE_OUTPUT_MESSAGES"] = fmt.Sprintf("%q", messagesJSON) } +} - return envVars +func addDetectionSafeOutputEnvVars(envVars map[string]string, data *WorkflowData) { + if !IsDetectionJobEnabled(data.SafeOutputs) { + return + } + envVars["GH_AW_DETECTION_CONCLUSION"] = fmt.Sprintf("${{ needs.%s.outputs.detection_conclusion }}", constants.DetectionJobName) + envVars["GH_AW_DETECTION_REASON"] = fmt.Sprintf("${{ needs.%s.outputs.detection_reason }}", constants.DetectionJobName) + envVars["GH_AW_THREAT_DETECTION_AIC"] = fmt.Sprintf("${{ needs.%s.outputs.aic }}", constants.DetectionJobName) } // resolveSafeOutputsEnvironment resolves the effective GitHub deployment environment for diff --git a/pkg/workflow/compiler_safe_outputs_steps.go b/pkg/workflow/compiler_safe_outputs_steps.go index d90e2ca3bd2..1211b31b728 100644 --- a/pkg/workflow/compiler_safe_outputs_steps.go +++ b/pkg/workflow/compiler_safe_outputs_steps.go @@ -91,210 +91,178 @@ func (c *Compiler) buildHandlerManagerStep(data *WorkflowData) ([]string, error) consolidatedSafeOutputsStepsLog.Print("Building handler manager step") var steps []string + steps = append(steps, c.buildCreateCheckRunAppTokenSteps(data)...) + steps = append(steps, buildHandlerManagerStepHeader(data)...) + if err := c.addHandlerManagerEnv(&steps, data); err != nil { + return nil, err + } + steps = append(steps, " with:\n") + configToken := "" + if data.SafeOutputs != nil && data.SafeOutputs.GitHubToken != "" { + configToken = data.SafeOutputs.GitHubToken + } + c.addSafeOutputGitHubTokenForConfig(&steps, data, configToken) + steps = append(steps, buildHandlerManagerScriptLines()...) + return steps, nil +} - // Add per-handler GitHub App token minting steps before the handler manager step. - // These run before the main handler step so the minted token expressions (e.g. - // ${{ steps.create-check-run-app-token.outputs.token }}) are resolved at runtime. - if data.SafeOutputs != nil && data.SafeOutputs.CreateCheckRun != nil && data.SafeOutputs.CreateCheckRun.GitHubApp != nil { - consolidatedSafeOutputsStepsLog.Print("Adding per-handler GitHub App token minting step for create-check-run") - permissions := NewPermissionsContentsReadChecksWrite() - for _, step := range c.buildGitHubAppTokenMintStep(data.SafeOutputs.CreateCheckRun.GitHubApp, permissions, "") { - steps = append(steps, replaceStepID(step, "safe-outputs-app-token", "create-check-run-app-token")) - } +func (c *Compiler) buildCreateCheckRunAppTokenSteps(data *WorkflowData) []string { + if data.SafeOutputs == nil || data.SafeOutputs.CreateCheckRun == nil || data.SafeOutputs.CreateCheckRun.GitHubApp == nil { + return nil } + consolidatedSafeOutputsStepsLog.Print("Adding per-handler GitHub App token minting step for create-check-run") + permissions := NewPermissionsContentsReadChecksWrite() + var steps []string + for _, step := range c.buildGitHubAppTokenMintStep(data.SafeOutputs.CreateCheckRun.GitHubApp, permissions, "") { + steps = append(steps, replaceStepID(step, "safe-outputs-app-token", "create-check-run-app-token")) + } + return steps +} - // Step name and metadata - steps = append(steps, " - name: Process Safe Outputs\n") - steps = append(steps, " id: process_safe_outputs\n") - steps = append(steps, fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data))) +func buildHandlerManagerStepHeader(data *WorkflowData) []string { + return []string{ + " - name: Process Safe Outputs\n", + " id: process_safe_outputs\n", + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), + " env:\n", + " GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}\n", + " GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }}\n", + } +} - // Environment variables - steps = append(steps, " env:\n") - steps = append(steps, " GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}\n") - steps = append(steps, " GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }}\n") +func (c *Compiler) addHandlerManagerEnv(steps *[]string, data *WorkflowData) error { + if err := c.addHandlerManagerDomainEnv(steps, data); err != nil { + return err + } + addHandlerManagerBaseEnv(steps, data) + addHandlerManagerCustomTypeEnv(steps, data) + c.addCustomSafeOutputEnvVars(steps, data) + c.addHandlerManagerConfigEnvVar(steps, data) + c.addAllSafeOutputConfigEnvVars(steps, data) + addHandlerManagerCITokenEnv(steps, data) + addHandlerManagerProjectEnv(steps, data) + addHandlerManagerAgentTokenEnv(steps, data) + addHandlerManagerAgentSessionEnv(steps, data) + addHandlerManagerGitHubTokenEnv(steps, data) + return nil +} - // Add allowed domains configuration for URL sanitization in safe output handlers. - // Without this, sanitizeContent() in safe_output_handler_manager.cjs only allows - // default GitHub domains, causing user-configured allowed domains to be redacted. +func (c *Compiler) addHandlerManagerDomainEnv(steps *[]string, data *WorkflowData) error { var domainsStr string if data.SafeOutputs != nil && len(data.SafeOutputs.AllowedDomains) > 0 { - // allowed-domains: additional domains unioned with engine/network base set; supports ecosystem identifiers expanded, err := c.computeExpandedAllowedDomainsForSanitization(data) if err != nil { - return nil, err + return err } domainsStr = expanded } else { computed, err := c.computeAllowedDomainsForSanitization(data) if err != nil { - return nil, err + return err } domainsStr = computed } if domainsStr != "" { - steps = append(steps, fmt.Sprintf(" GH_AW_ALLOWED_DOMAINS: %q\n", domainsStr)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_ALLOWED_DOMAINS: %q\n", domainsStr)) } + return nil +} + +func addHandlerManagerBaseEnv(steps *[]string, data *WorkflowData) { if data.SafeOutputs != nil && data.SafeOutputs.URLs != "" { - steps = append(steps, fmt.Sprintf(" GH_AW_SAFE_OUTPUTS_URLS: %q\n", data.SafeOutputs.URLs)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_SAFE_OUTPUTS_URLS: %q\n", data.SafeOutputs.URLs)) } - // Pass GitHub server/API URLs so buildAllowedDomains() can add GHES domains dynamically - steps = append(steps, " GITHUB_SERVER_URL: ${{ github.server_url }}\n") - steps = append(steps, " GITHUB_API_URL: ${{ github.api_url }}\n") - - // Note: The project handler manager has been removed. - // All project-related operations are now handled by the unified handler. + *steps = append(*steps, " GITHUB_SERVER_URL: ${{ github.server_url }}\n") + *steps = append(*steps, " GITHUB_API_URL: ${{ github.api_url }}\n") +} - // Add GH_AW_SAFE_OUTPUT_JOBS so the handler manager knows which message types are - // handled by custom safe-output job steps and should be silently skipped rather than - // reported as "No handler loaded for message type '...'". +func addHandlerManagerCustomTypeEnv(steps *[]string, data *WorkflowData) { if customJobsJSON := buildCustomSafeOutputJobsJSON(data); customJobsJSON != "" { - steps = append(steps, fmt.Sprintf(" GH_AW_SAFE_OUTPUT_JOBS: %q\n", customJobsJSON)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_SAFE_OUTPUT_JOBS: %q\n", customJobsJSON)) consolidatedSafeOutputsStepsLog.Print("Added GH_AW_SAFE_OUTPUT_JOBS env var for custom safe job types") } - - // Add GH_AW_SAFE_OUTPUT_SCRIPTS so the handler manager can load inline script handlers. - // The env var maps normalized script names to their .cjs filenames in the actions folder. if customScriptsJSON := buildCustomSafeOutputScriptsJSON(data); customScriptsJSON != "" { - steps = append(steps, fmt.Sprintf(" GH_AW_SAFE_OUTPUT_SCRIPTS: %q\n", customScriptsJSON)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_SAFE_OUTPUT_SCRIPTS: %q\n", customScriptsJSON)) consolidatedSafeOutputsStepsLog.Print("Added GH_AW_SAFE_OUTPUT_SCRIPTS env var for custom script handlers") } - - // Add GH_AW_SAFE_OUTPUT_ACTIONS so the handler manager can load custom action handlers. - // The env var maps normalized action names to themselves (reserved for future extensibility). if customActionsJSON := buildCustomSafeOutputActionsJSON(data); customActionsJSON != "" { - steps = append(steps, fmt.Sprintf(" GH_AW_SAFE_OUTPUT_ACTIONS: %q\n", customActionsJSON)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_SAFE_OUTPUT_ACTIONS: %q\n", customActionsJSON)) consolidatedSafeOutputsStepsLog.Print("Added GH_AW_SAFE_OUTPUT_ACTIONS env var for custom action handlers") } +} - // Add custom safe output env vars - c.addCustomSafeOutputEnvVars(&steps, data) - - // Add handler manager config as JSON - c.addHandlerManagerConfigEnvVar(&steps, data) - - // Add all safe output configuration env vars (still needed by individual handlers) - c.addAllSafeOutputConfigEnvVars(&steps, data) - - // Add extra empty commit token if create-pull-request or push-to-pull-request-branch is configured. - // This token is used to push an empty commit after code changes to trigger CI events, - // working around the GITHUB_TOKEN limitation where events don't trigger other workflows. - // Only emit this env var when one of these safe outputs is actually configured. - if usesPatchesAndCheckouts(data.SafeOutputs) { - var ciTriggerToken string - if data.SafeOutputs.CreatePullRequests != nil && data.SafeOutputs.CreatePullRequests.GithubTokenForExtraEmptyCommit != "" { - ciTriggerToken = data.SafeOutputs.CreatePullRequests.GithubTokenForExtraEmptyCommit - } else if data.SafeOutputs.PushToPullRequestBranch != nil && data.SafeOutputs.PushToPullRequestBranch.GithubTokenForExtraEmptyCommit != "" { - ciTriggerToken = data.SafeOutputs.PushToPullRequestBranch.GithubTokenForExtraEmptyCommit - } - - switch ciTriggerToken { - case "app": - steps = append(steps, " GH_AW_CI_TRIGGER_TOKEN: ${{ steps.safe-outputs-app-token.outputs.token || '' }}\n") - consolidatedSafeOutputsStepsLog.Print("Extra empty commit using GitHub App token") - default: - // Use the magic GH_AW_CI_TRIGGER_TOKEN secret (default behavior when not explicitly configured) - steps = append(steps, fmt.Sprintf(" GH_AW_CI_TRIGGER_TOKEN: %s\n", getEffectiveCITriggerGitHubToken(ciTriggerToken))) - consolidatedSafeOutputsStepsLog.Print("Extra empty commit using GH_AW_CI_TRIGGER_TOKEN") - } +func addHandlerManagerCITokenEnv(steps *[]string, data *WorkflowData) { + if !usesPatchesAndCheckouts(data.SafeOutputs) { + return + } + var ciTriggerToken string + if data.SafeOutputs.CreatePullRequests != nil && data.SafeOutputs.CreatePullRequests.GithubTokenForExtraEmptyCommit != "" { + ciTriggerToken = data.SafeOutputs.CreatePullRequests.GithubTokenForExtraEmptyCommit + } else if data.SafeOutputs.PushToPullRequestBranch != nil && data.SafeOutputs.PushToPullRequestBranch.GithubTokenForExtraEmptyCommit != "" { + ciTriggerToken = data.SafeOutputs.PushToPullRequestBranch.GithubTokenForExtraEmptyCommit } + if ciTriggerToken == "app" { + *steps = append(*steps, " GH_AW_CI_TRIGGER_TOKEN: ${{ steps.safe-outputs-app-token.outputs.token || '' }}\n") + consolidatedSafeOutputsStepsLog.Print("Extra empty commit using GitHub App token") + return + } + *steps = append(*steps, fmt.Sprintf(" GH_AW_CI_TRIGGER_TOKEN: %s\n", getEffectiveCITriggerGitHubToken(ciTriggerToken))) + consolidatedSafeOutputsStepsLog.Print("Extra empty commit using GH_AW_CI_TRIGGER_TOKEN") +} - // Add GH_AW_PROJECT_URL and GH_AW_PROJECT_GITHUB_TOKEN environment variables for project operations - // These are set from the project URL and token configured in any project-related safe-output: - // - update-project - // - create-project-status-update - // - create-project - // - // The project field is REQUIRED in update-project and create-project-status-update (enforced by schema validation) - // Agents can optionally override this per-message by including a project field in their output - // - // Note: If multiple project configs are present, we prefer update-project > create-project-status-update > create-project - // This is only relevant for the environment variables - each configuration must explicitly specify its own settings +func addHandlerManagerProjectEnv(steps *[]string, data *WorkflowData) { projectURL, projectToken := resolveProjectURLAndToken(data.SafeOutputs) - if projectURL != "" { - steps = append(steps, fmt.Sprintf(" GH_AW_PROJECT_URL: %q\n", projectURL)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_PROJECT_URL: %q\n", projectURL)) } - if projectToken != "" { - steps = append(steps, fmt.Sprintf(" GH_AW_PROJECT_GITHUB_TOKEN: %s\n", projectToken)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_PROJECT_GITHUB_TOKEN: %s\n", projectToken)) } +} - // Add GH_AW_ASSIGN_TO_AGENT_TOKEN when assign-to-agent is configured OR when create-issue - // or create-pull-request is configured with copilot in assignees. All handlers create a - // dedicated Octokit using this token (agent token preference chain), which is required - // because the Copilot assignment API only accepts PATs (not GitHub App tokens). This env - // var is evaluated as a GitHub Actions expression, so it resolves to the actual token value - // before the step runs. +func addHandlerManagerAgentTokenEnv(steps *[]string, data *WorkflowData) { if data.SafeOutputs != nil && data.SafeOutputs.AssignToAgent != nil { agentTokenStr := getEffectiveCopilotCodingAgentGitHubToken(data.SafeOutputs.AssignToAgent.GitHubToken) - //nolint:gosec // G101: False positive - this is a GitHub Actions expression template, not a hardcoded credential - steps = append(steps, fmt.Sprintf(" GH_AW_ASSIGN_TO_AGENT_TOKEN: %s\n", agentTokenStr)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_ASSIGN_TO_AGENT_TOKEN: %s\n", agentTokenStr)) consolidatedSafeOutputsStepsLog.Print("Added GH_AW_ASSIGN_TO_AGENT_TOKEN env var for assign-to-agent handler") } else if data.SafeOutputs != nil && data.SafeOutputs.CreateIssues != nil && hasCopilotAssignee(data.SafeOutputs.CreateIssues.Assignees) { agentTokenStr := getEffectiveCopilotCodingAgentGitHubToken(data.SafeOutputs.CreateIssues.GitHubToken) - //nolint:gosec // G101: False positive - this is a GitHub Actions expression template, not a hardcoded credential - steps = append(steps, fmt.Sprintf(" GH_AW_ASSIGN_TO_AGENT_TOKEN: %s\n", agentTokenStr)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_ASSIGN_TO_AGENT_TOKEN: %s\n", agentTokenStr)) consolidatedSafeOutputsStepsLog.Print("Added GH_AW_ASSIGN_TO_AGENT_TOKEN env var for create-issue copilot assignment handler") } else if data.SafeOutputs != nil && data.SafeOutputs.CreatePullRequests != nil && hasCopilotAssignee(data.SafeOutputs.CreatePullRequests.Assignees) { agentTokenStr := getEffectiveCopilotCodingAgentGitHubToken(data.SafeOutputs.CreatePullRequests.GitHubToken) - //nolint:gosec // G101: False positive - this is a GitHub Actions expression template, not a hardcoded credential - steps = append(steps, fmt.Sprintf(" GH_AW_ASSIGN_TO_AGENT_TOKEN: %s\n", agentTokenStr)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_ASSIGN_TO_AGENT_TOKEN: %s\n", agentTokenStr)) consolidatedSafeOutputsStepsLog.Print("Added GH_AW_ASSIGN_TO_AGENT_TOKEN env var for create-pull-request copilot assignment handler") } +} - // Add GH_AW_AGENT_SESSION_TOKEN when create-agent-session is configured. - // The create_agent_session handler passes this token as GH_TOKEN to the gh CLI - // (agent token preference chain), which is required because the default GITHUB_TOKEN - // does not have permission to create agent sessions via gh agent-task create. +func addHandlerManagerAgentSessionEnv(steps *[]string, data *WorkflowData) { if data.SafeOutputs != nil && data.SafeOutputs.CreateAgentSessions != nil { agentSessionTokenStr := getEffectiveCopilotCodingAgentGitHubToken(data.SafeOutputs.CreateAgentSessions.GitHubToken) - //nolint:gosec // G101: False positive - this is a GitHub Actions expression template, not a hardcoded credential - steps = append(steps, fmt.Sprintf(" GH_AW_AGENT_SESSION_TOKEN: %s\n", agentSessionTokenStr)) + *steps = append(*steps, fmt.Sprintf(" GH_AW_AGENT_SESSION_TOKEN: %s\n", agentSessionTokenStr)) consolidatedSafeOutputsStepsLog.Print("Added GH_AW_AGENT_SESSION_TOKEN env var for create-agent-session handler") } +} - // When create-pull-request or push-to-pull-request-branch is configured with a custom token - // (including GitHub App), expose that token as GITHUB_TOKEN so that git CLI operations in - // the JavaScript handlers can authenticate. The create_pull_request.cjs handler reads - // process.env.GITHUB_TOKEN to enable dynamic repo checkout for multi-repo/cross-repo - // scenarios (allowed-repos). Without this, the handler falls back to the default - // repo-scoped token which lacks access to other repos. - if usesPatchesAndCheckouts(data.SafeOutputs) { - gitToken, isCustom := resolvePRCheckoutToken(data.SafeOutputs, NewCheckoutManager(data.CheckoutConfigs)) - // Only override GITHUB_TOKEN when a custom token (app or PAT) is explicitly configured. - // When no custom token is set, the default repo-scoped GITHUB_TOKEN from GitHub Actions - // is already in the environment and overriding it with the same default is unnecessary. - if isCustom { - //nolint:gosec // G101: False positive - this is a GitHub Actions expression template, not a hardcoded credential - steps = append(steps, fmt.Sprintf(" GITHUB_TOKEN: %s\n", gitToken)) - consolidatedSafeOutputsStepsLog.Printf("Adding GITHUB_TOKEN env var for cross-repo git CLI operations") - } +func addHandlerManagerGitHubTokenEnv(steps *[]string, data *WorkflowData) { + if !usesPatchesAndCheckouts(data.SafeOutputs) { + return } - - // With section for github-token - // Use the standard safe-outputs token for the shared github-script client. - // Project operations use GH_AW_PROJECT_GITHUB_TOKEN from env with dedicated handler logic. - steps = append(steps, " with:\n") - // Token precedence for the handler manager step: - // 1. Safe-outputs level token (so.GitHubToken) - // 2. Magic secret fallback via getEffectiveSafeOutputGitHubToken() - // - // Note: We do NOT fall back to per-output tokens (add-comment, create-issue, etc.) - // because those are specific to their operations. The handler manager needs a - // general-purpose token for the github-script client. - configToken := "" - if data.SafeOutputs != nil && data.SafeOutputs.GitHubToken != "" { - configToken = data.SafeOutputs.GitHubToken + gitToken, isCustom := resolvePRCheckoutToken(data.SafeOutputs, NewCheckoutManager(data.CheckoutConfigs)) + if isCustom { + *steps = append(*steps, fmt.Sprintf(" GITHUB_TOKEN: %s\n", gitToken)) + consolidatedSafeOutputsStepsLog.Printf("Adding GITHUB_TOKEN env var for cross-repo git CLI operations") } - c.addSafeOutputGitHubTokenForConfig(&steps, data, configToken) - - steps = append(steps, " script: |\n") - steps = append(steps, " const { setupGlobals } = require('"+SetupActionDestination+"/setup_globals.cjs');\n") - steps = append(steps, " setupGlobals(core, github, context, exec, io, getOctokit);\n") - steps = append(steps, " const { main } = require('"+SetupActionDestination+"/safe_output_handler_manager.cjs');\n") - steps = append(steps, " await main();\n") +} - return steps, nil +func buildHandlerManagerScriptLines() []string { + return []string{ + " script: |\n", + " const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n", + " setupGlobals(core, github, context, exec, io, getOctokit);\n", + " const { main } = require('" + SetupActionDestination + "/safe_output_handler_manager.cjs');\n", + " await main();\n", + } } // replaceStepID replaces all occurrences of oldID with newID in a YAML step string. diff --git a/pkg/workflow/compiler_string_api.go b/pkg/workflow/compiler_string_api.go index 2a322a7cca5..7a64fa1617e 100644 --- a/pkg/workflow/compiler_string_api.go +++ b/pkg/workflow/compiler_string_api.go @@ -58,17 +58,29 @@ func (c *Compiler) ParseWorkflowString(content string, virtualPath string) (*Wor workflowLog.Printf("ParseWorkflowString: parsing %d bytes with virtual path %s", len(content), virtualPath) cleanPath := filepath.Clean(virtualPath) - contentBytes := []byte(content) - - // Store content so downstream code can use it instead of reading from disk. - // Cleared in CompileToYAML after compilation completes. c.contentOverride = content - - // Enable inline prompt mode for string-based compilation (Wasm/browser) - // since runtime-import macros cannot resolve without filesystem access c.inlinePrompt = true - // Parse frontmatter directly from content string + parseResult, err := c.parseAndValidateWorkflowStringFrontmatter(content, cleanPath) + if err != nil { + return nil, err + } + workflowData, engineSetup, err := c.buildWorkflowDataFromStringParseResult(parseResult) + if err != nil { + return nil, err + } + if err := c.validateStringWorkflowData(cleanPath, workflowData); err != nil { + return nil, err + } + if err := c.finalizeStringWorkflowData(parseResult, workflowData, engineSetup); err != nil { + return nil, err + } + + return workflowData, nil +} + +func (c *Compiler) parseAndValidateWorkflowStringFrontmatter(content, cleanPath string) (*frontmatterParseResult, error) { + contentBytes := []byte(content) result, err := parser.ExtractFrontmatterFromContent(content) if err != nil { frontmatterStart := 2 @@ -77,53 +89,47 @@ func (c *Compiler) ParseWorkflowString(content string, virtualPath string) (*Wor } return nil, c.createFrontmatterError(cleanPath, content, err, frontmatterStart) } - if len(result.Frontmatter) == 0 { return nil, errors.New("no frontmatter found") } - compilerStringAPILog.Printf("ParseWorkflowString: extracted frontmatter with %d fields", len(result.Frontmatter)) - - // Preprocess schedule fields if err := c.preprocessScheduleFields(result.Frontmatter, cleanPath, content); err != nil { return nil, err } - frontmatterForValidation := c.copyFrontmatterWithoutInternalMarkers(result.Frontmatter) - - // Check if "on" field is missing - distinguish redirect-only placeholders from shared workflows - _, hasOnField := frontmatterForValidation["on"] - if !hasOnField { - // Check if this is a redirect-only placeholder (has redirect field but no 'on' trigger). - // Redirect-only files are distinct from regular shared workflows: they are placeholders - // pointing to a workflow's new canonical location and should not be treated as importable components. - if redirectVal, hasRedirect := frontmatterForValidation["redirect"]; hasRedirect { - if redirectStr, ok := redirectVal.(string); ok { - if redirectTarget := strings.TrimSpace(redirectStr); redirectTarget != "" { - compilerStringAPILog.Printf("ParseWorkflowString: redirect-only workflow detected: redirect=%s", redirectTarget) - return nil, &RedirectOnlyWorkflowError{Path: cleanPath, Target: redirectTarget} - } - } - } - compilerStringAPILog.Printf("ParseWorkflowString: no 'on' field, treating as shared workflow: %s", cleanPath) - return nil, &SharedWorkflowError{Path: cleanPath} + if err := validateStringWorkflowOnField(cleanPath, frontmatterForValidation); err != nil { + return nil, err } - if err := c.validateEngineBeforeSchema(cleanPath, contentBytes, result, frontmatterForValidation); err != nil { compilerStringAPILog.Printf("ParseWorkflowString: string engine pre-validation failed for %s", cleanPath) return nil, err } - - // Validate frontmatter against schema if err := parser.ValidateMainWorkflowFrontmatterWithSchemaAndLocation(frontmatterForValidation, cleanPath); err != nil { compilerStringAPILog.Printf("ParseWorkflowString: schema validation failed for %s", cleanPath) return nil, err } - compilerStringAPILog.Printf("ParseWorkflowString: frontmatter validated, frontmatter_fields=%d", len(frontmatterForValidation)) + return newStringFrontmatterParseResult(cleanPath, contentBytes, result, frontmatterForValidation), nil +} - // Build parse result to reuse the rest of the orchestrator pipeline - parseResult := &frontmatterParseResult{ +func validateStringWorkflowOnField(cleanPath string, frontmatterForValidation map[string]any) error { + if _, hasOnField := frontmatterForValidation["on"]; hasOnField { + return nil + } + if redirectVal, hasRedirect := frontmatterForValidation["redirect"]; hasRedirect { + if redirectStr, ok := redirectVal.(string); ok { + if redirectTarget := strings.TrimSpace(redirectStr); redirectTarget != "" { + compilerStringAPILog.Printf("ParseWorkflowString: redirect-only workflow detected: redirect=%s", redirectTarget) + return &RedirectOnlyWorkflowError{Path: cleanPath, Target: redirectTarget} + } + } + } + compilerStringAPILog.Printf("ParseWorkflowString: no 'on' field, treating as shared workflow: %s", cleanPath) + return &SharedWorkflowError{Path: cleanPath} +} + +func newStringFrontmatterParseResult(cleanPath string, contentBytes []byte, result *parser.FrontmatterResult, frontmatterForValidation map[string]any) *frontmatterParseResult { + return &frontmatterParseResult{ cleanPath: cleanPath, content: contentBytes, frontmatterResult: result, @@ -131,64 +137,53 @@ func (c *Compiler) ParseWorkflowString(content string, virtualPath string) (*Wor markdownDir: filepath.Dir(cleanPath), isSharedWorkflow: false, } +} - // Setup engine and process imports +func (c *Compiler) buildWorkflowDataFromStringParseResult(parseResult *frontmatterParseResult) (*WorkflowData, *engineSetupResult, error) { engineSetup, err := c.setupEngineAndImports(parseResult.frontmatterResult, parseResult.cleanPath, parseResult.content, parseResult.markdownDir) if err != nil { - return nil, err + return nil, nil, err } - - // Process tools and markdown toolsResult, err := c.processToolsAndMarkdown(parseResult.frontmatterResult, parseResult.cleanPath, parseResult.markdownDir, engineSetup.agenticEngine, engineSetup.engineSetting, engineSetup.importsResult) if err != nil { - return nil, err + return nil, nil, err } - - // Build initial workflow data structure workflowData := c.buildInitialWorkflowData(parseResult.frontmatterResult, toolsResult, engineSetup, engineSetup.importsResult) - workflowData.WorkflowID = GetWorkflowIDFromPath(cleanPath) + workflowData.WorkflowID = GetWorkflowIDFromPath(parseResult.cleanPath) + return workflowData, engineSetup, nil +} - // Validate bash tool configuration +func (c *Compiler) validateStringWorkflowData(cleanPath string, workflowData *WorkflowData) error { if err := validateBashToolConfig(workflowData.ParsedTools, workflowData.Name); err != nil { - return nil, fmt.Errorf("%s: %w", cleanPath, err) + return fmt.Errorf("%s: %w", cleanPath, err) } - - // Validate optional engine.mcp.session-timeout configuration. if err := c.validateEngineMCPSessionTimeout(workflowData); err != nil { - return nil, fmt.Errorf("%s: %w", cleanPath, err) + return fmt.Errorf("%s: %w", cleanPath, err) } - - // Validate optional engine.mcp.tool-timeout configuration. if err := c.validateEngineMCPToolTimeout(workflowData); err != nil { - return nil, fmt.Errorf("%s: %w", cleanPath, err) + return fmt.Errorf("%s: %w", cleanPath, err) } - - // Validate GitHub tool configuration if err := validateGitHubToolConfig(workflowData.ParsedTools, workflowData.Name); err != nil { - return nil, fmt.Errorf("%s: %w", cleanPath, err) + return fmt.Errorf("%s: %w", cleanPath, err) } - - // Validate GitHub tool read-only configuration if err := validateGitHubReadOnly(workflowData.ParsedTools, workflowData.Name); err != nil { - return nil, fmt.Errorf("%s: %w", cleanPath, err) + return fmt.Errorf("%s: %w", cleanPath, err) } - - // Validate GitHub guard policy configuration if err := validateGitHubGuardPolicy(workflowData.ParsedTools, workflowData.Name); err != nil { - return nil, fmt.Errorf("%s: %w", cleanPath, err) + return fmt.Errorf("%s: %w", cleanPath, err) } emitGitHubLockdownGuardPolicyWarning(c, workflowData.ParsedTools, cleanPath) - - // Validate integrity-reactions feature configuration var gatewayConfig *MCPGatewayRuntimeConfig if workflowData.SandboxConfig != nil { gatewayConfig = workflowData.SandboxConfig.MCP } if err := validateIntegrityReactions(workflowData.ParsedTools, workflowData.Name, workflowData, gatewayConfig); err != nil { - return nil, fmt.Errorf("%s: %w", cleanPath, err) + return fmt.Errorf("%s: %w", cleanPath, err) } + return nil +} - // Setup action cache and resolver +func (c *Compiler) finalizeStringWorkflowData(parseResult *frontmatterParseResult, workflowData *WorkflowData, engineSetup *engineSetupResult) error { actionCache, actionResolver := c.getSharedActionResolver() workflowData.Ctx = c.ctx workflowData.ActionCache = actionCache @@ -198,7 +193,7 @@ func (c *Compiler) ParseWorkflowString(content string, virtualPath string) (*Wor // Extract YAML configuration sections if err := c.extractYAMLSections(parseResult.frontmatterResult.Frontmatter, workflowData); err != nil { - return nil, fmt.Errorf("failed to extract YAML sections: %w", err) + return fmt.Errorf("failed to extract YAML sections: %w", err) } // Merge features from imports @@ -206,20 +201,19 @@ func (c *Compiler) ParseWorkflowString(content string, virtualPath string) (*Wor compilerStringAPILog.Printf("ParseWorkflowString: merging %d features from imports", len(engineSetup.importsResult.MergedFeatures)) mergedFeatures, err := c.MergeFeatures(workflowData.Features, engineSetup.importsResult.MergedFeatures) if err != nil { - return nil, fmt.Errorf("failed to merge features from imports: %w", err) + return fmt.Errorf("failed to merge features from imports: %w", err) } workflowData.Features = mergedFeatures } // Process and merge custom steps if err := c.processAndMergeSteps(parseResult.frontmatterResult.Frontmatter, workflowData, engineSetup.importsResult); err != nil { - return nil, err + return err } // Apply defaults - if err := c.applyDefaults(workflowData, cleanPath); err != nil { - return nil, err + if err := c.applyDefaults(workflowData, parseResult.cleanPath); err != nil { + return err } - - return workflowData, nil + return nil } diff --git a/pkg/workflow/compiler_unlock_job.go b/pkg/workflow/compiler_unlock_job.go index 6c91dbffe72..12e142c314b 100644 --- a/pkg/workflow/compiler_unlock_job.go +++ b/pkg/workflow/compiler_unlock_job.go @@ -27,97 +27,89 @@ func (c *Compiler) buildUnlockJob(data *WorkflowData, threatDetectionEnabled boo var steps []string - // Add setup step to copy scripts setupActionRef := c.resolveActionReference("./actions/setup", data) if setupActionRef == "" && !c.actionMode.IsScript() { return nil, errors.New("setup action reference is required but could not be resolved") } - // For dev mode (local action path), checkout the actions folder first - steps = append(steps, c.generateCheckoutActionsFolder(data)...) + steps = append(steps, c.buildUnlockSetupSteps(data, setupActionRef)...) + steps = append(steps, buildUnlockIssueStep(data)...) - // Unlock job doesn't need project support - // Unlock job depends on activation, reuse its trace ID - unlockTraceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) - unlockParentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) - steps = append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, unlockTraceID, unlockParentSpanID)...) - - // Add unlock step - // Build condition: only unlock if issue was locked by activation job - // Must match lock condition: event type is 'issues' or 'issue_comment' - eventTypeCheck := BuildOr( - BuildEventTypeEquals("issues"), - BuildEventTypeEquals("issue_comment"), - ) - lockedOutputCheck := BuildEquals( - BuildPropertyAccess(fmt.Sprintf("needs.%s.outputs.issue_locked", constants.ActivationJobName)), - BuildStringLiteral("true"), - ) + compilerUnlockJobLog.Print("Added unlock issue step to dedicated unlock job") + + needs := unlockJobNeeds(threatDetectionEnabled) + permissions := c.unlockJobPermissions(data) - unlockCondition := BuildAnd(eventTypeCheck, lockedOutputCheck) + compilerUnlockJobLog.Printf("Job built successfully: dependencies=%v", needs) - steps = append(steps, " - name: Unlock issue after agentic workflow\n") - steps = append(steps, " id: unlock-issue\n") - steps = append(steps, fmt.Sprintf(" if: %s\n", RenderCondition(unlockCondition))) - steps = append(steps, fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data))) - steps = append(steps, " with:\n") - steps = append(steps, " script: |\n") - steps = append(steps, generateGitHubScriptWithRequire("unlock-issue.cjs")) + // In script mode, explicitly add a cleanup step (mirrors post.js in dev/release/action mode). + if c.actionMode.IsScript() { + steps = append(steps, c.generateScriptModeCleanupStep()) + } - compilerUnlockJobLog.Print("Added unlock issue step to dedicated unlock job") + job := &Job{ + Name: "unlock", + Needs: needs, + If: unlockJobCondition(), + RunsOn: c.formatFrameworkJobRunsOn(data), + Permissions: permissions, + Steps: steps, + TimeoutMinutes: 5, // Short timeout - unlock is a quick operation + } - // Build the condition for this job: - // 1. always() - run even if agent or other jobs fail - // 2. activation was not skipped - skip unlock when activation was never triggered - // (e.g. the event did not match any trigger condition, so no locking occurred) - // 3. issue was locked (checked at step level for clarity in workflow YAML) + return job, nil +} + +func (c *Compiler) buildUnlockSetupSteps(data *WorkflowData, setupActionRef string) []string { + var steps []string + steps = append(steps, c.generateCheckoutActionsFolder(data)...) + unlockTraceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) + unlockParentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) + return append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, unlockTraceID, unlockParentSpanID)...) +} + +func buildUnlockIssueStep(data *WorkflowData) []string { + unlockCondition := BuildAnd( + BuildOr(BuildEventTypeEquals("issues"), BuildEventTypeEquals("issue_comment")), + BuildEquals(BuildPropertyAccess(fmt.Sprintf("needs.%s.outputs.issue_locked", constants.ActivationJobName)), BuildStringLiteral("true")), + ) + return []string{ + " - name: Unlock issue after agentic workflow\n", + " id: unlock-issue\n", + fmt.Sprintf(" if: %s\n", RenderCondition(unlockCondition)), + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), + " with:\n", + " script: |\n", + generateGitHubScriptWithRequire("unlock-issue.cjs"), + } +} + +func unlockJobCondition() string { alwaysFunc := BuildFunctionCall("always") activationNotSkipped := BuildNotEquals( BuildPropertyAccess(fmt.Sprintf("needs.%s.result", constants.ActivationJobName)), BuildStringLiteral("skipped"), ) - jobCondition := BuildAnd(alwaysFunc, activationNotSkipped) + return RenderCondition(BuildAnd(alwaysFunc, activationNotSkipped)) +} - // Create the unlock job - // This job depends on activation (for issue_locked output) and agent (to run after workflow) - // When threat detection is enabled, it also depends on the detection job +func unlockJobNeeds(threatDetectionEnabled bool) []string { needs := []string{string(constants.ActivationJobName), string(constants.AgentJobName)} if threatDetectionEnabled { needs = append(needs, string(constants.DetectionJobName)) compilerUnlockJobLog.Print("Added detection job dependency to unlock job") } + return needs +} - // Determine permissions - need contents: read for dev mode checkout, issues: write for unlocking - var permissions string +func (c *Compiler) unlockJobPermissions(data *WorkflowData) string { needsContentsRead := (c.actionMode.IsDev() || c.actionMode.IsScript()) && len(c.generateCheckoutActionsFolder(data)) > 0 if needsContentsRead { perms := NewPermissionsContentsRead() - // Add issues write permission for unlocking - perms.Set(PermissionIssues, PermissionWrite) - permissions = perms.RenderToYAML() - } else { - // Only need issues write permission - perms := NewPermissions() perms.Set(PermissionIssues, PermissionWrite) - permissions = perms.RenderToYAML() + return perms.RenderToYAML() } - - compilerUnlockJobLog.Printf("Job built successfully: dependencies=%v", needs) - - // In script mode, explicitly add a cleanup step (mirrors post.js in dev/release/action mode). - if c.actionMode.IsScript() { - steps = append(steps, c.generateScriptModeCleanupStep()) - } - - job := &Job{ - Name: "unlock", - Needs: needs, - If: RenderCondition(jobCondition), - RunsOn: c.formatFrameworkJobRunsOn(data), - Permissions: permissions, - Steps: steps, - TimeoutMinutes: 5, // Short timeout - unlock is a quick operation - } - - return job, nil + perms := NewPermissions() + perms.Set(PermissionIssues, PermissionWrite) + return perms.RenderToYAML() } diff --git a/pkg/workflow/compiler_workflow_call.go b/pkg/workflow/compiler_workflow_call.go index 25bb0ac32ed..471ae80c963 100644 --- a/pkg/workflow/compiler_workflow_call.go +++ b/pkg/workflow/compiler_workflow_call.go @@ -110,14 +110,7 @@ func (c *Compiler) injectWorkflowCallOutputs(onSection string, safeOutputs *Safe workflowCallLog.Printf("Generated %d workflow_call outputs to inject", len(generatedOutputs)) // Parse the on section YAML - var onData map[string]any - if err := yaml.Unmarshal([]byte(onSection), &onData); err != nil { - workflowCallLog.Printf("Warning: failed to parse on section for workflow_call outputs injection: %v", err) - return onSection - } - - // Get the 'on' map - onMap, ok := onData["on"].(map[string]any) + onData, onMap, ok := parseWorkflowCallOnMap(onSection, "outputs") if !ok { return onSection } @@ -129,22 +122,45 @@ func (c *Compiler) injectWorkflowCallOutputs(onSection string, safeOutputs *Safe } // Convert workflow_call to a map (it may be nil if declared without options) - var workflowCallMap map[string]any + workflowCallMap := workflowCallValueMap(workflowCallVal) + + // Merge auto-generated outputs with any existing user-defined outputs. + mergedOutputs := mergeWorkflowCallOutputs(generatedOutputs, workflowCallMap["outputs"]) + workflowCallLog.Printf("Merged workflow_call outputs: total=%d", len(mergedOutputs)) + workflowCallMap["outputs"] = mergedOutputs + onMap["workflow_call"] = workflowCallMap + + return marshalWorkflowCallOnSection(onData, onSection, "outputs") +} + +func parseWorkflowCallOnMap(onSection, purpose string) (map[string]any, map[string]any, bool) { + var onData map[string]any + if err := yaml.Unmarshal([]byte(onSection), &onData); err != nil { + workflowCallLog.Printf("Warning: failed to parse on section for workflow_call outputs injection: %v", err) + return nil, nil, false + } + onMap, ok := onData["on"].(map[string]any) + if !ok { + return nil, nil, false + } + return onData, onMap, true +} + +func workflowCallValueMap(workflowCallVal any) map[string]any { if workflowCallVal == nil { - workflowCallMap = make(map[string]any) - } else if m, ok := workflowCallVal.(map[string]any); ok { - workflowCallMap = m - } else { - workflowCallMap = make(map[string]any) + return make(map[string]any) + } + if m, ok := workflowCallVal.(map[string]any); ok { + return m } + return make(map[string]any) +} - // Merge auto-generated outputs with any existing user-defined outputs. - // User-defined outputs take precedence (their keys overwrite generated ones). +func mergeWorkflowCallOutputs(generated map[string]workflowCallOutputEntry, existing any) map[string]workflowCallOutputEntry { mergedOutputs := make(map[string]workflowCallOutputEntry) - maps.Copy(mergedOutputs, generatedOutputs) - if existingOutputs, hasOutputs := workflowCallMap["outputs"].(map[string]any); hasOutputs { + maps.Copy(mergedOutputs, generated) + if existingOutputs, hasOutputs := existing.(map[string]any); hasOutputs { for k, v := range existingOutputs { - // User-defined entries may be maps with description+value or plain strings if outputMap, ok := v.(map[string]any); ok { entry := workflowCallOutputEntry{} if desc, ok := outputMap["description"].(string); ok { @@ -157,16 +173,13 @@ func (c *Compiler) injectWorkflowCallOutputs(onSection string, safeOutputs *Safe } } } + return mergedOutputs +} - workflowCallLog.Printf("Merged workflow_call outputs: total=%d", len(mergedOutputs)) - workflowCallMap["outputs"] = mergedOutputs - onMap["workflow_call"] = workflowCallMap - - // Re-marshal to YAML - newOnData := map[string]any{"on": onMap} - newYAML, err := yaml.Marshal(newOnData) +func marshalWorkflowCallOnSection(onData map[string]any, onSection, purpose string) string { + newYAML, err := yaml.Marshal(map[string]any{"on": onData["on"]}) if err != nil { - workflowCallLog.Printf("Warning: failed to marshal on section with workflow_call outputs: %v", err) + workflowCallLog.Printf("Warning: failed to marshal on section with workflow_call %s: %v", purpose, err) return onSection } @@ -272,56 +285,66 @@ func injectWorkflowCallSecretsSection(onSection string, secrets []string) string } // Normalize onData["on"] to map[string]any, handling string and slice shorthand forms. + onMap, ok := normalizeWorkflowCallOnMap(onData) + if !ok { + return onSection + } + onData["on"] = onMap + + workflowCallVal, hasWorkflowCall := onMap["workflow_call"] + if !hasWorkflowCall { + return onSection + } + + // Convert workflow_call to a map (it may be nil if declared without options). + workflowCallMap := workflowCallValueMap(workflowCallVal) + + // Build the auto-generated secrets map (required: false for all entries). + generatedSecrets := buildWorkflowCallSecretsMap(secretsToInject, workflowCallMap["secrets"]) + + // Convert to a plain map for marshaling. + workflowCallMap["secrets"] = workflowCallSecretsOutputMap(generatedSecrets) + onMap["workflow_call"] = workflowCallMap + + // Re-marshal to YAML. + return marshalWorkflowCallOnSection(onData, onSection, "secrets") +} + +func normalizeWorkflowCallOnMap(onData map[string]any) (map[string]any, bool) { rawOn, hasOn := onData["on"] if !hasOn { - return onSection + return nil, false } - var onMap map[string]any switch v := rawOn.(type) { case map[string]any: - onMap = v + return v, true case string: - onMap = map[string]any{v: nil} + return map[string]any{v: nil}, true case []any: - onMap = make(map[string]any, len(v)) + onMap := make(map[string]any, len(v)) for _, event := range v { if eventName, ok := event.(string); ok { onMap[eventName] = nil } } + return onMap, true case []string: - onMap = make(map[string]any, len(v)) + onMap := make(map[string]any, len(v)) for _, eventName := range v { onMap[eventName] = nil } + return onMap, true default: - return onSection - } - onData["on"] = onMap - - workflowCallVal, hasWorkflowCall := onMap["workflow_call"] - if !hasWorkflowCall { - return onSection - } - - // Convert workflow_call to a map (it may be nil if declared without options). - var workflowCallMap map[string]any - if workflowCallVal == nil { - workflowCallMap = make(map[string]any) - } else if m, ok := workflowCallVal.(map[string]any); ok { - workflowCallMap = m - } else { - workflowCallMap = make(map[string]any) + return nil, false } +} - // Build the auto-generated secrets map (required: false for all entries). +func buildWorkflowCallSecretsMap(secretsToInject []string, existing any) map[string]workflowCallSecretEntry { generatedSecrets := make(map[string]workflowCallSecretEntry, len(secretsToInject)) for _, name := range secretsToInject { generatedSecrets[name] = workflowCallSecretEntry{Required: false} } - - // Merge: user-defined entries take precedence over generated ones. - if existingSecrets, hasSecrets := workflowCallMap["secrets"].(map[string]any); hasSecrets { + if existingSecrets, hasSecrets := existing.(map[string]any); hasSecrets { for k, v := range existingSecrets { if entryMap, ok := v.(map[string]any); ok { entry := workflowCallSecretEntry{} @@ -332,22 +355,13 @@ func injectWorkflowCallSecretsSection(onSection string, secrets []string) string } } } + return generatedSecrets +} - // Convert to a plain map for marshaling. +func workflowCallSecretsOutputMap(generatedSecrets map[string]workflowCallSecretEntry) map[string]any { secretsOut := make(map[string]any, len(generatedSecrets)) for k, v := range generatedSecrets { secretsOut[k] = map[string]any{"required": v.Required} } - workflowCallMap["secrets"] = secretsOut - onMap["workflow_call"] = workflowCallMap - - // Re-marshal to YAML. - newOnData := map[string]any{"on": onMap} - newYAML, err := yaml.Marshal(newOnData) - if err != nil { - workflowCallLog.Printf("Warning: failed to marshal on section with workflow_call secrets: %v", err) - return onSection - } - - return strings.TrimSuffix(string(newYAML), "\n") + return secretsOut } diff --git a/pkg/workflow/compiler_yaml.go b/pkg/workflow/compiler_yaml.go index 73ec659c3c3..37ce80b71f9 100644 --- a/pkg/workflow/compiler_yaml.go +++ b/pkg/workflow/compiler_yaml.go @@ -92,66 +92,12 @@ func (c *Compiler) generateWorkflowBody(yaml *strings.Builder, data *WorkflowDat func (c *Compiler) generateYAML(data *WorkflowData, markdownPath string) (string, []string, []string, error) { compilerYamlLog.Printf("Generating YAML for workflow: %s", data.Name) - // Compute frontmatter hash BEFORE building jobs so that the stable hash is - // available to heredoc-delimiter generation throughout job construction. - // Using the hex-encoded SHA-256 frontmatter hash string as an HMAC key keeps - // the compiled lock file identical across repeated compilations of the same workflow. - var frontmatterHash string - var bodyHash string - if markdownPath != "" { - baseDir := filepath.Dir(markdownPath) - cache := parser.NewImportCache(baseDir) - - // computeWorkflowHash calls the parsed-content path when RawMarkdown is - // available (fast path), falling back to a disk read otherwise. - computeWorkflowHash := func( - fromParsed func() (string, error), - fromFile func() (string, error), - ) (string, error) { - if data.RawMarkdown != "" { - return fromParsed() - } - compilerYamlLog.Printf("RawMarkdown not set; falling back to reading file from disk: %s", markdownPath) - return fromFile() - } - - hash, err := computeWorkflowHash( - func() (string, error) { - return parser.ComputeFrontmatterHashFromParsedContent(data.FrontmatterYAML, data.RawMarkdown, data.RawFrontmatter, baseDir, cache, parser.DefaultFileReader) - }, - func() (string, error) { - return parser.ComputeFrontmatterHashFromFileWithParsedFrontmatter(markdownPath, data.RawFrontmatter, cache, parser.DefaultFileReader) - }, - ) - if err != nil { - return "", nil, nil, fmt.Errorf("failed to generate workflow YAML: could not compute stable frontmatter hash for %q: %w", markdownPath, err) - } - frontmatterHash = hash - compilerYamlLog.Printf("Computed frontmatter hash: %s", hash) - - // Compute body hash to cover changes to the markdown body that are not captured - // by the frontmatter hash. This enables stale-check: full detection. - bHash, bErr := computeWorkflowHash( - func() (string, error) { - return parser.ComputeBodyHashFromParsedContent(data.RawMarkdown, data.FrontmatterYAML, baseDir, parser.DefaultFileReader) - }, - func() (string, error) { - return parser.ComputeBodyHashFromFile(markdownPath) - }, - ) - if bErr != nil { - compilerYamlLog.Printf("Warning: could not compute body hash for %q: %v", markdownPath, bErr) - // Non-fatal: continue without body hash - } else { - bodyHash = bHash - compilerYamlLog.Printf("Computed body hash: %s", bodyHash) - } + frontmatterHash, bodyHash, err := computeWorkflowHashes(data, markdownPath) + if err != nil { + return "", nil, nil, err } - // Store hash on WorkflowData so job-building helpers (MCP renderers, prompt - // step generators, etc.) can derive stable heredoc delimiters from it. data.FrontmatterHash = frontmatterHash - // Build all jobs and validate dependencies if err := c.buildJobsAndValidate(data, markdownPath); err != nil { return "", nil, nil, fmt.Errorf("failed to build and validate jobs: %w", err) } @@ -165,46 +111,18 @@ func (c *Compiler) generateYAML(data *WorkflowData, markdownPath string) (string var yaml strings.Builder yaml.Grow(initialBuilderCapacity) - // Generate workflow body first so we can collect secrets and custom actions - // for inclusion in the header comment. - var body strings.Builder - body.Grow(initialBuilderCapacity) - c.generateWorkflowBody(&body, data) - bodyContent := body.String() - - // Collect secrets and external action references from the generated body. - // These are returned to the caller so they can be used for safe update enforcement - // without requiring a second scan of the full YAML content. + bodyContent := c.generateWorkflowBodyString(data, initialBuilderCapacity) secrets := CollectSecretReferences(bodyContent) actions := CollectActionReferences(bodyContent) - // If this workflow has a workflow_call trigger, inject on.workflow_call.secrets: - // declarations so callers can map secrets explicitly instead of using secrets: inherit. - // We update data.On and regenerate the body so the compiled output includes the - // declarations. The set of secrets does not change between the two passes (the - // injected declarations do not add new ${{ secrets.* }} references). if hasWorkflowCallTrigger(data.On) && len(secrets) > 0 { - updatedOn := injectWorkflowCallSecretsSection(data.On, secrets) - if updatedOn != data.On { - data.On = updatedOn - body.Reset() - body.Grow(initialBuilderCapacity) - c.generateWorkflowBody(&body, data) - bodyContent = body.String() - compilerYamlLog.Printf("Regenerated workflow body with on.workflow_call.secrets declarations") - } + bodyContent = c.regenerateBodyWithWorkflowCallSecrets(data, secrets, bodyContent, initialBuilderCapacity) } - // Generate workflow header comments (including metadata as first line, plus secrets/actions lists) c.generateWorkflowHeader(&yaml, data, frontmatterHash, bodyHash, secrets, actions) - - // Append the workflow body yaml.WriteString(bodyContent) - yamlContent := yaml.String() - // If we're in non-cloning trial mode and this workflow has issue triggers, - // replace github.event.issue.number with inputs.issue_number if c.trialMode && c.hasIssueTrigger(data.On) { compilerYamlLog.Print("Trial mode enabled, replacing issue number references") yamlContent = c.replaceIssueNumberReferences(yamlContent) @@ -219,3 +137,68 @@ func (c *Compiler) generateYAML(data *WorkflowData, markdownPath string) (string compilerYamlLog.Printf("Successfully generated YAML for workflow: %s (%d bytes)", data.Name, len(yamlContent)) return yamlContent, secrets, actions, nil } + +func computeWorkflowHashes(data *WorkflowData, markdownPath string) (string, string, error) { + if markdownPath == "" { + return "", "", nil + } + baseDir := filepath.Dir(markdownPath) + cache := parser.NewImportCache(baseDir) + + frontmatterHash, err := computeWorkflowHash(data, markdownPath, + func() (string, error) { + return parser.ComputeFrontmatterHashFromParsedContent(data.FrontmatterYAML, data.RawMarkdown, data.RawFrontmatter, baseDir, cache, parser.DefaultFileReader) + }, + func() (string, error) { + return parser.ComputeFrontmatterHashFromFileWithParsedFrontmatter(markdownPath, data.RawFrontmatter, cache, parser.DefaultFileReader) + }, + ) + if err != nil { + return "", "", fmt.Errorf("failed to generate workflow YAML: could not compute stable frontmatter hash for %q: %w", markdownPath, err) + } + compilerYamlLog.Printf("Computed frontmatter hash: %s", frontmatterHash) + bodyHash := computeWorkflowBodyHash(data, markdownPath, baseDir) + return frontmatterHash, bodyHash, nil +} + +func computeWorkflowHash(data *WorkflowData, markdownPath string, fromParsed func() (string, error), fromFile func() (string, error)) (string, error) { + if data.RawMarkdown != "" { + return fromParsed() + } + compilerYamlLog.Printf("RawMarkdown not set; falling back to reading file from disk: %s", markdownPath) + return fromFile() +} + +func computeWorkflowBodyHash(data *WorkflowData, markdownPath string, baseDir string) string { + bodyHash, err := computeWorkflowHash(data, markdownPath, + func() (string, error) { + return parser.ComputeBodyHashFromParsedContent(data.RawMarkdown, data.FrontmatterYAML, baseDir, parser.DefaultFileReader) + }, + func() (string, error) { + return parser.ComputeBodyHashFromFile(markdownPath) + }, + ) + if err != nil { + compilerYamlLog.Printf("Warning: could not compute body hash for %q: %v", markdownPath, err) + return "" + } + compilerYamlLog.Printf("Computed body hash: %s", bodyHash) + return bodyHash +} + +func (c *Compiler) generateWorkflowBodyString(data *WorkflowData, capacity int) string { + var body strings.Builder + body.Grow(capacity) + c.generateWorkflowBody(&body, data) + return body.String() +} + +func (c *Compiler) regenerateBodyWithWorkflowCallSecrets(data *WorkflowData, secrets []string, bodyContent string, capacity int) string { + updatedOn := injectWorkflowCallSecretsSection(data.On, secrets) + if updatedOn == data.On { + return bodyContent + } + data.On = updatedOn + compilerYamlLog.Printf("Regenerated workflow body with on.workflow_call.secrets declarations") + return c.generateWorkflowBodyString(data, capacity) +} diff --git a/pkg/workflow/compiler_yaml_ai_execution.go b/pkg/workflow/compiler_yaml_ai_execution.go index 12624d7c924..9d4cfd3938d 100644 --- a/pkg/workflow/compiler_yaml_ai_execution.go +++ b/pkg/workflow/compiler_yaml_ai_execution.go @@ -262,11 +262,6 @@ func (c *Compiler) generateDetectAgentErrorsStep(yaml *strings.Builder, data *Wo // comment-memory state. // It returns the resolved CodingAgentEngine for use in subsequent phases. func (c *Compiler) generateEngineInstallAndPreAgentSteps(yaml *strings.Builder, data *WorkflowData, needsGitConfig bool) (CodingAgentEngine, error) { - // Configure git credentials for agentic workflows. - // Git credential configuration requires a .git directory in the workspace, which is only - // present when the repository was checked out. Skip these steps when checkout is disabled - // and no custom steps perform a checkout, since git remote set-url origin would fail - // with "fatal: not a git repository" otherwise. compilerYamlLog.Printf("Git credential configuration needed: %t", needsGitConfig) if needsGitConfig { gitConfigSteps := c.generateGitConfigurationSteps() @@ -275,73 +270,44 @@ func (c *Compiler) generateEngineInstallAndPreAgentSteps(yaml *strings.Builder, } } - // Add step to checkout PR branch if the event is pull_request c.generatePRReadyForReviewCheckout(yaml, data) - // Add Node.js setup if the engine requires it and it's not already set up in custom steps engine, err := c.getAgenticEngine(data.AI) if err != nil { return nil, fmt.Errorf("failed to resolve agentic engine from AI configuration: %w", err) } - // Ensure MCP gateway defaults are set before generating aw_info.json - // This is needed so that awmg_version is populated correctly if HasMCPServers(data) { ensureDefaultMCPGatewayConfig(data) } - // Add engine-specific installation steps (includes Node.js setup and secret validation for npm-based engines) - installSteps := engine.GetInstallationSteps(data) - compilerYamlLog.Printf("Adding %d engine installation steps for %s", len(installSteps), engine.GetID()) - for _, step := range installSteps { - for _, line := range step { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - - // Add Playwright CLI install steps when playwright is configured in CLI mode. - // These run after Node.js is available (set up by the engine install steps above). - for _, step := range generatePlaywrightCLIInstallSteps(data) { - for _, line := range step { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - - // GH_AW_SAFE_OUTPUTS is now set at job level, no setup step needed - - // Mint the GitHub MCP App token directly in the agent job. - // The token cannot be passed via job outputs from the activation job because - // actions/create-github-app-token calls ::add-mask:: on the token, and the - // GitHub Actions runner silently drops masked values in job outputs (runner v2.308+). - // By minting the token here, the app-id / private-key secrets are accessed only - // within this job and the minted token is available as steps.github-mcp-app-token.outputs.token. + writeEngineInstallSteps(yaml, data, engine) for _, step := range c.generateGitHubMCPAppTokenMintingSteps(data) { yaml.WriteString(step) } - // Add GitHub MCP lockdown detection step if needed c.generateGitHubMCPLockdownDetectionStep(yaml, data) - - // Add step to parse blocked-users and approval-labels guard variables into JSON arrays c.generateParseGuardVarsStep(yaml, data) - - // Stop DIFC proxy before starting the MCP gateway. The proxy must be stopped first - // to avoid double-filtering: the gateway uses the same guard policy for the agent phase. c.generateStopDIFCProxyStep(yaml, data) + writePreAgentRestoreSteps(yaml, data) + c.generatePreAgentSteps(yaml, data) - // Stop-time safety checks are now handled by a dedicated job (stop_time_check) - // No longer generated in the main job steps - - // Restore agent config folders from the base branch snapshot in the activation artifact. - // The activation job saved these before the PR checkout ran, so this step overwrites any - // PR-branch-injected files (e.g. forked skill/instruction files) with trusted base content. - // The .github/mcp.json file is also removed since it may come from the PR branch. - // The folder and file lists match those used in the save step (derived from engine registry). - // - // IMPORTANT: This must run BEFORE pre-agent-steps (below) so that APM-restored skills - // placed in .github/skills/ by pre-agent-steps are not clobbered by this restore. + if err := c.generateMCPSetup(yaml, data.Tools, engine, data); err != nil { + return nil, fmt.Errorf("failed to generate MCP setup: %w", err) + } + c.generateMCPCLIMountStep(yaml, data) + + return engine, nil +} + +func writeEngineInstallSteps(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine) { + installSteps := engine.GetInstallationSteps(data) + compilerYamlLog.Printf("Adding %d engine installation steps for %s", len(installSteps), engine.GetID()) + writeNestedLineSteps(yaml, installSteps) + writeNestedLineSteps(yaml, generatePlaywrightCLIInstallSteps(data)) +} + +func writePreAgentRestoreSteps(yaml *strings.Builder, data *WorkflowData) { if ShouldGeneratePRCheckoutStep(data) { registry := GetGlobalEngineRegistry() generateRestoreBaseGitHubFoldersStep(yaml, @@ -349,35 +315,21 @@ func (c *Compiler) generateEngineInstallAndPreAgentSteps(yaml *strings.Builder, registry.GetAllAgentManifestFiles(), ) } - - // Restore inline sub-agents written during the activation job. - // This step runs AFTER the base-branch restore so the engine-specific agent directory - // is not clobbered. Inline sub-agents are enabled by default. if isFeatureEnabled(constants.FeatureFlag("inline-agents"), data) { generateRestoreInlineSubAgentsStep(yaml, data) } - // Restore the engine-specific skills directory when inline skills are enabled or when - // explicit frontmatter skills were installed during activation. if isFeatureEnabled(constants.FeatureFlag("inline-agents"), data) || len(data.Skills) > 0 { generateRestoreInlineSkillsStep(yaml, data) } +} - // Add pre-agent-steps (if any) after base-branch restore but before MCP setup. - // Running after base restore ensures APM-restored skills (.github/skills/) are not - // overwritten by the restore step above in PR context. - // Running before MCP setup ensures pre-agent-steps can install/configure MCP - // dependencies that the gateway may reference when it starts. - c.generatePreAgentSteps(yaml, data) - - // Add MCP setup - if err := c.generateMCPSetup(yaml, data.Tools, engine, data); err != nil { - return nil, fmt.Errorf("failed to generate MCP setup: %w", err) +func writeNestedLineSteps(yaml *strings.Builder, steps []GitHubActionStep) { + for _, step := range steps { + for _, line := range step { + yaml.WriteString(line) + yaml.WriteByte('\n') + } } - - // Mount MCP servers as CLI tools (runs after gateway is started) - c.generateMCPCLIMountStep(yaml, data) - - return engine, nil } // generateAgentRunSteps emits the git credentials cleaner, engine config steps, CLI proxy start, @@ -387,77 +339,30 @@ func (c *Compiler) generateEngineInstallAndPreAgentSteps(yaml *strings.Builder, // It returns the initial set of artifact paths (to be extended by the caller) and the // agent stdio log path constant. func (c *Compiler) generateAgentRunSteps(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine, needsGitConfig bool) ([]string, string, error) { - // Collect artifact paths for unified upload at the end var artifactPaths []string artifactPaths = append(artifactPaths, constants.AwPromptsFile) logFileFull := constants.AgentStdioLogPath - // Clean credentials before executing the agentic engine. - // This removes git credentials from .git/config and, when known credential-leaking - // actions were detected, also removes cloud-provider / registry credentials. - credentialsCleanerSteps := c.generateCredentialsCleanerStep(data.KnownActionCredentialEnvVars) - for _, line := range credentialsCleanerSteps { + for _, line := range c.generateCredentialsCleanerStep(data.KnownActionCredentialEnvVars) { yaml.WriteString(line) } - - // Emit an audit step after credentials have been cleaned but before the agent begins - // execution. This captures a file listing of agent-related directories so the final - // pre-agent state (including any config written by MCP setup and engine config steps) - // is visible in the agent artifact without exposing raw credentials. c.generatePreAgentAuditStep(yaml) - // Emit engine config steps (from RenderConfig) before the AI execution step. - // These steps write runtime config files to disk (e.g. provider/model config files). - // Most engines return no steps here; only engines that require config files use this. - if len(data.EngineConfigSteps) > 0 { - compilerYamlLog.Printf("Adding %d engine config steps for %s", len(data.EngineConfigSteps), engine.GetID()) - for _, step := range data.EngineConfigSteps { - stepYAML, err := ConvertStepToYAML(step) - if err != nil { - return nil, "", fmt.Errorf("failed to render engine config step: %w", err) - } - yaml.WriteString(stepYAML) - } + if err := writeEngineConfigSteps(yaml, data, engine); err != nil { + return nil, "", err } - - // Start CLI proxy on the host before AWF execution. When features.cli-proxy is enabled, - // the compiler starts a difc-proxy container on the host that AWF's cli-proxy sidecar - // connects to via host.docker.internal:18443. c.generateStartCliProxyStep(yaml, data) + writeDockerSbxCredentialRefreshStep(yaml, data) - // Refresh sbx credentials immediately before AWF execution. Docker Hub OAuth - // tokens obtained during the daemon-setup step can expire between workflow steps, - // causing "user is not authenticated to Docker" errors when AWF calls `sbx create`. - if isDockerSbxRuntime(data) { - refreshStep := generateDockerSbxCredentialRefreshStep() - for _, line := range refreshStep { - yaml.WriteString(line) - yaml.WriteString("\n") - } - } - - // Add AI execution step using the agentic engine compilerYamlLog.Printf("Generating engine execution steps for %s", engine.GetID()) c.generateEngineExecutionSteps(yaml, data, engine, logFileFull) - - // Stop CLI proxy after AWF execution (always runs to ensure cleanup) c.generateStopCliProxyStep(yaml, data) - - // Detect agent errors on the host runner immediately after the AWF container exits. - // GITHUB_OUTPUT is not accessible inside the AWF sandbox, so this step must run here - // (on the host runner) rather than from within the container. Engines that provide a - // detection script via GetErrorDetectionScriptId will emit this step. c.generateDetectAgentErrorsStep(yaml, data, engine) - // Mark that we've completed agent execution - step order validation starts from here compilerYamlLog.Print("Marking agent execution as complete for step order tracking") c.stepOrderTracker.MarkAgentExecutionComplete() - // Regenerate git credentials after agent execution - // This allows safe-outputs operations (like create_pull_request) to work properly - // We regenerate the credentials rather than restoring from backup. - // Only emit these steps when a checkout was performed (requires a .git directory). if needsGitConfig { gitConfigStepsAfterAgent := c.generateGitConfigurationSteps() for _, line := range gitConfigStepsAfterAgent { @@ -465,39 +370,13 @@ func (c *Compiler) generateAgentRunSteps(yaml *strings.Builder, data *WorkflowDa } } - // Collect firewall logs BEFORE secret redaction so secrets in logs can be redacted - for _, step := range engine.GetFirewallLogsCollectionStep(data) { - for _, line := range step { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } + writeNestedLineSteps(yaml, engine.GetFirewallLogsCollectionStep(data)) + writeNestedLineSteps(yaml, engine.GetPreBundleSteps(data)) - // Run engine pre-bundle steps to relocate files before secret redaction. - // This ensures all artifact paths share a common ancestor under /tmp/gh-aw/. - for _, step := range engine.GetPreBundleSteps(data) { - for _, line := range step { - yaml.WriteString(line) - yaml.WriteByte('\n') - } - } - - // Stop MCP gateway after agent execution and before secret redaction - // This ensures the gateway process is properly cleaned up - // The MCP gateway is always enabled, even when agent sandbox is disabled c.generateStopMCPGateway(yaml, data) - - // Add secret redaction step BEFORE any artifact uploads - // This ensures all artifacts are scanned for secrets before being uploaded c.generateSecretRedactionStep(yaml, yaml.String(), data) - - // Append the agent step summary to the real $GITHUB_STEP_SUMMARY after secrets are redacted. - // The agent writes its GITHUB_STEP_SUMMARY content to AgentStepSummaryPath (a file inside - // /tmp/gh-aw/ that is reachable in both AWF sandbox and non-sandbox modes). - // secret redaction already scanned this file, so it is safe to append. c.generateAgentStepSummaryAppend(yaml) - // Add output collection step only if safe-outputs feature is used (GH_AW_SAFE_OUTPUTS functionality) if data.SafeOutputs != nil { if err := c.generateOutputCollectionStep(yaml, data); err != nil { return nil, "", err @@ -506,3 +385,28 @@ func (c *Compiler) generateAgentRunSteps(yaml *strings.Builder, data *WorkflowDa return artifactPaths, logFileFull, nil } + +func writeEngineConfigSteps(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine) error { + if len(data.EngineConfigSteps) == 0 { + return nil + } + compilerYamlLog.Printf("Adding %d engine config steps for %s", len(data.EngineConfigSteps), engine.GetID()) + for _, step := range data.EngineConfigSteps { + stepYAML, err := ConvertStepToYAML(step) + if err != nil { + return fmt.Errorf("failed to render engine config step: %w", err) + } + yaml.WriteString(stepYAML) + } + return nil +} + +func writeDockerSbxCredentialRefreshStep(yaml *strings.Builder, data *WorkflowData) { + if !isDockerSbxRuntime(data) { + return + } + for _, line := range generateDockerSbxCredentialRefreshStep() { + yaml.WriteString(line) + yaml.WriteString("\n") + } +} diff --git a/pkg/workflow/compiler_yaml_checkout.go b/pkg/workflow/compiler_yaml_checkout.go index 67266d579da..1f2308903c2 100644 --- a/pkg/workflow/compiler_yaml_checkout.go +++ b/pkg/workflow/compiler_yaml_checkout.go @@ -12,140 +12,115 @@ import ( // It returns the CheckoutManager (needed later for token invalidation and dev-mode restore) // and a flag indicating whether the default workspace checkout was emitted. func (c *Compiler) generateInitialAndCheckoutSteps(yaml *strings.Builder, data *WorkflowData) (*CheckoutManager, bool, error) { - // Mask OTLP telemetry headers early so authentication tokens cannot leak in runner - // debug logs. The workflow-level OTEL_EXPORTER_OTLP_HEADERS env var is available - // from the very first step, so masking can happen before any other work. + writeEarlyMaskSteps(yaml, data) + c.generatePreSteps(yaml, data) + + needsCheckout := c.shouldAddCheckoutStep(data) + compilerYamlLog.Printf("Checkout step needed: %t", needsCheckout) + checkoutMgr := c.prepareCheckoutManager(yaml, data) + + if needsCheckout { + c.writeDefaultCheckoutAndDevSteps(yaml, data, checkoutMgr) + } + + writeCheckoutManagerSteps(yaml, checkoutMgr, c.getActionPin) + c.writeImportCheckoutSteps(yaml, data) + + if err := writeMergeRemoteGithubFolderStep(yaml, data); err != nil { + return nil, false, err + } + + return checkoutMgr, needsCheckout, nil +} + +func writeEarlyMaskSteps(yaml *strings.Builder, data *WorkflowData) { if isOTLPHeadersPresent(data) { yaml.WriteString(generateOTLPHeadersMaskStep()) } - // Mask custom OTLP attribute values so user-supplied values cannot leak into runner logs. if isOTLPAttributesPresent(data) { yaml.WriteString(generateOTLPAttributesMaskStep()) } +} - // Add pre-steps before checkout and the subsequent built-in steps in this agent job. - // This allows users to mint short-lived tokens (via custom actions) in the same - // job as checkout, so the tokens are never dropped by the GitHub Actions runner's - // add-mask behaviour that silently suppresses masked values across job boundaries. - // Step outputs are available as ${{ steps..outputs. }} and can be - // referenced directly in checkout.token. Some compiler-injected setup steps may - // still be emitted earlier than these pre-steps. - c.generatePreSteps(yaml, data) - - // Determine if we need to add a checkout step - needsCheckout := c.shouldAddCheckoutStep(data) - compilerYamlLog.Printf("Checkout step needed: %t", needsCheckout) - - // Build a CheckoutManager with any user-configured checkouts +func (c *Compiler) prepareCheckoutManager(yaml *strings.Builder, data *WorkflowData) *CheckoutManager { checkoutMgr := NewCheckoutManager(data.CheckoutConfigs) - - // Propagate the platform (host) repo resolved by the activation job so that - // checkout steps in this job and in safe_outputs can use the correct repository - // for .github/.agents sparse checkouts when called cross-repo. - // The activation job exposes this as needs.activation.outputs.target_repo. if hasWorkflowCallTrigger(data.On) && !data.InlinedImports { checkoutMgr.SetCrossRepoTargetRepo("${{ needs.activation.outputs.target_repo }}") } - - // Mint checkout app tokens directly in the agent job before checkout steps are executed. - // Tokens cannot be passed via job outputs from the activation job because - // actions/create-github-app-token calls ::add-mask:: on the token, and the GitHub Actions - // runner silently drops masked values when used as job outputs (runner v2.308+). - // By minting here, the token is available as steps.checkout-app-token-{index}.outputs.token - // within the same job, just like the github-mcp-app-token pattern. if checkoutMgr.HasAppAuth() { compilerYamlLog.Print("Generating checkout app token minting steps in agent job") for _, step := range checkoutMgr.GenerateCheckoutAppTokenSteps(c, resolveCheckoutPermissions(data)) { yaml.WriteString(step) } } + return checkoutMgr +} - // Add checkout step first if needed - if needsCheckout { - // Emit the default workspace checkout, applying any user-supplied overrides - defaultLines := checkoutMgr.GenerateDefaultCheckoutStep( - c.trialMode, - c.trialLogicalRepoSlug, - c.getActionPin, - ) - for _, line := range defaultLines { - yaml.WriteString(line) - } - - // Add CLI build steps in dev mode (after automatic checkout, before other steps) - // This builds the gh-aw CLI and Docker image for use by the agentic-workflows MCP server - // Only generate build steps if agentic-workflows tool is enabled - if c.actionMode.IsDev() { - if _, hasAgenticWorkflows := data.Tools["agentic-workflows"]; hasAgenticWorkflows { - compilerYamlLog.Printf("Generating CLI build steps for dev mode (agentic-workflows tool enabled)") - c.generateDevModeCLIBuildSteps(yaml) - } else { - compilerYamlLog.Printf("Skipping CLI build steps in dev mode (agentic-workflows tool not enabled)") - } - } +func (c *Compiler) writeDefaultCheckoutAndDevSteps(yaml *strings.Builder, data *WorkflowData, checkoutMgr *CheckoutManager) { + for _, line := range checkoutMgr.GenerateDefaultCheckoutStep(c.trialMode, c.trialLogicalRepoSlug, c.getActionPin) { + yaml.WriteString(line) + } + if !c.actionMode.IsDev() { + return } + if _, hasAgenticWorkflows := data.Tools["agentic-workflows"]; hasAgenticWorkflows { + compilerYamlLog.Printf("Generating CLI build steps for dev mode (agentic-workflows tool enabled)") + c.generateDevModeCLIBuildSteps(yaml) + } else { + compilerYamlLog.Printf("Skipping CLI build steps in dev mode (agentic-workflows tool not enabled)") + } +} - // Emit additional (non-default) user-configured checkouts - additionalLines := checkoutMgr.GenerateAdditionalCheckoutSteps(c.getActionPin) - for _, line := range additionalLines { +func writeCheckoutManagerSteps(yaml *strings.Builder, checkoutMgr *CheckoutManager, getActionPin func(string) string) { + for _, line := range checkoutMgr.GenerateAdditionalCheckoutSteps(getActionPin) { yaml.WriteString(line) } - - // Emit a manifest step that records the path and resolved default branch for each - // non-default cross-repo checkout. The safe-outputs MCP server reads this file to - // resolve base branches without making any credentialed network calls. - for _, line := range checkoutMgr.GenerateCheckoutManifestStep(c.getActionPin) { + for _, line := range checkoutMgr.GenerateCheckoutManifestStep(getActionPin) { yaml.WriteString(line) } +} - // Add checkout steps for repository imports - // Each repository import needs to be checked out into a temporary folder - // so the merge script can copy files from it +func (c *Compiler) writeImportCheckoutSteps(yaml *strings.Builder, data *WorkflowData) { if len(data.RepositoryImports) > 0 { compilerYamlLog.Printf("Adding checkout steps for %d repository imports", len(data.RepositoryImports)) c.generateRepositoryImportCheckouts(yaml, data.RepositoryImports) } - - // Add checkout step for legacy agent import (if present) - // This handles the older import format where a specific agent file is imported if data.AgentFile != "" && data.AgentImportSpec != "" { compilerYamlLog.Printf("Adding checkout step for legacy agent import: %s", data.AgentImportSpec) c.generateLegacyAgentImportCheckout(yaml, data.AgentImportSpec) } +} - // Add merge remote .github folder step for repository imports or agent imports - needsGithubMerge := (len(data.RepositoryImports) > 0) || (data.AgentFile != "" && data.AgentImportSpec != "") - if needsGithubMerge { - compilerYamlLog.Printf("Adding merge remote .github folder step") - yaml.WriteString(" - name: Merge remote .github folder\n") - fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) - yaml.WriteString(" env:\n") - - // Set repository imports if present - if len(data.RepositoryImports) > 0 { - // Convert to JSON array for the script - repoImportsJSON, err := json.Marshal(data.RepositoryImports) - if err != nil { - return nil, false, fmt.Errorf("failed to marshal repository imports for merge step: %w", err) - } - writeYAMLEnv(yaml, " ", "GH_AW_REPOSITORY_IMPORTS", string(repoImportsJSON)) - } - - // Set agent import spec if present (legacy path) - if data.AgentFile != "" && data.AgentImportSpec != "" { - writeYAMLEnv(yaml, " ", "GH_AW_AGENT_FILE", data.AgentFile) - writeYAMLEnv(yaml, " ", "GH_AW_AGENT_IMPORT_SPEC", data.AgentImportSpec) +func writeMergeRemoteGithubFolderStep(yaml *strings.Builder, data *WorkflowData) error { + if len(data.RepositoryImports) == 0 && (data.AgentFile == "" || data.AgentImportSpec == "") { + return nil + } + compilerYamlLog.Printf("Adding merge remote .github folder step") + yaml.WriteString(" - name: Merge remote .github folder\n") + fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) + yaml.WriteString(" env:\n") + if len(data.RepositoryImports) > 0 { + repoImportsJSON, err := json.Marshal(data.RepositoryImports) + if err != nil { + return fmt.Errorf("failed to marshal repository imports for merge step: %w", err) } - - yaml.WriteString(" with:\n") - yaml.WriteString(" script: |\n") - yaml.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") - yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") - yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/merge_remote_agent_github_folder.cjs');\n") - yaml.WriteString(" await main();\n") + writeYAMLEnv(yaml, " ", "GH_AW_REPOSITORY_IMPORTS", string(repoImportsJSON)) } + if data.AgentFile != "" && data.AgentImportSpec != "" { + writeYAMLEnv(yaml, " ", "GH_AW_AGENT_FILE", data.AgentFile) + writeYAMLEnv(yaml, " ", "GH_AW_AGENT_IMPORT_SPEC", data.AgentImportSpec) + } + writeMergeRemoteGithubFolderScript(yaml) + return nil +} - return checkoutMgr, needsCheckout, nil +func writeMergeRemoteGithubFolderScript(yaml *strings.Builder) { + yaml.WriteString(" with:\n") + yaml.WriteString(" script: |\n") + yaml.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") + yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") + yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/merge_remote_agent_github_folder.cjs');\n") + yaml.WriteString(" await main();\n") } // generateRepositoryImportCheckouts generates checkout steps for repository imports diff --git a/pkg/workflow/compiler_yaml_header.go b/pkg/workflow/compiler_yaml_header.go index a7a85f8b63a..8613d68455c 100644 --- a/pkg/workflow/compiler_yaml_header.go +++ b/pkg/workflow/compiler_yaml_header.go @@ -23,81 +23,22 @@ func (c *Compiler) generateWorkflowHeader(yaml *strings.Builder, data *WorkflowD return } - // Add lock metadata as the very first line for easy machine parsing. - // Single-line JSON format to minimize merge conflicts. - if frontmatterHash != "" { - agentInfo := AgentMetadataInfo{} - // Agent ID: prefer EngineConfig.ID, fall back to legacy AI field - if data.EngineConfig != nil && data.EngineConfig.ID != "" { - agentInfo.AgentID = data.EngineConfig.ID - } else if data.AI != "" { - agentInfo.AgentID = data.AI - } - // Agent model: only include if statically configured - if data.EngineConfig != nil && data.EngineConfig.Model != "" { - agentInfo.AgentModel = data.EngineConfig.Model - } - // Detection agent info: only if threat detection has its own engine config - if data.SafeOutputs != nil && data.SafeOutputs.ThreatDetection != nil && data.SafeOutputs.ThreatDetection.EngineConfig != nil { - agentInfo.DetectionAgentID = data.SafeOutputs.ThreatDetection.EngineConfig.ID - agentInfo.DetectionAgentModel = data.SafeOutputs.ThreatDetection.EngineConfig.Model - } - agentInfo.EngineVersions = collectEngineVersionsForMetadata(data) - agentInfo.AgentImageRunner = resolveAgentImageRunnerIdentifier(data.RawFrontmatter) - metadata := GenerateLockMetadata(LockHashInfo{FrontmatterHash: frontmatterHash, BodyHash: bodyHash}, data.StopTime, c.effectiveStrictMode(data.RawFrontmatter), agentInfo) - if metadata.CompilerVersion == "" && c.GetActionTag() != "" { - metadata.CompilerVersion = c.GetVersion() - } - metadataJSON, err := metadata.ToJSON() - if err != nil { - // Fallback to legacy format if JSON serialization fails - fmt.Fprintf(yaml, "# frontmatter-hash: %s\n", frontmatterHash) - } else { - fmt.Fprintf(yaml, "# gh-aw-metadata: %s\n", metadataJSON) - } - } - - // Embed the gh-aw-manifest immediately after gh-aw-metadata for easy machine parsing. - // The manifest records all secrets, external actions, container images, and frontmatter - // skills detected at compile time so that subsequent compilations can perform safe update - // enforcement. - manifest := NewGHAWManifest(secrets, actions, data.ActionResolutionFailures, data.DockerImagePins, data.Redirect, data.Skills, data.RawFrontmatter["on"]) - if manifestJSON, err := manifest.ToJSON(); err == nil { - fmt.Fprintf(yaml, "# gh-aw-manifest: %s\n", manifestJSON) - } else { - compilerYamlHeaderLog.Printf("Failed to serialize gh-aw-manifest: %v. Safe update mode will not be available for future compilations of this workflow.", err) - } - - // Add workflow header with logo and instructions - sourceFile := "the corresponding .md file" - if data.Source != "" { - sourceFile = data.Source - } - header := GenerateWorkflowHeader(sourceFile, "gh-aw", "") - yaml.WriteString(header) - - // Add description comment if provided - if data.Description != "" { - cleanDescription := stringutil.StripANSI(data.Description) - // Split description into lines and prefix each with "# " - descriptionLines := strings.SplitSeq(strings.TrimSpace(cleanDescription), "\n") - for line := range descriptionLines { - fmt.Fprintf(yaml, "# %s\n", strings.TrimSpace(line)) - } - } + c.writeHeaderMetadata(yaml, data, frontmatterHash, bodyHash) + writeHeaderManifest(yaml, data, secrets, actions) + writeGeneratedWorkflowBanner(yaml, data) + writeDescriptionAndSourceComments(yaml, data) + writeResolvedWorkflowManifest(yaml, data) + writeInlinedImportsComment(yaml, data) + writeEnvSourcesComment(yaml, data) + writeListComment(yaml, "Secrets used:", secrets) + writeListComment(yaml, "Custom actions used:", actions) + writeListComment(yaml, "Container images used:", data.DockerImages) + writeStopAndManualApprovalComments(yaml, data) - // Add source comment if provided - if data.Source != "" { - yaml.WriteString("#\n") - cleanSource := stringutil.StripANSI(data.Source) - // Normalize to Unix paths (forward slashes) for cross-platform compatibility - cleanSource = filepath.ToSlash(cleanSource) - fmt.Fprintf(yaml, "# Source: %s\n", cleanSource) - } + yaml.WriteString("\n") +} - // Add manifest of imported/included files if any exist - // Build a user-visible imports list by filtering out internal builtin engine paths - // (e.g. "@builtin:engines/copilot.md") which are implementation details. +func writeResolvedWorkflowManifest(yaml *strings.Builder, data *WorkflowData) { var visibleImports []string for _, file := range data.ImportedFiles { if !strings.HasPrefix(file, parser.BuiltinPathPrefix) { @@ -129,65 +70,112 @@ func (c *Compiler) generateWorkflowHeader(yaml *strings.Builder, data *WorkflowD } } } +} - // Add inlined-imports comment to indicate the field was used at compile time - if data.InlinedImports { - yaml.WriteString("#\n") - yaml.WriteString("# inlined-imports: true\n") - } - - // Add frontmatter-declared env vars with source attribution. - // Note: programmatically injected env vars (e.g. OTEL_* from OTLP config) are not listed here. +func writeEnvSourcesComment(yaml *strings.Builder, data *WorkflowData) { if len(data.EnvSources) > 0 { yaml.WriteString("#\n") yaml.WriteString("# Frontmatter env variables:\n") - // Sort keys for deterministic output keys := sliceutil.SortedKeys(data.EnvSources) for _, k := range keys { fmt.Fprintf(yaml, "# - %s: %s\n", k, data.EnvSources[k]) } } +} - // Add list of secrets referenced in the workflow - if len(secrets) > 0 { - yaml.WriteString("#\n") - yaml.WriteString("# Secrets used:\n") - for _, s := range secrets { - fmt.Fprintf(yaml, "# - %s\n", s) - } - } - - // Add list of external custom actions referenced in the workflow - if len(actions) > 0 { +func writeInlinedImportsComment(yaml *strings.Builder, data *WorkflowData) { + if data.InlinedImports { yaml.WriteString("#\n") - yaml.WriteString("# Custom actions used:\n") - for _, a := range actions { - fmt.Fprintf(yaml, "# - %s\n", a) - } + yaml.WriteString("# inlined-imports: true\n") } +} - // Add list of container images used in the workflow - if len(data.DockerImages) > 0 { +func writeListComment(yaml *strings.Builder, title string, items []string) { + if len(items) > 0 { yaml.WriteString("#\n") - yaml.WriteString("# Container images used:\n") - for _, img := range data.DockerImages { - fmt.Fprintf(yaml, "# - %s\n", img) + fmt.Fprintf(yaml, "# %s\n", title) + for _, item := range items { + fmt.Fprintf(yaml, "# - %s\n", item) } } +} - // Add stop-time comment if configured +func writeStopAndManualApprovalComments(yaml *strings.Builder, data *WorkflowData) { if data.StopTime != "" { yaml.WriteString("#\n") cleanStopTime := stringutil.StripANSI(data.StopTime) fmt.Fprintf(yaml, "# Effective stop-time: %s\n", cleanStopTime) } - - // Add manual-approval comment if configured if data.ManualApproval != "" { yaml.WriteString("#\n") cleanManualApproval := stringutil.StripANSI(data.ManualApproval) fmt.Fprintf(yaml, "# Manual approval required: environment '%s'\n", cleanManualApproval) } +} - yaml.WriteString("\n") +func (c *Compiler) writeHeaderMetadata(yaml *strings.Builder, data *WorkflowData, frontmatterHash string, bodyHash string) { + if frontmatterHash == "" { + return + } + agentInfo := buildHeaderAgentMetadata(data) + metadata := GenerateLockMetadata(LockHashInfo{FrontmatterHash: frontmatterHash, BodyHash: bodyHash}, data.StopTime, c.effectiveStrictMode(data.RawFrontmatter), agentInfo) + if metadata.CompilerVersion == "" && c.GetActionTag() != "" { + metadata.CompilerVersion = c.GetVersion() + } + if metadataJSON, err := metadata.ToJSON(); err == nil { + fmt.Fprintf(yaml, "# gh-aw-metadata: %s\n", metadataJSON) + } else { + fmt.Fprintf(yaml, "# frontmatter-hash: %s\n", frontmatterHash) + } +} + +func buildHeaderAgentMetadata(data *WorkflowData) AgentMetadataInfo { + agentInfo := AgentMetadataInfo{} + if data.EngineConfig != nil && data.EngineConfig.ID != "" { + agentInfo.AgentID = data.EngineConfig.ID + } else if data.AI != "" { + agentInfo.AgentID = data.AI + } + if data.EngineConfig != nil && data.EngineConfig.Model != "" { + agentInfo.AgentModel = data.EngineConfig.Model + } + if data.SafeOutputs != nil && data.SafeOutputs.ThreatDetection != nil && data.SafeOutputs.ThreatDetection.EngineConfig != nil { + agentInfo.DetectionAgentID = data.SafeOutputs.ThreatDetection.EngineConfig.ID + agentInfo.DetectionAgentModel = data.SafeOutputs.ThreatDetection.EngineConfig.Model + } + agentInfo.EngineVersions = collectEngineVersionsForMetadata(data) + agentInfo.AgentImageRunner = resolveAgentImageRunnerIdentifier(data.RawFrontmatter) + return agentInfo +} + +func writeHeaderManifest(yaml *strings.Builder, data *WorkflowData, secrets []string, actions []string) { + manifest := NewGHAWManifest(secrets, actions, data.ActionResolutionFailures, data.DockerImagePins, data.Redirect, data.Skills, data.RawFrontmatter["on"]) + if manifestJSON, err := manifest.ToJSON(); err == nil { + fmt.Fprintf(yaml, "# gh-aw-manifest: %s\n", manifestJSON) + } else { + compilerYamlHeaderLog.Printf("Failed to serialize gh-aw-manifest: %v. Safe update mode will not be available for future compilations of this workflow.", err) + } +} + +func writeGeneratedWorkflowBanner(yaml *strings.Builder, data *WorkflowData) { + sourceFile := "the corresponding .md file" + if data.Source != "" { + sourceFile = data.Source + } + yaml.WriteString(GenerateWorkflowHeader(sourceFile, "gh-aw", "")) +} + +func writeDescriptionAndSourceComments(yaml *strings.Builder, data *WorkflowData) { + if data.Description != "" { + cleanDescription := stringutil.StripANSI(data.Description) + descriptionLines := strings.SplitSeq(strings.TrimSpace(cleanDescription), "\n") + for line := range descriptionLines { + fmt.Fprintf(yaml, "# %s\n", strings.TrimSpace(line)) + } + } + if data.Source != "" { + yaml.WriteString("#\n") + cleanSource := filepath.ToSlash(stringutil.StripANSI(data.Source)) + fmt.Fprintf(yaml, "# Source: %s\n", cleanSource) + } } diff --git a/pkg/workflow/compiler_yaml_normalize.go b/pkg/workflow/compiler_yaml_normalize.go index 5f80a60daba..a0ef7796fef 100644 --- a/pkg/workflow/compiler_yaml_normalize.go +++ b/pkg/workflow/compiler_yaml_normalize.go @@ -29,110 +29,114 @@ const maxConsecutiveBlankLines = 2 // the input byte-by-byte and builds the result with a single pre-allocated strings.Builder. func normalizeBlankLines(yamlContent string) string { compilerYAMLNormalizeLog.Printf("Normalizing blank lines in %d bytes of YAML", len(yamlContent)) - var b strings.Builder - b.Grow(len(yamlContent)) - - // lastNonBlankEnd tracks the builder length immediately after writing the last - // non-blank line (including its trailing newline). It starts at 0 and is only - // advanced when a substantive line is written, so it stays 0 when all lines - // are whitespace-only or the input is empty. Every line — blank or not — still - // gets a '\n' written to b, so b.Len() and lastNonBlankEnd may diverge when - // there are trailing blank lines. - lastNonBlankEnd := 0 - // blankRun counts consecutive blank lines emitted since the last non-blank - // structural line, so runs longer than maxConsecutiveBlankLines can be - // collapsed outside block scalars. - blankRun := 0 - inBlockScalar := false - pendingBlockScalar := false - blockScalarHeaderIndent := 0 - blockScalarIndent := 0 + + n := newBlankLineNormalizer(len(yamlContent)) pos := 0 for pos < len(yamlContent) { - // Find the end of the current line. - end := strings.IndexByte(yamlContent[pos:], '\n') - var line string + end, line := nextYAMLLine(yamlContent, pos) + n.writeLine(line) if end == -1 { - line = yamlContent[pos:] - } else { - line = yamlContent[pos : pos+end] + break } + pos += end + 1 + } - processStructuralLine := true - trimmed := strings.TrimRight(line, " \t") - if pendingBlockScalar || inBlockScalar { - if trimmed == "" { - // Whitespace-only lines inside block scalars are still semantically - // blank, so emit them as empty lines but never cap the run. - b.WriteByte('\n') - processStructuralLine = false - } else { - lineIndent := countLeadingSpaces(line) - if pendingBlockScalar { - if lineIndent <= blockScalarHeaderIndent { - pendingBlockScalar = false - } else { - blockScalarIndent = lineIndent - inBlockScalar = true - pendingBlockScalar = false - } - } - if inBlockScalar { - if lineIndent < blockScalarIndent { - inBlockScalar = false - } else { - b.WriteString(line) - b.WriteByte('\n') - lastNonBlankEnd = b.Len() - processStructuralLine = false - } - } - } - } + return n.result() +} + +type blankLineNormalizer struct { + b strings.Builder + lastNonBlankEnd int + blankRun int + inBlockScalar bool + pendingBlockScalar bool + blockScalarHeaderIndent int + blockScalarIndent int +} + +func newBlankLineNormalizer(size int) *blankLineNormalizer { + n := &blankLineNormalizer{} + n.b.Grow(size) + return n +} + +func nextYAMLLine(content string, pos int) (int, string) { + end := strings.IndexByte(content[pos:], '\n') + if end == -1 { + return end, content[pos:] + } + return end, content[pos : pos+end] +} - if processStructuralLine { - if trimmed == "" { - // Blank structural line: emit at most maxConsecutiveBlankLines in a - // row so yamllint's empty-lines rule is never exceeded. lastNonBlankEnd - // is NOT updated here so that trailing blank lines (including a blank - // final "line" produced by a file that ends with "\n\n" or by - // whitespace-only text after the last real line) are excluded from the - // returned slice. - if blankRun < maxConsecutiveBlankLines { - b.WriteByte('\n') - blankRun++ - } - } else { - b.WriteString(trimmed) - b.WriteByte('\n') - lastNonBlankEnd = b.Len() - blankRun = 0 - if headerIndent, ok := blockScalarHeaderIndentForLine(trimmed); ok { - pendingBlockScalar = true - blockScalarHeaderIndent = headerIndent - } - } +func (n *blankLineNormalizer) writeLine(line string) { + processStructuralLine, trimmed := n.writeBlockScalarLine(line) + if !processStructuralLine { + return + } + n.writeStructuralLine(trimmed) +} + +func (n *blankLineNormalizer) writeBlockScalarLine(line string) (bool, string) { + trimmed := strings.TrimRight(line, " \t") + if !n.pendingBlockScalar && !n.inBlockScalar { + return true, trimmed + } + if trimmed == "" { + n.b.WriteByte('\n') + return false, trimmed + } + lineIndent := countLeadingSpaces(line) + if n.pendingBlockScalar { + if lineIndent <= n.blockScalarHeaderIndent { + n.pendingBlockScalar = false + } else { + n.blockScalarIndent = lineIndent + n.inBlockScalar = true + n.pendingBlockScalar = false } + } + if n.inBlockScalar && lineIndent >= n.blockScalarIndent { + n.b.WriteString(line) + n.b.WriteByte('\n') + n.lastNonBlankEnd = n.b.Len() + return false, trimmed + } + n.inBlockScalar = false + return true, trimmed +} - if end == -1 { - break +func (n *blankLineNormalizer) writeStructuralLine(trimmed string) { + if trimmed == "" { + if n.blankRun < maxConsecutiveBlankLines { + n.b.WriteByte('\n') + n.blankRun++ } - pos += end + 1 + return + } + n.b.WriteString(trimmed) + n.b.WriteByte('\n') + n.lastNonBlankEnd = n.b.Len() + n.blankRun = 0 + if headerIndent, ok := blockScalarHeaderIndentForLine(trimmed); ok { + n.pendingBlockScalar = true + n.blockScalarHeaderIndent = headerIndent } +} +func (n *blankLineNormalizer) result() string { // When lastNonBlankEnd is still 0 there were no non-blank lines at all // (empty input or all-whitespace). Return a single newline, which matches // the original strings.TrimRight(…, "\n") + "\n" behaviour for that case. // NOTE: b.String()[:0] must NOT be used here; the early return is intentional. - if lastNonBlankEnd == 0 { + if n.lastNonBlankEnd == 0 { compilerYAMLNormalizeLog.Print("Input contained no non-blank lines, returning single newline") return "\n" } // Slice the builder string to drop trailing blank lines. b.String() copies // the builder's internal buffer into a new string once; the slice avoids a // second copy that a separate strings.Builder trim would incur. - compilerYAMLNormalizeLog.Printf("Normalized YAML to %d bytes", lastNonBlankEnd) - return b.String()[:lastNonBlankEnd] + compilerYAMLNormalizeLog.Printf("Normalized YAML to %d bytes", n.lastNonBlankEnd) + return n.b.String()[:n.lastNonBlankEnd] } func countLeadingSpaces(line string) int { diff --git a/pkg/workflow/compiler_yaml_post_agent.go b/pkg/workflow/compiler_yaml_post_agent.go index f16eaf73865..93ab0e1b78d 100644 --- a/pkg/workflow/compiler_yaml_post_agent.go +++ b/pkg/workflow/compiler_yaml_post_agent.go @@ -14,113 +14,84 @@ import ( func (c *Compiler) collectArtifactPaths(data *WorkflowData, engine CodingAgentEngine, logFileFull string, initialPaths []string) []string { paths := initialPaths - // Merge engine-declared output files into the unified artifact instead of creating a - // separate agent_outputs artifact. - paths = append(paths, getEngineArtifactPaths(engine)...) + paths = append(paths, c.baseArtifactPaths(data, engine, logFileFull)...) + paths = append(paths, safeOutputArtifactPaths(data)...) + paths = append(paths, patchArtifactPaths(data)...) + paths = append(paths, firewallArtifactPaths(data)...) - // Collect MCP logs. - paths = append(paths, constants.TmpMcpLogsDir) + // For ARC/DinD, rewrite all /tmp/gh-aw/ paths to ${{ runner.temp }}/gh-aw/ so + // the artifact upload has a single root. A consolidation step (emitted before upload) + // copies the files from /tmp/gh-aw/ to the runner.temp location. See gh-aw#34896 Bug B. + if isArcDindTopology(data) { + paths = rewriteTmpGhAwPathsForArcDind(paths) + } - // Collect DIFC proxy logs (proxy-tls certs + container stderr) when proxy was injected - paths = append(paths, difcProxyLogPaths(data)...) + compilerYamlLog.Printf("Collected %d artifact path(s) for unified agent upload", len(paths)) + return paths +} - // Collect MCPScripts logs path if mcp-scripts is enabled +func (c *Compiler) baseArtifactPaths(data *WorkflowData, engine CodingAgentEngine, logFileFull string) []string { + paths := append([]string{}, getEngineArtifactPaths(engine)...) + paths = append(paths, constants.TmpMcpLogsDir) + paths = append(paths, difcProxyLogPaths(data)...) if IsMCPScriptsEnabled(data.MCPScripts) { paths = append(paths, constants.TmpMcpScriptsLogsDir) } - - // Include the aggregated agent_usage.json in the agent artifact so third-party - // tools can consume structured token data without parsing the step summary. - // Requires AWF v0.25.8+ if isFirewallEnabled(data) { paths = append(paths, constants.TmpGhAwDirSlash+constants.TokenUsageFilename) } - - // Collect agent stdio logs path for unified upload - paths = append(paths, logFileFull) - - // Include the pre-agent audit file (file listing of agent-related directories captured - // before agent execution) so it is available in the agent artifact for post-run inspection. - paths = append(paths, constants.PreAgentAuditFilePath) - - // Collect agent-generated files path for unified upload - // This directory is used by workflows that instruct the agent to write files - // (e.g., smoke-claude status summaries) - paths = append(paths, constants.TmpGhAwAgentDir) - - // Collect GitHub API rate-limit log for observability. - // Written by github_rate_limit_logger.cjs during REST API calls. - paths = append(paths, constants.TmpGhAwDirSlash+constants.GithubRateLimitsFilename) - - // Collect OTLP span mirror — enables post-hoc trace debugging without a live collector. - // Written by send_otlp_span.cjs; each line is a full OTLP/HTTP JSON traces payload. - // Only included when OTLP is configured for this workflow. + paths = append(paths, + logFileFull, + constants.PreAgentAuditFilePath, + constants.TmpGhAwAgentDir, + constants.TmpGhAwDirSlash+constants.GithubRateLimitsFilename, + ) if isOTLPEnabled(data) { paths = append(paths, constants.TmpGhAwDirSlash+constants.OtelJsonlFilename) paths = append(paths, constants.TmpGhAwDirSlash+constants.OtlpExportErrorsFilename) } + return paths +} - // Collect safe outputs and agent output paths for the unified artifact. - // These were previously uploaded as separate safe-output and agent-output artifacts. - if data.SafeOutputs != nil { - // Raw safe-output NDJSON (copied to /tmp/gh-aw/ by generateOutputCollectionStep) - paths = append(paths, constants.TmpGhAwDirSlash+constants.SafeOutputsFilename) - // Processed agent output JSON produced by collect_ndjson_output.cjs - paths = append(paths, constants.TmpGhAwDirSlash+constants.AgentOutputFilename) - if data.SafeOutputs.CommentMemory != nil { - paths = append(paths, constants.TmpCommentMemoryDir) - } +func safeOutputArtifactPaths(data *WorkflowData) []string { + if data.SafeOutputs == nil { + return nil } - - // Collect git patch path if safe-outputs with PR operations is configured. - // NOTE: Git patch generation has been moved to the safe-outputs MCP server. - // The patch is now generated when create_pull_request or push_to_pull_request_branch - // tools are called, providing immediate error feedback if no changes are present. - // Include patches in the artifact when: - // 1. Safe outputs needs them for checkout (non-staged create_pull_request/push_to_pull_request_branch) - // 2. Threat detection is enabled (detection job needs patches for security analysis, even when the - // safe-output handler is staged and doesn't need checkout itself) - threatDetectionNeedsPatches := IsDetectionJobEnabled(data.SafeOutputs) - if usesPatchesAndCheckouts(data.SafeOutputs) || threatDetectionNeedsPatches { - paths = append(paths, constants.TmpAwPatchGlob) - // Bundle files are generated when patch-format: bundle is configured. - // Both formats use the same download path in the safe_outputs job, so - // include the bundle glob unconditionally alongside the patch glob. - // The artifact upload step already sets if-no-files-found: ignore, so - // this is safe even when no bundle files exist. - paths = append(paths, constants.TmpAwBundleGlob) + paths := []string{ + constants.TmpGhAwDirSlash + constants.SafeOutputsFilename, + constants.TmpGhAwDirSlash + constants.AgentOutputFilename, + } + if data.SafeOutputs.CommentMemory != nil { + paths = append(paths, constants.TmpCommentMemoryDir) } + return paths +} - // Include firewall audit/observability logs in the unified agent artifact - // so all agent job outputs ship as a single artifact (AWF v0.25.0+). - if isFirewallEnabled(data) { - if isArcDindTopology(data) { - // On ARC/DinD, logs are under ${{ runner.temp }}/gh-aw (daemon-visible path). - // Use ${{ runner.temp }} because `with:` blocks expand Actions expressions, not shell vars. - paths = append(paths, constants.AWFConfigFilePathExpr) - paths = append(paths, constants.AWFProxyLogsDirExpr+"/") - paths = append(paths, constants.AWFAuditDirExpr+"/") - paths = append(paths, constants.AWFReflectFilePathExpr) - } else { - paths = append(paths, constants.AWFConfigFilePath) - paths = append(paths, constants.AWFProxyLogsDir+"/") - paths = append(paths, constants.AWFAuditDir+"/") - // Include the AWF /reflect payload persisted by the agent harness. - // Co-located under /tmp/gh-aw/sandbox/firewall/ so the existing - // chmod -R a+rX step covers its permissions before upload. - paths = append(paths, constants.AWFReflectFilePath) - } +func patchArtifactPaths(data *WorkflowData) []string { + if !usesPatchesAndCheckouts(data.SafeOutputs) && !IsDetectionJobEnabled(data.SafeOutputs) { + return nil } + return []string{constants.TmpAwPatchGlob, constants.TmpAwBundleGlob} +} - // For ARC/DinD, rewrite all /tmp/gh-aw/ paths to ${{ runner.temp }}/gh-aw/ so - // the artifact upload has a single root. A consolidation step (emitted before upload) - // copies the files from /tmp/gh-aw/ to the runner.temp location. See gh-aw#34896 Bug B. +func firewallArtifactPaths(data *WorkflowData) []string { + if !isFirewallEnabled(data) { + return nil + } if isArcDindTopology(data) { - paths = rewriteTmpGhAwPathsForArcDind(paths) + return []string{ + constants.AWFConfigFilePathExpr, + constants.AWFProxyLogsDirExpr + "/", + constants.AWFAuditDirExpr + "/", + constants.AWFReflectFilePathExpr, + } + } + return []string{ + constants.AWFConfigFilePath, + constants.AWFProxyLogsDir + "/", + constants.AWFAuditDir + "/", + constants.AWFReflectFilePath, } - - compilerYamlLog.Printf("Collected %d artifact path(s) for unified agent upload", len(paths)) - return paths } // generateSummarySteps emits all GITHUB_STEP_SUMMARY log-parsing steps for the agent job. @@ -173,98 +144,58 @@ func (c *Compiler) generateSummarySteps(yaml *strings.Builder, data *WorkflowDat // and step-order validation. func (c *Compiler) generatePostAgentCollectionAndUpload(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine, artifactPaths []string, logFileFull string, checkoutMgr *CheckoutManager) error { compilerYamlLog.Print("Generating post-agent collection and upload steps") - // Generate engine output cleanup step so workspace files are removed after collection. - // The engine-declared output paths are gathered by collectArtifactPaths below. + + c.generatePostAgentCleanupAndLogs(yaml, data, engine) + artifactPaths = c.collectArtifactPaths(data, engine, logFileFull, artifactPaths) + c.generateSummarySteps(yaml, data, engine) + c.generatePostAgentUploadsAndPostSteps(yaml, data) + c.generateUnifiedAgentArtifactUpload(yaml, data, artifactPaths) + c.generateAgentRestoreActionsSetupStep(yaml, checkoutMgr) + if err := c.stepOrderTracker.ValidateStepOrdering(); err != nil { + return fmt.Errorf("step ordering validation failed: %w", err) + } + return nil +} + +func (c *Compiler) generatePostAgentCleanupAndLogs(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine) { if len(getEngineArtifactPaths(engine)) > 0 { c.generateEngineOutputCleanup(yaml, engine) } - - // Extract and upload squid access logs (if any proxy tools were used) c.generateExtractAccessLogs(yaml, data.Tools) c.generateUploadAccessLogs(yaml, data.Tools) - - // Collect all artifact paths for the unified upload. - artifactPaths = c.collectArtifactPaths(data, engine, logFileFull, artifactPaths) - - // Emit all GITHUB_STEP_SUMMARY log-parsing steps. - c.generateSummarySteps(yaml, data, engine) - - // Write a minimal agent_output.json placeholder when the engine fails before - // producing any safe outputs, so downstream safe_outputs and conclusion jobs - // receive a valid (empty) JSON file instead of an ENOENT error. - // The placeholder is only written if the engine did not already write the file. if data.SafeOutputs != nil { c.generateAgentOutputPlaceholderStep(yaml) } - - // Add post-execution cleanup step for Copilot engine if copilotEngine, ok := engine.(*CopilotEngine); ok { - cleanupStep := copilotEngine.GetCleanupStep(data) - for _, line := range cleanupStep { + for _, line := range copilotEngine.GetCleanupStep(data) { yaml.WriteString(line) yaml.WriteByte('\n') } } +} - // Add repo-memory artifact upload to save state for push job +func (c *Compiler) generatePostAgentUploadsAndPostSteps(yaml *strings.Builder, data *WorkflowData) { generateRepoMemoryArtifactUpload(yaml, data, c.getActionPin) - - // Add cache-memory git commit steps (after agent execution, before validation) - // This commits agent-written changes to the current integrity branch. generateCacheMemoryGitCommitSteps(yaml, data) - - // Add cache-memory validation (after agent execution) - // This validates file types before cache is saved or uploaded generateCacheMemoryValidation(yaml, data) - - // Add cache-memory artifact upload (after agent execution) - // This ensures artifacts are uploaded after the agent has finished modifying the cache generateCacheMemoryArtifactUpload(yaml, data, c.getActionPin) - - // Add safe-outputs assets artifact upload (after agent execution) - // This creates a separate artifact for assets that will be downloaded by upload_assets job generateSafeOutputsAssetsArtifactUpload(yaml, data, c.getActionPin) - - // Add safe-outputs upload-artifact staging upload (after agent execution) - // This creates a separate artifact for files the model staged for artifact upload, - // to be downloaded and processed by the upload_artifact job generateSafeOutputsArtifactStagingUpload(yaml, data, c.getActionPin) - - // Add post-steps (if any) after AI execution c.generatePostSteps(yaml, data) - - // For ARC/DinD, consolidate all artifact files under ${{ runner.temp }}/gh-aw/ - // before upload. Without this, upload-artifact receives paths from two roots - // (/tmp/gh-aw/ and ${{ runner.temp }}/gh-aw/), computes "/" as the common ancestor, - // and creates a nested directory layout that breaks downstream artifact downloads. - // See gh-aw#34896 Bug B. if isArcDindTopology(data) { c.generateArcDindArtifactConsolidationStep(yaml) } +} - // Generate single unified artifact upload with all collected paths. - // In workflow_call context, apply the per-invocation prefix to avoid name clashes. +func (c *Compiler) generateUnifiedAgentArtifactUpload(yaml *strings.Builder, data *WorkflowData, artifactPaths []string) { agentArtifactPrefix := artifactPrefixExprForDownstreamJob(data) compilerYamlLog.Printf("Emitting unified agent artifact upload with %d path(s)", len(artifactPaths)) c.generateUnifiedArtifactUpload(yaml, artifactPaths, agentArtifactPrefix) +} - // In dev mode the setup action is referenced via a local path (./actions/setup), so its files - // live in the workspace. When a checkout: entry targets an external repository without a path - // (e.g. "checkout: [{repository: owner/other-repo}]"), actions/checkout replaces the workspace - // root with the external repository content, removing the actions/setup directory. - // Without restoring it, the runner's post-step for Setup Scripts would fail with - // "Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under .../actions/setup". - // We add a restore checkout step (if: always()) as the final step so the post-step - // can always find action.yml and complete its /tmp/gh-aw cleanup. +func (c *Compiler) generateAgentRestoreActionsSetupStep(yaml *strings.Builder, checkoutMgr *CheckoutManager) { if c.actionMode.IsDev() && checkoutMgr.HasExternalRootCheckout() { yaml.WriteString(c.generateRestoreActionsSetupStep()) compilerYamlLog.Print("Added restore actions folder step to agent job (dev mode with external root checkout)") } - - // Validate step ordering - this is a compiler check to ensure security - if err := c.stepOrderTracker.ValidateStepOrdering(); err != nil { - // This is a compiler bug if validation fails - return fmt.Errorf("step ordering validation failed: %w", err) - } - return nil } diff --git a/pkg/workflow/compiler_yaml_prompt.go b/pkg/workflow/compiler_yaml_prompt.go index de8107b3378..5d3e1c1ca4e 100644 --- a/pkg/workflow/compiler_yaml_prompt.go +++ b/pkg/workflow/compiler_yaml_prompt.go @@ -48,287 +48,210 @@ func splitContentIntoChunks(content string) []string { func (c *Compiler) generatePrompt(yaml *strings.Builder, data *WorkflowData, preActivationJobCreated bool, beforeActivationJobs []string) { compilerYamlPromptLog.Printf("Generating prompt for workflow: %s (markdown size: %d bytes)", data.Name, len(data.MarkdownContent)) - // Collect built-in prompt sections (these should be prepended to user prompt) builtinSections := c.collectPromptSections(data) compilerYamlPromptLog.Printf("Collected %d built-in prompt sections", len(builtinSections)) - // NEW APPROACH: Use runtime-import macros for imports without inputs - // - Imported markdown without inputs uses runtime-import macros (loaded at runtime) - // - Imported markdown with inputs is still inlined (compile-time substitution required) - // - Main workflow markdown body uses runtime-import to allow editing without recompilation - // This ensures consistency for most imports while maintaining import inputs functionality - // - // NOTE: When an engine does not support native agent-file handling - // (GetCapabilities().NativeAgentFile == false), the agent file content is already present in the - // prompt via the standard mechanisms below — no special Step 0 is needed: - // - Agent files WITHOUT inputs: path is in data.ImportPaths → included by Step 1b. - // - Agent files WITH inputs: content is in data.ImportedMarkdown → included by Step 1a. - // - inlined-imports mode: data.AgentFile is cleared; content is in data.ImportPaths. - // All current engines (Claude, Codex, Gemini, Copilot) use this mechanism: NativeAgentFile is false, - // and they read the fully-assembled prompt.txt in GetExecutionSteps. + state := &promptGenerationState{} + c.addPromptImportChunks(state, data) + c.addMainWorkflowExpressionMappings(state, data) + state.expressionMappings = filterExpressionsForActivation(state.expressionMappings, data.Jobs, beforeActivationJobs) + addExperimentExpressionMappings(state, data) + c.addMainWorkflowPromptChunks(state, data) + applyWorkflowDispatchFallbacks(state.expressionMappings, data.HasDispatchItemNumber) + + allExpressionMappings := c.generateUnifiedPromptCreationStep(yaml, builtinSections, state.userPromptChunks, state.expressionMappings, data) + allExpressionMappings = mergeKnownNeedsExpressionMappings(allExpressionMappings, data, preActivationJobCreated) + c.generatePromptPostProcessingSteps(yaml, state.expressionMappings, allExpressionMappings, data) +} - var userPromptChunks []string - var expressionMappings []*ExpressionMapping +type promptGenerationState struct { + userPromptChunks []string + expressionMappings []*ExpressionMapping +} - // Step 1a/1b: Process imports in declaration order, interleaving: - // - compile-time inlined markdown (imports with inputs) - // - runtime-import macros (imports without inputs) - // In older workflow data (without PromptImports), fall back to legacy grouped handling. +func (c *Compiler) addPromptImportChunks(state *promptGenerationState, data *WorkflowData) { if len(data.PromptImports) > 0 { - compilerYamlPromptLog.Printf("Processing %d ordered prompt import entries", len(data.PromptImports)) - workspaceRoot := "" - hasImportInputs := len(data.ImportInputs) > 0 - if data.InlinedImports && c.markdownPath != "" { - workspaceRoot = resolveWorkspaceRoot(c.markdownPath) - } - for _, entry := range data.PromptImports { - if entry.Markdown != "" { - cleaned := removeXMLComments(entry.Markdown) - if hasImportInputs { - cleaned = SubstituteImportInputs(cleaned, data.ImportInputs) - } - chunks, exprMaps := extractPromptChunksFromMarkdown(cleaned) - userPromptChunks = append(userPromptChunks, chunks...) - expressionMappings = append(expressionMappings, exprMaps...) - continue - } - if entry.ImportPath == "" { - continue - } - importPath := filepath.ToSlash(entry.ImportPath) - if workspaceRoot != "" { - rawContent, err := os.ReadFile(filepath.Join(workspaceRoot, importPath)) - if err != nil { - compilerYamlPromptLog.Printf("Warning: failed to read import file %s (%v), falling back to runtime-import", importPath, err) - userPromptChunks = append(userPromptChunks, fmt.Sprintf("{{#runtime-import %s}}", importPath)) - continue - } - importedBody, extractErr := parser.ExtractMarkdownContent(string(rawContent)) - if extractErr != nil { - importedBody = string(rawContent) - } - chunks, exprMaps := extractPromptChunksFromMarkdown(importedBody) - userPromptChunks = append(userPromptChunks, chunks...) - expressionMappings = append(expressionMappings, exprMaps...) - continue - } - userPromptChunks = append(userPromptChunks, fmt.Sprintf("{{#runtime-import %s}}", importPath)) - } - } else { - // Step 1a: Process and inline imported markdown with inputs (if any) - // Imports with inputs MUST be inlined because substitution happens at compile time - if data.ImportedMarkdown != "" { - compilerYamlPromptLog.Printf("Processing imported markdown (%d bytes)", len(data.ImportedMarkdown)) + c.addOrderedPromptImportChunks(state, data) + return + } + c.addLegacyPromptImportChunks(state, data) +} - // Clean, substitute, and post-process imported markdown - cleaned := removeXMLComments(data.ImportedMarkdown) - if len(data.ImportInputs) > 0 { - compilerYamlPromptLog.Printf("Substituting %d import input values", len(data.ImportInputs)) - cleaned = SubstituteImportInputs(cleaned, data.ImportInputs) - } - chunks, exprMaps := extractPromptChunksFromMarkdown(cleaned) - userPromptChunks = append(userPromptChunks, chunks...) - expressionMappings = append(expressionMappings, exprMaps...) - compilerYamlPromptLog.Printf("Inlined imported markdown with inputs in %d chunks", len(chunks)) +func (c *Compiler) addOrderedPromptImportChunks(state *promptGenerationState, data *WorkflowData) { + compilerYamlPromptLog.Printf("Processing %d ordered prompt import entries", len(data.PromptImports)) + workspaceRoot := "" + hasImportInputs := len(data.ImportInputs) > 0 + if data.InlinedImports && c.markdownPath != "" { + workspaceRoot = resolveWorkspaceRoot(c.markdownPath) + } + for _, entry := range data.PromptImports { + if entry.Markdown != "" { + addMarkdownPromptChunks(state, entry.Markdown, hasImportInputs, data.ImportInputs) + continue } - - // Step 1b: For imports without inputs: - // - inlinedImports mode (inlined-imports: true frontmatter): read and inline content at compile time - // - normal mode: generate runtime-import macros (loaded at runtime) - if len(data.ImportPaths) > 0 { - if data.InlinedImports && c.markdownPath != "" { - // inlinedImports mode: read import file content from disk and embed directly - compilerYamlPromptLog.Printf("Inlining %d imports without inputs at compile time", len(data.ImportPaths)) - workspaceRoot := resolveWorkspaceRoot(c.markdownPath) - for _, importPath := range data.ImportPaths { - importPath = filepath.ToSlash(importPath) - rawContent, err := os.ReadFile(filepath.Join(workspaceRoot, importPath)) - if err != nil { - // Fall back to runtime-import macro if file cannot be read - compilerYamlPromptLog.Printf("Warning: failed to read import file %s (%v), falling back to runtime-import", importPath, err) - userPromptChunks = append(userPromptChunks, fmt.Sprintf("{{#runtime-import %s}}", importPath)) - continue - } - importedBody, extractErr := parser.ExtractMarkdownContent(string(rawContent)) - if extractErr != nil { - importedBody = string(rawContent) - } - chunks, exprMaps := extractPromptChunksFromMarkdown(importedBody) - userPromptChunks = append(userPromptChunks, chunks...) - expressionMappings = append(expressionMappings, exprMaps...) - compilerYamlPromptLog.Printf("Inlined import without inputs: %s", importPath) - } - } else { - // Normal mode: generate runtime-import macros (loaded at workflow runtime) - compilerYamlPromptLog.Printf("Generating runtime-import macros for %d imports without inputs", len(data.ImportPaths)) - for _, importPath := range data.ImportPaths { - importPath = filepath.ToSlash(importPath) - userPromptChunks = append(userPromptChunks, fmt.Sprintf("{{#runtime-import %s}}", importPath)) - compilerYamlPromptLog.Printf("Added runtime-import macro for: %s", importPath) - } - } + if entry.ImportPath != "" { + c.addPromptImportPathChunk(state, filepath.ToSlash(entry.ImportPath), workspaceRoot) } } +} - // Step 1.5: Extract expressions from main workflow markdown (not imported content) - // This is needed for needs.* expressions and other compile-time expressions - // The main workflow markdown uses runtime-import, but expressions like needs.* must be - // available at compile time for the substitute placeholders step - // Use MainWorkflowMarkdown (not MarkdownContent) to avoid extracting from imported content - // Skip this step when inlinePrompt is true because expression extraction happens in Step 2 - if !c.inlinePrompt && !data.InlinedImports && data.MainWorkflowMarkdown != "" { - compilerYamlPromptLog.Printf("Extracting expressions from main workflow markdown (%d bytes)", len(data.MainWorkflowMarkdown)) - - // Create a new extractor for main workflow markdown - mainExtractor := NewExpressionExtractor() - mainExprMappings, err := mainExtractor.ExtractExpressions(data.MainWorkflowMarkdown) - if err == nil && len(mainExprMappings) > 0 { - compilerYamlPromptLog.Printf("Extracted %d expressions from main workflow markdown", len(mainExprMappings)) - // Merge with imported expressions (append to existing mappings) - expressionMappings = append(expressionMappings, mainExprMappings...) - } +func addMarkdownPromptChunks(state *promptGenerationState, markdown string, substituteInputs bool, importInputs map[string]any) { + cleaned := removeXMLComments(markdown) + if substituteInputs { + cleaned = SubstituteImportInputs(cleaned, importInputs) } + chunks, exprMaps := extractPromptChunksFromMarkdown(cleaned) + state.userPromptChunks = append(state.userPromptChunks, chunks...) + state.expressionMappings = append(state.expressionMappings, exprMaps...) +} - // Filter out expression mappings referencing custom jobs that run AFTER activation. - // These jobs (which explicitly depend on activation) cannot have outputs available when - // the activation job builds and substitutes the prompt. Keeping them would cause actionlint - // errors because the jobs are not in activation's needs, yet their outputs would be - // referenced in activation's step env vars. - expressionMappings = filterExpressionsForActivation(expressionMappings, data.Jobs, beforeActivationJobs) - - // Add expression mappings for declared experiments. - // These ensure the interpolation and substitution steps have GH_AW_EXPERIMENTS_* env vars - // set from pick-experiment step outputs, which is required for: - // - Step 2.5 of interpolate_prompt.cjs: substitutes __GH_AW_EXPERIMENTS_*__ placeholders - // produced by runtime_import.cjs from {{#if experiments.name}} template conditionals. - // - The substitute_placeholders step: replaces any remaining occurrences. - if len(data.Experiments) > 0 { - experimentMappings := ExperimentExpressionMappings(data.Experiments) - compilerYamlPromptLog.Printf("Adding %d experiment expression mapping(s)", len(experimentMappings)) - expressionMappings = append(expressionMappings, experimentMappings...) +func (c *Compiler) addPromptImportPathChunk(state *promptGenerationState, importPath, workspaceRoot string) bool { + if workspaceRoot == "" { + state.userPromptChunks = append(state.userPromptChunks, fmt.Sprintf("{{#runtime-import %s}}", importPath)) + return false } + rawContent, err := os.ReadFile(filepath.Join(workspaceRoot, importPath)) + if err != nil { + compilerYamlPromptLog.Printf("Warning: failed to read import file %s (%v), falling back to runtime-import", importPath, err) + state.userPromptChunks = append(state.userPromptChunks, fmt.Sprintf("{{#runtime-import %s}}", importPath)) + return false + } + importedBody, extractErr := parser.ExtractMarkdownContent(string(rawContent)) + if extractErr != nil { + importedBody = string(rawContent) + } + chunks, exprMaps := extractPromptChunksFromMarkdown(importedBody) + state.userPromptChunks = append(state.userPromptChunks, chunks...) + state.expressionMappings = append(state.expressionMappings, exprMaps...) + return true +} - // Step 2: Add main workflow markdown content to the prompt - if c.inlinePrompt || data.InlinedImports { - // Inline mode (Wasm/browser): embed the markdown content directly in the YAML - // since runtime-import macros cannot resolve without filesystem access - if data.MainWorkflowMarkdown != "" { - compilerYamlPromptLog.Printf("Inlining main workflow markdown (%d bytes)", len(data.MainWorkflowMarkdown)) - - inlinedMarkdown := removeXMLComments(data.MainWorkflowMarkdown) - inlinedMarkdown = wrapExpressionsInTemplateConditionals(inlinedMarkdown) +func (c *Compiler) addLegacyPromptImportChunks(state *promptGenerationState, data *WorkflowData) { + if data.ImportedMarkdown != "" { + compilerYamlPromptLog.Printf("Processing imported markdown (%d bytes)", len(data.ImportedMarkdown)) + if len(data.ImportInputs) > 0 { + compilerYamlPromptLog.Printf("Substituting %d import input values", len(data.ImportInputs)) + } + before := len(state.userPromptChunks) + addMarkdownPromptChunks(state, data.ImportedMarkdown, len(data.ImportInputs) > 0, data.ImportInputs) + compilerYamlPromptLog.Printf("Inlined imported markdown with inputs in %d chunks", len(state.userPromptChunks)-before) + } + if len(data.ImportPaths) > 0 { + c.addLegacyImportPathChunks(state, data) + } +} - // Extract expressions and replace with env var references - inlineExtractor := NewExpressionExtractor() - inlineExprMappings, err := inlineExtractor.ExtractExpressions(inlinedMarkdown) - if err == nil && len(inlineExprMappings) > 0 { - inlinedMarkdown = inlineExtractor.ReplaceExpressionsWithEnvVars(inlinedMarkdown) - expressionMappings = append(expressionMappings, inlineExprMappings...) +func (c *Compiler) addLegacyImportPathChunks(state *promptGenerationState, data *WorkflowData) { + if data.InlinedImports && c.markdownPath != "" { + compilerYamlPromptLog.Printf("Inlining %d imports without inputs at compile time", len(data.ImportPaths)) + workspaceRoot := resolveWorkspaceRoot(c.markdownPath) + for _, importPath := range data.ImportPaths { + importPath = filepath.ToSlash(importPath) + if c.addPromptImportPathChunk(state, importPath, workspaceRoot) { + compilerYamlPromptLog.Printf("Inlined import without inputs: %s", importPath) } - - inlinedChunks := splitContentIntoChunks(inlinedMarkdown) - userPromptChunks = append(userPromptChunks, inlinedChunks...) - compilerYamlPromptLog.Printf("Inlined main workflow markdown in %d chunks", len(inlinedChunks)) } - } else { - // Normal mode: use runtime-import macro so users can edit without recompilation - workflowBasename := filepath.Base(c.markdownPath) - - // Determine the directory path relative to workspace root - // For a workflow at ".github/workflows/test.md", the runtime-import path should be ".github/workflows/test.md" - // This makes the path explicit and matches the actual file location in the repository - var workflowFilePath string - - // Normalize path separators first to handle both Unix and Windows paths consistently - normalizedPath := filepath.ToSlash(c.markdownPath) - - // Look for "/.github/" as a directory (not just substring in repo name like "username.github.io") - // We need to match the directory component, not arbitrary substrings. - // Use LastIndex so that when the repo itself is named ".github" (path like - // "/root/.github/.github/workflows/file.md"), we find the actual .github - // workflows directory rather than the repo root directory. - githubDirPattern := "/.github/" - githubIndex := strings.LastIndex(normalizedPath, githubDirPattern) + return + } + compilerYamlPromptLog.Printf("Generating runtime-import macros for %d imports without inputs", len(data.ImportPaths)) + for _, importPath := range data.ImportPaths { + importPath = filepath.ToSlash(importPath) + state.userPromptChunks = append(state.userPromptChunks, fmt.Sprintf("{{#runtime-import %s}}", importPath)) + compilerYamlPromptLog.Printf("Added runtime-import macro for: %s", importPath) + } +} - if githubIndex != -1 { - // Extract everything from ".github/" onwards (inclusive) - // +1 to skip the leading slash, so we get ".github/workflows/..." not "/.github/workflows/..." - workflowFilePath = normalizedPath[githubIndex+1:] - } else if strings.HasPrefix(normalizedPath, constants.GithubDir) { - // Relative path already starting with ".github/" — use as-is. - // This can happen when the compiler is invoked with a relative markdown path - // (e.g. ".github/workflows/test.md") rather than an absolute one. - workflowFilePath = normalizedPath - } else { - // For non-standard paths (like /tmp/test.md), just use the basename - workflowFilePath = workflowBasename - } +func (c *Compiler) addMainWorkflowExpressionMappings(state *promptGenerationState, data *WorkflowData) { + if c.inlinePrompt || data.InlinedImports || data.MainWorkflowMarkdown == "" { + return + } + compilerYamlPromptLog.Printf("Extracting expressions from main workflow markdown (%d bytes)", len(data.MainWorkflowMarkdown)) + mainExtractor := NewExpressionExtractor() + mainExprMappings, err := mainExtractor.ExtractExpressions(data.MainWorkflowMarkdown) + if err == nil && len(mainExprMappings) > 0 { + compilerYamlPromptLog.Printf("Extracted %d expressions from main workflow markdown", len(mainExprMappings)) + state.expressionMappings = append(state.expressionMappings, mainExprMappings...) + } +} - // Create a runtime-import macro for the main workflow markdown - // The runtime_import.cjs helper will extract and process the markdown body at runtime - // The path uses .github/ prefix for clarity (e.g., .github/workflows/test.md) - runtimeImportMacro := fmt.Sprintf("{{#runtime-import %s}}", workflowFilePath) - compilerYamlPromptLog.Printf("Using runtime-import for main workflow markdown: %s", workflowFilePath) +func addExperimentExpressionMappings(state *promptGenerationState, data *WorkflowData) { + if len(data.Experiments) == 0 { + return + } + experimentMappings := ExperimentExpressionMappings(data.Experiments) + compilerYamlPromptLog.Printf("Adding %d experiment expression mapping(s)", len(experimentMappings)) + state.expressionMappings = append(state.expressionMappings, experimentMappings...) +} - // Append runtime-import macro after imported chunks - userPromptChunks = append(userPromptChunks, runtimeImportMacro) +func (c *Compiler) addMainWorkflowPromptChunks(state *promptGenerationState, data *WorkflowData) { + if c.inlinePrompt || data.InlinedImports { + addInlinedMainWorkflowPromptChunks(state, data) + return } + workflowFilePath := runtimeImportWorkflowPath(c.markdownPath) + runtimeImportMacro := fmt.Sprintf("{{#runtime-import %s}}", workflowFilePath) + compilerYamlPromptLog.Printf("Using runtime-import for main workflow markdown: %s", workflowFilePath) + state.userPromptChunks = append(state.userPromptChunks, runtimeImportMacro) +} - // Enhance entity number expressions with || inputs.item_number fallback when the - // workflow has a workflow_dispatch trigger with item_number (generated by the label - // trigger shorthand). This is applied after all expression mappings (including inline - // mode ones) have been collected so that every entity number reference gets the fallback. - applyWorkflowDispatchFallbacks(expressionMappings, data.HasDispatchItemNumber) +func addInlinedMainWorkflowPromptChunks(state *promptGenerationState, data *WorkflowData) { + if data.MainWorkflowMarkdown == "" { + return + } + compilerYamlPromptLog.Printf("Inlining main workflow markdown (%d bytes)", len(data.MainWorkflowMarkdown)) + inlinedMarkdown := removeXMLComments(data.MainWorkflowMarkdown) + inlinedMarkdown = wrapExpressionsInTemplateConditionals(inlinedMarkdown) + inlineExtractor := NewExpressionExtractor() + inlineExprMappings, err := inlineExtractor.ExtractExpressions(inlinedMarkdown) + if err == nil && len(inlineExprMappings) > 0 { + inlinedMarkdown = inlineExtractor.ReplaceExpressionsWithEnvVars(inlinedMarkdown) + state.expressionMappings = append(state.expressionMappings, inlineExprMappings...) + } + inlinedChunks := splitContentIntoChunks(inlinedMarkdown) + state.userPromptChunks = append(state.userPromptChunks, inlinedChunks...) + compilerYamlPromptLog.Printf("Inlined main workflow markdown in %d chunks", len(inlinedChunks)) +} - // Generate a single unified prompt creation step WITHOUT known needs expressions - // Known needs expressions are added later for the substitution step only - // This returns the combined expression mappings for use in the substitution step - allExpressionMappings := c.generateUnifiedPromptCreationStep(yaml, builtinSections, userPromptChunks, expressionMappings, data) +func runtimeImportWorkflowPath(markdownPath string) string { + workflowBasename := filepath.Base(markdownPath) + normalizedPath := filepath.ToSlash(markdownPath) + githubDirPattern := "/.github/" + githubIndex := strings.LastIndex(normalizedPath, githubDirPattern) + if githubIndex != -1 { + return normalizedPath[githubIndex+1:] + } + if strings.HasPrefix(normalizedPath, constants.GithubDir) { + return normalizedPath + } + return workflowBasename +} - // Step 1.6: Add all known needs.* expressions for the substitution step ONLY - // Since the markdown may change without recompilation (via runtime-import), we need to - // ensure all known needs.* variables are available for interpolation in the substitution step. - // These are NOT added to the prompt creation step because they're not needed there. +func mergeKnownNeedsExpressionMappings(allExpressionMappings []*ExpressionMapping, data *WorkflowData, preActivationJobCreated bool) []*ExpressionMapping { knownNeedsExpressions := generateKnownNeedsExpressions(data, preActivationJobCreated) - if len(knownNeedsExpressions) > 0 { - compilerYamlPromptLog.Printf("Adding %d known needs.* expressions for substitution step only", len(knownNeedsExpressions)) - // Merge known needs expressions with the returned expression mappings for substitution - // We use a map to avoid duplicates (expressions from markdown take precedence) - expressionMap := make(map[string]*ExpressionMapping) - // First add known needs expressions (these have lower priority) - for _, mapping := range knownNeedsExpressions { - expressionMap[mapping.EnvVar] = mapping - } - // Then add/override with expressions from allExpressionMappings (these have higher priority) - for _, mapping := range allExpressionMappings { - expressionMap[mapping.EnvVar] = mapping - } - // Convert back to slice in sorted order (by environment variable name) for deterministic output - allExpressionMappings = make([]*ExpressionMapping, 0, len(expressionMap)) - // Get all keys and sort them - envVarNames := sliceutil.SortedKeys(expressionMap) - // Add mappings in sorted order - for _, envVar := range envVarNames { - allExpressionMappings = append(allExpressionMappings, expressionMap[envVar]) - } + if len(knownNeedsExpressions) == 0 { + return allExpressionMappings } + compilerYamlPromptLog.Printf("Adding %d known needs.* expressions for substitution step only", len(knownNeedsExpressions)) + expressionMap := make(map[string]*ExpressionMapping) + for _, mapping := range knownNeedsExpressions { + expressionMap[mapping.EnvVar] = mapping + } + for _, mapping := range allExpressionMappings { + expressionMap[mapping.EnvVar] = mapping + } + merged := make([]*ExpressionMapping, 0, len(expressionMap)) + for _, envVar := range sliceutil.SortedKeys(expressionMap) { + merged = append(merged, expressionMap[envVar]) + } + return merged +} - // Add combined interpolation and template rendering step - // This step processes runtime-import macros, so it must run BEFORE placeholder substitution +func (c *Compiler) generatePromptPostProcessingSteps(yaml *strings.Builder, expressionMappings, allExpressionMappings []*ExpressionMapping, data *WorkflowData) { c.generateInterpolationAndTemplateStep(yaml, expressionMappings, data) - - // Generate JavaScript-based placeholder substitution step - // This MUST run AFTER interpolation because placeholders in runtime-imported files - // (like changeset.md) need to be substituted after the file is imported - // Now includes the known needs.* expressions if len(allExpressionMappings) > 0 { generatePlaceholderSubstitutionStep(yaml, allExpressionMappings, " ", data) } - - // Validate that all placeholders have been substituted writePromptBashStep(yaml, "Validate prompt placeholders", "validate_prompt_placeholders.sh") - - // Print prompt (merged into prompt generation) writePromptBashStep(yaml, "Print prompt", "print_prompt_summary.sh") } diff --git a/pkg/workflow/compiler_yaml_runtime_setup.go b/pkg/workflow/compiler_yaml_runtime_setup.go index 1331aa4d6cf..d10a5e9fa6e 100644 --- a/pkg/workflow/compiler_yaml_runtime_setup.go +++ b/pkg/workflow/compiler_yaml_runtime_setup.go @@ -329,89 +329,78 @@ func (c *Compiler) addCustomStepsAsIs(yaml *strings.Builder, customSteps string) // Like addCustomStepsAsIs it sanitizes any ${{ ... }} expressions found in run: fields before writing. func (c *Compiler) addCustomStepsWithRuntimeInsertion(yaml *strings.Builder, customSteps string, runtimeSetupSteps []GitHubActionStep, tools *ToolsConfig) { customSteps = c.sanitizeAndWarnCustomSteps(customSteps) - // Remove "steps:" line and adjust indentation lines := strings.Split(customSteps, "\n") if len(lines) <= 1 { return } insertedRuntime := false - i := 1 // Start from index 1 to skip "steps:" line - + i := 1 for i < len(lines) { line := lines[i] - // Skip empty lines if strings.TrimSpace(line) == "" { yaml.WriteString("\n") i++ continue } - // Add the line with proper indentation yaml.WriteString(" " + line + "\n") - // Check if this line starts a step with "- name:" or "- uses:" trimmed := strings.TrimSpace(line) - isStepStart := strings.HasPrefix(trimmed, "- name:") || strings.HasPrefix(trimmed, "- uses:") - - if isStepStart && !insertedRuntime { - // This is the start of a step, check if it's a checkout step - isCheckoutStep := false - - // Look ahead to find "uses:" line with "checkout" - for j := i + 1; j < len(lines); j++ { - nextLine := lines[j] - nextTrimmed := strings.TrimSpace(nextLine) - - // Stop if we hit the next step - if strings.HasPrefix(nextTrimmed, "- name:") || strings.HasPrefix(nextTrimmed, "- uses:") { - break - } - - // Check if this is a uses line with checkout - if strings.Contains(nextTrimmed, "uses:") && strings.Contains(nextTrimmed, "checkout") { - isCheckoutStep = true - break - } - } + if isCustomStepStart(trimmed) && !insertedRuntime && customStepContainsCheckout(lines, i+1) { + i = copyCustomStepRemainder(yaml, lines, i+1) + writeRuntimeSetupSteps(yaml, runtimeSetupSteps) + insertedRuntime = true + continue + } - if isCheckoutStep { - // This is a checkout step, copy all its lines until the next step - i++ - for i < len(lines) { - nextLine := lines[i] - nextTrimmed := strings.TrimSpace(nextLine) - - // Stop if we hit the next step - if strings.HasPrefix(nextTrimmed, "- name:") || strings.HasPrefix(nextTrimmed, "- uses:") { - break - } - - // Add the line - if nextTrimmed == "" { - yaml.WriteString("\n") - } else { - yaml.WriteString(" " + nextLine + "\n") - } - i++ - } - - // Now insert runtime steps after the checkout step - compilerYamlLog.Printf("Inserting %d runtime setup steps after checkout in custom steps", len(runtimeSetupSteps)) - for _, step := range runtimeSetupSteps { - for _, stepLine := range step { - yaml.WriteString(stepLine + "\n") - } - } - - insertedRuntime = true - continue // Continue with the next iteration (i is already advanced) - } + i++ + } +} + +func isCustomStepStart(trimmed string) bool { + return strings.HasPrefix(trimmed, "- name:") || strings.HasPrefix(trimmed, "- uses:") +} + +func customStepContainsCheckout(lines []string, start int) bool { + for j := start; j < len(lines); j++ { + nextTrimmed := strings.TrimSpace(lines[j]) + if isCustomStepStart(nextTrimmed) { + break } + if strings.Contains(nextTrimmed, "uses:") && strings.Contains(nextTrimmed, "checkout") { + return true + } + } + return false +} +func copyCustomStepRemainder(yaml *strings.Builder, lines []string, start int) int { + i := start + for i < len(lines) { + nextLine := lines[i] + nextTrimmed := strings.TrimSpace(nextLine) + if isCustomStepStart(nextTrimmed) { + break + } + if nextTrimmed == "" { + yaml.WriteString("\n") + } else { + yaml.WriteString(" " + nextLine + "\n") + } i++ } + return i +} + +func writeRuntimeSetupSteps(yaml *strings.Builder, runtimeSetupSteps []GitHubActionStep) { + compilerYamlLog.Printf("Inserting %d runtime setup steps after checkout in custom steps", len(runtimeSetupSteps)) + for _, step := range runtimeSetupSteps { + for _, stepLine := range step { + yaml.WriteString(stepLine + "\n") + } + } } // sanitizeAndWarnCustomSteps applies sanitizeCustomStepsYAML to the custom steps string, diff --git a/pkg/workflow/compiler_yaml_step_conversion.go b/pkg/workflow/compiler_yaml_step_conversion.go index ec946c3166b..068d1fea735 100644 --- a/pkg/workflow/compiler_yaml_step_conversion.go +++ b/pkg/workflow/compiler_yaml_step_conversion.go @@ -106,9 +106,6 @@ func unquoteUsesWithComments(yamlStr string) string { // renderStepFromMap renders a GitHub Actions step from a map to YAML func (c *Compiler) renderStepFromMap(out *strings.Builder, step map[string]any, data *WorkflowData, indent string) { - // Before rendering, extract any ${{ ... }} expressions from the run: field into - // env: variables to prevent shell injection attacks. A compiler warning is emitted - // for every expression that is moved so that authors know their script was changed. if sanitized, warnings, changed := sanitizeRunStepExpressions(step); changed { stepConversionLog.Printf("Sanitized run-step expressions: %d warning(s) emitted", len(warnings)) for _, w := range warnings { @@ -131,78 +128,54 @@ func (c *Compiler) renderStepFromMap(out *strings.Builder, step map[string]any, for _, field := range fieldOrder { if value, exists := step[field]; exists { - // Add proper indentation for non-first fields - if !firstField { - out.WriteString(indent + " ") - } - firstField = false - - // Render the field based on its type - switch v := value.(type) { - case string: - // Handle multi-line strings (especially for 'run' field) - if field == "run" && strings.Contains(v, "\n") { - fmt.Fprintf(out, "%s: |\n", field) - lines := strings.SplitSeq(v, "\n") - for line := range lines { - fmt.Fprintf(out, "%s %s\n", indent, line) - } - } else { - fmt.Fprintf(out, "%s: %s\n", field, v) - } - case map[string]any: - // For complex fields like "with" or "env" — sort keys for stable output. - fmt.Fprintf(out, "%s:\n", field) - for _, key := range sliceutil.SortedKeys(v) { - if field == "env" { - fmt.Fprintf(out, "%s %s: %s\n", indent, key, formatStepEnvValueForYAML(v[key])) - } else { - fmt.Fprintf(out, "%s %s: %v\n", indent, key, v[key]) - } - } - default: - fmt.Fprintf(out, "%s: %v\n", field, v) - } + renderStepField(out, field, value, indent, &firstField, field == "run") } } - // Add any remaining fields not in the predefined order for field, value := range step { - // Skip fields we've already processed skip := slices.Contains(fieldOrder, field) if skip { continue } + renderStepField(out, field, value, indent, &firstField, true) + } +} + +func renderStepField(out *strings.Builder, field string, value any, indent string, firstField *bool, multilineStrings bool) { + if !*firstField { + out.WriteString(indent + " ") + } + *firstField = false + + switch v := value.(type) { + case string: + renderStepStringField(out, field, v, indent, multilineStrings) + case map[string]any: + renderStepMapField(out, field, v, indent) + default: + fmt.Fprintf(out, "%s: %v\n", field, v) + } +} - if !firstField { - out.WriteString(indent + " ") +func renderStepStringField(out *strings.Builder, field, value, indent string, multilineStrings bool) { + if multilineStrings && strings.Contains(value, "\n") { + fmt.Fprintf(out, "%s: |\n", field) + lines := strings.SplitSeq(value, "\n") + for line := range lines { + fmt.Fprintf(out, "%s %s\n", indent, line) } - firstField = false - - switch v := value.(type) { - case string: - // Handle multi-line strings - if strings.Contains(v, "\n") { - fmt.Fprintf(out, "%s: |\n", field) - lines := strings.SplitSeq(v, "\n") - for line := range lines { - fmt.Fprintf(out, "%s %s\n", indent, line) - } - } else { - fmt.Fprintf(out, "%s: %s\n", field, v) - } - case map[string]any: - // Sort keys for stable output. - fmt.Fprintf(out, "%s:\n", field) - for _, key := range sliceutil.SortedKeys(v) { - if field == "env" { - fmt.Fprintf(out, "%s %s: %s\n", indent, key, formatStepEnvValueForYAML(v[key])) - } else { - fmt.Fprintf(out, "%s %s: %v\n", indent, key, v[key]) - } - } - default: - fmt.Fprintf(out, "%s: %v\n", field, v) + } else { + fmt.Fprintf(out, "%s: %s\n", field, value) + } +} + +func renderStepMapField(out *strings.Builder, field string, value map[string]any, indent string) { + fmt.Fprintf(out, "%s:\n", field) + for _, key := range sliceutil.SortedKeys(value) { + if field == "env" { + fmt.Fprintf(out, "%s %s: %s\n", indent, key, formatStepEnvValueForYAML(value[key])) + } else { + fmt.Fprintf(out, "%s %s: %v\n", indent, key, value[key]) } } } diff --git a/pkg/workflow/compiler_yaml_step_generation.go b/pkg/workflow/compiler_yaml_step_generation.go index 475e824d6db..4735012f5c9 100644 --- a/pkg/workflow/compiler_yaml_step_generation.go +++ b/pkg/workflow/compiler_yaml_step_generation.go @@ -175,67 +175,50 @@ func (c *Compiler) generateSetupStepWithArtifactClientCondition(data *WorkflowDa lines := c.generateOTLPOIDCMintStep(data) hasOTLPOIDC := len(lines) > 0 artifactClientCondition = strings.TrimSpace(artifactClientCondition) + setupEngineID := setupEngineIDForWorkflow(data) - setupEngineID := "" - if data != nil { - if data.EngineConfig != nil && data.EngineConfig.ID != "" { - setupEngineID = data.EngineConfig.ID - } else if data.AI != "" { - setupEngineID = data.AI - } - } - - // Script mode: run the setup.sh script directly if c.actionMode.IsScript() { - setupLines := []string{ - " - name: Setup Scripts\n", - " id: setup\n", - " run: |\n", - " bash /tmp/gh-aw/actions-source/actions/setup/setup.sh\n", - " env:\n", - fmt.Sprintf(" INPUT_DESTINATION: %s\n", destination), - " INPUT_JOB_NAME: ${{ github.job }}\n", - } - if data != nil { - setupLines = append(setupLines, - fmt.Sprintf(" GH_AW_SETUP_WORKFLOW_NAME: %q\n", data.Name), - fmt.Sprintf(" GH_AW_CURRENT_WORKFLOW_REF: %s\n", buildSetupWorkflowRefExpr(data)), - ) - if v := getVersionForSetup(data); v != "" { - setupLines = append(setupLines, fmt.Sprintf(" GH_AW_INFO_VERSION: %q\n", v)) - } - if v := getAWFVersionForSetup(data); v != "" { - setupLines = append(setupLines, fmt.Sprintf(" GH_AW_INFO_AWF_VERSION: %q\n", v)) - } - if data.Source != "" { - setupLines = append(setupLines, " GH_AW_INFO_BODY_MODIFIED: \"false\"\n") - } - if setupEngineID != "" { - setupLines = append(setupLines, fmt.Sprintf(" GH_AW_INFO_ENGINE_ID: %q\n", setupEngineID)) - } - } - if traceID != "" { - setupLines = append(setupLines, fmt.Sprintf(" INPUT_TRACE_ID: %s\n", traceID)) - } - if parentSpanID != "" { - setupLines = append(setupLines, fmt.Sprintf(" INPUT_PARENT_SPAN_ID: %s\n", parentSpanID)) - } - if hasOTLPOIDC { - setupLines = append(setupLines, " INPUT_OTLP_OIDC_TOKEN: ${{ steps.mint-otlp-oidc-token.outputs.token }}\n") - } - if enableArtifactClient { - if artifactClientCondition != "" { - setupLines = append(setupLines, fmt.Sprintf(" INPUT_SAFE_OUTPUT_ARTIFACT_CLIENT: %s\n", artifactClientCondition)) - } else { - setupLines = append(setupLines, " INPUT_SAFE_OUTPUT_ARTIFACT_CLIENT: 'true'\n") - } - } - lines = append(lines, setupLines...) - return lines + return append(lines, buildScriptModeSetupLines(data, destination, traceID, parentSpanID, setupEngineID, hasOTLPOIDC, enableArtifactClient, artifactClientCondition)...) } // Dev/Release mode: use the setup action compilerYamlStepGenerationLog.Printf("Generating setup step: ref=%s, destination=%s, artifactClient=%t, traceID=%q, parentSpanID=%q", setupActionRef, destination, enableArtifactClient, traceID, parentSpanID) + setupLines := buildActionModeSetupLines(data, setupActionRef, destination, traceID, parentSpanID, setupEngineID, hasOTLPOIDC, enableArtifactClient, artifactClientCondition) + lines = append(lines, setupLines...) + return lines +} + +func setupEngineIDForWorkflow(data *WorkflowData) string { + if data == nil { + return "" + } + if data.EngineConfig != nil && data.EngineConfig.ID != "" { + return data.EngineConfig.ID + } + if data.AI != "" { + return data.AI + } + return "" +} + +func buildScriptModeSetupLines(data *WorkflowData, destination string, traceID string, parentSpanID string, setupEngineID string, hasOTLPOIDC bool, enableArtifactClient bool, artifactClientCondition string) []string { + setupLines := []string{ + " - name: Setup Scripts\n", + " id: setup\n", + " run: |\n", + " bash /tmp/gh-aw/actions-source/actions/setup/setup.sh\n", + " env:\n", + fmt.Sprintf(" INPUT_DESTINATION: %s\n", destination), + " INPUT_JOB_NAME: ${{ github.job }}\n", + } + if data != nil { + setupLines = append(setupLines, setupWorkflowInfoEnvLines(data, setupEngineID)...) + } + setupLines = appendSetupTraceAndOTLPLines(setupLines, traceID, parentSpanID, hasOTLPOIDC, true) + return appendArtifactClientInput(setupLines, enableArtifactClient, artifactClientCondition, true) +} + +func buildActionModeSetupLines(data *WorkflowData, setupActionRef string, destination string, traceID string, parentSpanID string, setupEngineID string, hasOTLPOIDC bool, enableArtifactClient bool, artifactClientCondition string) []string { setupLines := []string{ " - name: Setup Scripts\n", " id: setup\n", @@ -244,46 +227,67 @@ func (c *Compiler) generateSetupStepWithArtifactClientCondition(data *WorkflowDa fmt.Sprintf(" destination: %s\n", destination), " job-name: ${{ github.job }}\n", } - if traceID != "" { - setupLines = append(setupLines, fmt.Sprintf(" trace-id: %s\n", traceID)) - } - if parentSpanID != "" { - setupLines = append(setupLines, fmt.Sprintf(" parent-span-id: %s\n", parentSpanID)) - } - if hasOTLPOIDC { - setupLines = append(setupLines, " otlp-oidc-token: ${{ steps.mint-otlp-oidc-token.outputs.token }}\n") - } - if enableArtifactClient { - if artifactClientCondition != "" { - setupLines = append(setupLines, fmt.Sprintf(" safe-output-artifact-client: %s\n", artifactClientCondition)) - } else { - setupLines = append(setupLines, " safe-output-artifact-client: 'true'\n") - } + setupLines = appendSetupTraceAndOTLPLines(setupLines, traceID, parentSpanID, hasOTLPOIDC, false) + setupLines = appendArtifactClientInput(setupLines, enableArtifactClient, artifactClientCondition, false) + setupLines = append(setupLines, " env:\n") + setupLines = append(setupLines, setupWorkflowInfoEnvLines(data, setupEngineID)...) + if hasWorkflowCallTrigger(data.On) { + setupLines = append(setupLines, " GH_AW_SETUP_AW_CONTEXT: ${{ inputs.aw_context }}\n") } - setupLines = append(setupLines, - " env:\n", + return setupLines +} + +func setupWorkflowInfoEnvLines(data *WorkflowData, setupEngineID string) []string { + lines := []string{ fmt.Sprintf(" GH_AW_SETUP_WORKFLOW_NAME: %q\n", data.Name), fmt.Sprintf(" GH_AW_CURRENT_WORKFLOW_REF: %s\n", buildSetupWorkflowRefExpr(data)), - ) + } if v := getVersionForSetup(data); v != "" { - setupLines = append(setupLines, fmt.Sprintf(" GH_AW_INFO_VERSION: %q\n", v)) + lines = append(lines, fmt.Sprintf(" GH_AW_INFO_VERSION: %q\n", v)) } if v := getAWFVersionForSetup(data); v != "" { - setupLines = append(setupLines, fmt.Sprintf(" GH_AW_INFO_AWF_VERSION: %q\n", v)) + lines = append(lines, fmt.Sprintf(" GH_AW_INFO_AWF_VERSION: %q\n", v)) } if data.Source != "" { - setupLines = append(setupLines, " GH_AW_INFO_BODY_MODIFIED: \"false\"\n") + lines = append(lines, " GH_AW_INFO_BODY_MODIFIED: \"false\"\n") } if setupEngineID != "" { - setupLines = append(setupLines, fmt.Sprintf(" GH_AW_INFO_ENGINE_ID: %q\n", setupEngineID)) - } - if hasWorkflowCallTrigger(data.On) { - setupLines = append(setupLines, " GH_AW_SETUP_AW_CONTEXT: ${{ inputs.aw_context }}\n") + lines = append(lines, fmt.Sprintf(" GH_AW_INFO_ENGINE_ID: %q\n", setupEngineID)) } - lines = append(lines, setupLines...) return lines } +func appendSetupTraceAndOTLPLines(setupLines []string, traceID string, parentSpanID string, hasOTLPOIDC bool, scriptMode bool) []string { + traceKey, parentKey, otlpKey := "trace-id", "parent-span-id", "otlp-oidc-token" + if scriptMode { + traceKey, parentKey, otlpKey = "INPUT_TRACE_ID", "INPUT_PARENT_SPAN_ID", "INPUT_OTLP_OIDC_TOKEN" + } + if traceID != "" { + setupLines = append(setupLines, fmt.Sprintf(" %s: %s\n", traceKey, traceID)) + } + if parentSpanID != "" { + setupLines = append(setupLines, fmt.Sprintf(" %s: %s\n", parentKey, parentSpanID)) + } + if hasOTLPOIDC { + setupLines = append(setupLines, fmt.Sprintf(" %s: ${{ steps.mint-otlp-oidc-token.outputs.token }}\n", otlpKey)) + } + return setupLines +} + +func appendArtifactClientInput(setupLines []string, enableArtifactClient bool, artifactClientCondition string, scriptMode bool) []string { + if !enableArtifactClient { + return setupLines + } + key := "safe-output-artifact-client" + if scriptMode { + key = "INPUT_SAFE_OUTPUT_ARTIFACT_CLIENT" + } + if artifactClientCondition != "" { + return append(setupLines, fmt.Sprintf(" %s: %s\n", key, artifactClientCondition)) + } + return append(setupLines, fmt.Sprintf(" %s: 'true'\n", key)) +} + // generateSetRuntimePathsStep generates a step that sets RUNNER_TEMP-based env vars // via $GITHUB_OUTPUT. These cannot be set in job-level env: because the runner context // is not available there (only in step-level env: and run: blocks). diff --git a/pkg/workflow/compiler_yaml_step_lifecycle.go b/pkg/workflow/compiler_yaml_step_lifecycle.go index a83ffc65a72..cd7dd5f3026 100644 --- a/pkg/workflow/compiler_yaml_step_lifecycle.go +++ b/pkg/workflow/compiler_yaml_step_lifecycle.go @@ -47,143 +47,163 @@ func writeStepsSection(yaml *strings.Builder, stepsYAML string) { } func (c *Compiler) generateCreateAwInfo(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine) { - // Engine ID (prefer EngineConfig.ID, fallback to AI field for backwards compatibility) + info := c.buildAwInfoContext(data, engine) + + compilerYamlStepLifecycleLog.Printf("Generating aw_info step: engine=%s, modelConfigured=%t, version=%s, firewallEnabled=%t, staged=%s", info.engineID, info.modelConfigured, info.version, info.firewallEnabled, info.stagedValue) + + yaml.WriteString(" - name: Generate agentic run info\n") + yaml.WriteString(" id: generate_aw_info\n") + yaml.WriteString(" env:\n") + c.writeAwInfoEnv(yaml, data, engine, info) + fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) + yaml.WriteString(" with:\n") + yaml.WriteString(" script: |\n") + yaml.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") + yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") + yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs');\n") + yaml.WriteString(" await main(core, context);\n") +} + +type awInfoContext struct { + engineID string + modelConfigured bool + modelEnvVar string + agentVersion string + version string + stagedValue string + domainsJSON string + firewallEnabled bool + firewallVersion string + mcpGatewayVersion string + firewallType string +} + +func (c *Compiler) buildAwInfoContext(data *WorkflowData, engine CodingAgentEngine) awInfoContext { engineID := engine.GetID() if data.EngineConfig != nil && data.EngineConfig.ID != "" { engineID = data.EngineConfig.ID } else if data.AI != "" { engineID = data.AI } - - // Model - explicit config or runtime env var via vars context - modelConfigured := data.EngineConfig != nil && data.EngineConfig.Model != "" - var modelEnvVar string - if !modelConfigured { - switch engineID { - case "copilot": - modelEnvVar = constants.EnvVarModelAgentCopilot - case "claude": - modelEnvVar = constants.EnvVarModelAgentClaude - case "codex": - modelEnvVar = constants.EnvVarModelAgentCodex - case "opencode": - modelEnvVar = constants.EnvVarModelAgentOpenCode - case "custom": - modelEnvVar = constants.EnvVarModelAgentCustom - default: - modelEnvVar = constants.EnvVarModelAgentCustom - } - } - - // Agent version - use the actual installation version (includes defaults) agentVersion := getInstallationVersion(data, engine) - - // Version: prefer explicit engine config version, fall back to the installation version - // so the run details always show the version being used rather than "(none)". - version := agentVersion + info := awInfoContext{ + engineID: engineID, + modelConfigured: data.EngineConfig != nil && data.EngineConfig.Model != "", + modelEnvVar: modelEnvVarForEngine(engineID), + agentVersion: agentVersion, + version: agentVersion, + stagedValue: "false", + domainsJSON: "[]", + } if data.EngineConfig != nil && data.EngineConfig.Version != "" { - version = data.EngineConfig.Version + info.version = data.EngineConfig.Version } - - // Staged value from safe-outputs configuration - stagedValue := "false" if data.SafeOutputs != nil && data.SafeOutputs.Staged != nil { - stagedValue = data.SafeOutputs.Staged.String() + info.stagedValue = data.SafeOutputs.Staged.String() } + c.populateAwInfoNetworkFields(data, &info) + return info +} - // Network configuration +func modelEnvVarForEngine(engineID string) string { + switch engineID { + case "copilot": + return constants.EnvVarModelAgentCopilot + case "claude": + return constants.EnvVarModelAgentClaude + case "codex": + return constants.EnvVarModelAgentCodex + case "opencode": + return constants.EnvVarModelAgentOpenCode + default: + return constants.EnvVarModelAgentCustom + } +} + +func (c *Compiler) populateAwInfoNetworkFields(data *WorkflowData, info *awInfoContext) { var allowedDomains []string - firewallEnabled := false - firewallVersion := "" if data.NetworkPermissions != nil { allowedDomains = data.NetworkPermissions.Allowed } if firewallConfig := getFirewallConfig(data); firewallConfig != nil { - firewallEnabled = firewallConfig.Enabled - firewallVersion = firewallConfig.Version - if firewallEnabled && firewallVersion == "" { - firewallVersion = string(constants.DefaultFirewallVersion) + info.firewallEnabled = firewallConfig.Enabled + info.firewallVersion = firewallConfig.Version + if info.firewallEnabled && info.firewallVersion == "" { + info.firewallVersion = string(constants.DefaultFirewallVersion) } } - - // Allowed domains as JSON array string - domainsJSON := "[]" if len(allowedDomains) > 0 { b, _ := json.Marshal(allowedDomains) //nolint:jsonmarshalignoredeerror // marshaling a string slice cannot fail - domainsJSON = string(b) + info.domainsJSON = string(b) } - - // MCP Gateway version - mcpGatewayVersion := "" if data.SandboxConfig != nil && data.SandboxConfig.MCP != nil && data.SandboxConfig.MCP.Version != "" { - mcpGatewayVersion = data.SandboxConfig.MCP.Version + info.mcpGatewayVersion = data.SandboxConfig.MCP.Version } - - // Firewall type - firewallType := "" if isFirewallEnabled(data) { - firewallType = "squid" + info.firewallType = "squid" } +} - compilerYamlStepLifecycleLog.Printf("Generating aw_info step: engine=%s, modelConfigured=%t, version=%s, firewallEnabled=%t, staged=%s", engineID, modelConfigured, version, firewallEnabled, stagedValue) +func (c *Compiler) writeAwInfoEnv(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine, info awInfoContext) { + c.writeAwInfoCoreEnv(yaml, data, engine, info) + c.writeAwInfoSourceEnv(yaml, data) + c.writeAwInfoLockdownEnv(yaml, data) + writeAwInfoJSONEnv(yaml, data) +} - yaml.WriteString(" - name: Generate agentic run info\n") - yaml.WriteString(" id: generate_aw_info\n") - yaml.WriteString(" env:\n") - fmt.Fprintf(yaml, " GH_AW_INFO_ENGINE_ID: \"%s\"\n", engineID) +func (c *Compiler) writeAwInfoCoreEnv(yaml *strings.Builder, data *WorkflowData, engine CodingAgentEngine, info awInfoContext) { + fmt.Fprintf(yaml, " GH_AW_INFO_ENGINE_ID: \"%s\"\n", info.engineID) fmt.Fprintf(yaml, " GH_AW_INFO_ENGINE_NAME: \"%s\"\n", engine.GetDisplayName()) - if modelConfigured { - fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: \"%s\"\n", data.EngineConfig.Model) - } else { - // Use the engine's default model as fallback when neither explicit model nor - // model variable is configured, so the run details show "agent" rather than "(none)". - defaultModel := getDefaultAgentModel(engineID) - defaultModelOverrideVar := getDefaultModelOverrideVar(engineID) - if defaultModel != "" && defaultModelOverrideVar != "" { - fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: %s\n", compilerenv.BuildModelOverrideExpression(modelEnvVar, defaultModelOverrideVar, defaultModel)) - } else if defaultModel != "" { - fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: ${{ vars.%s || '%s' }}\n", modelEnvVar, defaultModel) - } else if defaultModelOverrideVar != "" { - fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: %s\n", compilerenv.BuildModelOverrideExpressionEmptyFallback(modelEnvVar, defaultModelOverrideVar)) - } else { - fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: ${{ vars.%s || '' }}\n", modelEnvVar) - } - } - fmt.Fprintf(yaml, " GH_AW_INFO_VERSION: \"%s\"\n", version) - fmt.Fprintf(yaml, " GH_AW_INFO_AGENT_VERSION: \"%s\"\n", agentVersion) - // CLI version only for released builds + c.writeAwInfoModelEnv(yaml, data, info) + fmt.Fprintf(yaml, " GH_AW_INFO_VERSION: \"%s\"\n", info.version) + fmt.Fprintf(yaml, " GH_AW_INFO_AGENT_VERSION: \"%s\"\n", info.agentVersion) if IsReleasedVersion(c.version) { fmt.Fprintf(yaml, " GH_AW_INFO_CLI_VERSION: \"%s\"\n", c.version) } fmt.Fprintf(yaml, " GH_AW_INFO_WORKFLOW_NAME: \"%s\"\n", data.Name) fmt.Fprintf(yaml, " GH_AW_INFO_EXPERIMENTAL: \"%t\"\n", engine.IsExperimental()) fmt.Fprintf(yaml, " GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: \"%t\"\n", engine.GetCapabilities().ToolsAllowlist) - fmt.Fprintf(yaml, " GH_AW_INFO_STAGED: \"%s\"\n", stagedValue) - fmt.Fprintf(yaml, " GH_AW_INFO_ALLOWED_DOMAINS: '%s'\n", domainsJSON) - fmt.Fprintf(yaml, " GH_AW_INFO_FIREWALL_ENABLED: \"%t\"\n", firewallEnabled) - fmt.Fprintf(yaml, " GH_AW_INFO_AWF_VERSION: \"%s\"\n", firewallVersion) - fmt.Fprintf(yaml, " GH_AW_INFO_AWMG_VERSION: \"%s\"\n", mcpGatewayVersion) - fmt.Fprintf(yaml, " GH_AW_INFO_FIREWALL_TYPE: \"%s\"\n", firewallType) + fmt.Fprintf(yaml, " GH_AW_INFO_STAGED: \"%s\"\n", info.stagedValue) + fmt.Fprintf(yaml, " GH_AW_INFO_ALLOWED_DOMAINS: '%s'\n", info.domainsJSON) + fmt.Fprintf(yaml, " GH_AW_INFO_FIREWALL_ENABLED: \"%t\"\n", info.firewallEnabled) + fmt.Fprintf(yaml, " GH_AW_INFO_AWF_VERSION: \"%s\"\n", info.firewallVersion) + fmt.Fprintf(yaml, " GH_AW_INFO_AWMG_VERSION: \"%s\"\n", info.mcpGatewayVersion) + fmt.Fprintf(yaml, " GH_AW_INFO_FIREWALL_TYPE: \"%s\"\n", info.firewallType) +} + +func (c *Compiler) writeAwInfoModelEnv(yaml *strings.Builder, data *WorkflowData, info awInfoContext) { + if info.modelConfigured { + fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: \"%s\"\n", data.EngineConfig.Model) + return + } + defaultModel := getDefaultAgentModel(info.engineID) + defaultModelOverrideVar := getDefaultModelOverrideVar(info.engineID) + if defaultModel != "" && defaultModelOverrideVar != "" { + fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: %s\n", compilerenv.BuildModelOverrideExpression(info.modelEnvVar, defaultModelOverrideVar, defaultModel)) + } else if defaultModel != "" { + fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: ${{ vars.%s || '%s' }}\n", info.modelEnvVar, defaultModel) + } else if defaultModelOverrideVar != "" { + fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: %s\n", compilerenv.BuildModelOverrideExpressionEmptyFallback(info.modelEnvVar, defaultModelOverrideVar)) + } else { + fmt.Fprintf(yaml, " GH_AW_INFO_MODEL: ${{ vars.%s || '' }}\n", info.modelEnvVar) + } +} + +func (c *Compiler) writeAwInfoSourceEnv(yaml *strings.Builder, data *WorkflowData) { if data.Source != "" { fmt.Fprintf(yaml, " GH_AW_INFO_FRONTMATTER_SOURCE: %q\n", data.Source) - // Body-modified defaults to false at compile time; update flows may override this - // signal when source/body drift is detected before execution. yaml.WriteString(" GH_AW_INFO_BODY_MODIFIED: \"false\"\n") } if data.FrontmatterEmoji != "" { fmt.Fprintf(yaml, " GH_AW_INFO_FRONTMATTER_EMOJI: %q\n", data.FrontmatterEmoji) } - // Always include strict mode flag for lockdown validation. - // validateLockdownRequirements uses this to enforce strict: true for public repositories. - // Use effectiveStrictMode to infer strictness from the source (frontmatter), not just the CLI flag. fmt.Fprintf(yaml, " GH_AW_COMPILED_STRICT: \"%t\"\n", c.effectiveStrictMode(data.RawFrontmatter)) - // When a workflow_call trigger is present, pass the target_repo resolved by the - // resolve-host-repo step so it can be stored in aw_info.json for observability. if hasWorkflowCallTrigger(data.On) && !data.InlinedImports { yaml.WriteString(" GH_AW_INFO_TARGET_REPO: ${{ steps.resolve-host-repo.outputs.target_repo }}\n") } - // Include lockdown validation env vars when lockdown is explicitly enabled. - // validateLockdownRequirements is called from generate_aw_info.cjs and uses these vars. +} + +func (c *Compiler) writeAwInfoLockdownEnv(yaml *strings.Builder, data *WorkflowData) { githubTool, hasGitHub := data.Tools["github"] if hasGitHub && githubTool != false { toolConfig, _ := githubTool.(map[string]any) @@ -196,21 +216,14 @@ func (c *Compiler) generateCreateAwInfo(yaml *strings.Builder, data *WorkflowDat } } } - // Embed the `models` overlay from frontmatter so the activation job can merge it with - // the built-in models.json and write the combined catalog to /tmp/gh-aw/models.json. +} + +func writeAwInfoJSONEnv(yaml *strings.Builder, data *WorkflowData) { if len(data.ModelCosts) > 0 { - if modelCostsJSON, err := json.Marshal(data.ModelCosts); err == nil { - // Escape single quotes for YAML single-quoted scalar safety - escapedModelCostsJSON := strings.ReplaceAll(string(modelCostsJSON), "'", "''") - fmt.Fprintf(yaml, " GH_AW_INFO_MODEL_COSTS: '%s'\n", escapedModelCostsJSON) - } + writeEscapedJSONEnv(yaml, "GH_AW_INFO_MODEL_COSTS", data.ModelCosts) } if len(data.Features) > 0 { - if featuresJSON, err := json.Marshal(data.Features); err == nil { - // Escape single quotes for YAML single-quoted scalar safety - escapedFeaturesJSON := strings.ReplaceAll(string(featuresJSON), "'", "''") - fmt.Fprintf(yaml, " GH_AW_INFO_FEATURES: '%s'\n", escapedFeaturesJSON) - } + writeEscapedJSONEnv(yaml, "GH_AW_INFO_FEATURES", data.Features) } if len(data.Skills) > 0 { if skillsJSON, err := json.Marshal(data.Skills); err == nil { @@ -220,18 +233,33 @@ func (c *Compiler) generateCreateAwInfo(yaml *strings.Builder, data *WorkflowDat compilerYamlStepLifecycleLog.Printf("Failed to marshal skills for GH_AW_INFO_SKILLS, engine will not receive skill list: %v", err) } } - fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) +} + +func writeEscapedJSONEnv(yaml *strings.Builder, name string, value any) { + if jsonBytes, err := json.Marshal(value); err == nil { + escapedJSON := strings.ReplaceAll(string(jsonBytes), "'", "''") + fmt.Fprintf(yaml, " %s: '%s'\n", name, escapedJSON) + } +} + +func (c *Compiler) generateOutputCollectionStep(yaml *strings.Builder, data *WorkflowData) error { + writeSafeOutputCopyStep(yaml) + writeOutputCollectionStepHeader(yaml, data) + if err := c.writeOutputCollectionEnv(yaml, data); err != nil { + return err + } + yaml.WriteString(" with:\n") yaml.WriteString(" script: |\n") yaml.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") - yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs');\n") - yaml.WriteString(" await main(core, context);\n") + yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs');\n") + yaml.WriteString(" await main();\n") + + return nil } -func (c *Compiler) generateOutputCollectionStep(yaml *strings.Builder, data *WorkflowData) error { - // Copy the raw safe-output NDJSON to a /tmp/gh-aw/ path so it can be included in the - // unified agent artifact together with all other /tmp/gh-aw/ outputs. +func writeSafeOutputCopyStep(yaml *strings.Builder) { yaml.WriteString(" - name: Copy Safe Outputs\n") yaml.WriteString(" if: always()\n") yaml.WriteString(" env:\n") @@ -239,35 +267,21 @@ func (c *Compiler) generateOutputCollectionStep(yaml *strings.Builder, data *Wor yaml.WriteString(" run: |\n") fmt.Fprintf(yaml, " mkdir -p /tmp/gh-aw\n") fmt.Fprintf(yaml, " cp \"$GH_AW_SAFE_OUTPUTS\" /tmp/gh-aw/%s 2>/dev/null || true\n", constants.SafeOutputsFilename) +} +func writeOutputCollectionStepHeader(yaml *strings.Builder, data *WorkflowData) { yaml.WriteString(" - name: Ingest agent output\n") yaml.WriteString(" id: collect_output\n") yaml.WriteString(" if: always()\n") fmt.Fprintf(yaml, " uses: %s\n", getCachedActionPin("actions/github-script", data)) - - // Add environment variables for JSONL validation yaml.WriteString(" env:\n") yaml.WriteString(" GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}\n") +} - // Config is written to file, not passed as env var - - // Add allowed domains configuration for sanitization - // Use manually configured domains if available, otherwise compute from network configuration - var domainsStr string - if data.SafeOutputs != nil && len(data.SafeOutputs.AllowedDomains) > 0 { - // allowed-domains: additional domains unioned with engine/network base set; supports ecosystem identifiers - expanded, err := c.computeExpandedAllowedDomainsForSanitization(data) - if err != nil { - return err - } - domainsStr = expanded - } else { - // Fall back to computing from network configuration (same as firewall) - computed, err := c.computeAllowedDomainsForSanitization(data) - if err != nil { - return err - } - domainsStr = computed +func (c *Compiler) writeOutputCollectionEnv(yaml *strings.Builder, data *WorkflowData) error { + domainsStr, err := c.outputCollectionAllowedDomains(data) + if err != nil { + return err } if domainsStr != "" { fmt.Fprintf(yaml, " GH_AW_ALLOWED_DOMAINS: %q\n", domainsStr) @@ -275,19 +289,24 @@ func (c *Compiler) generateOutputCollectionStep(yaml *strings.Builder, data *Wor if data.SafeOutputs != nil && data.SafeOutputs.URLs != "" { fmt.Fprintf(yaml, " GH_AW_SAFE_OUTPUTS_URLS: %q\n", data.SafeOutputs.URLs) } - - // Add allowed GitHub references configuration for reference escaping if data.SafeOutputs != nil && data.SafeOutputs.AllowGitHubReferences != nil { refsStr := strings.Join(data.SafeOutputs.AllowGitHubReferences, ",") fmt.Fprintf(yaml, " GH_AW_ALLOWED_GITHUB_REFS: %q\n", refsStr) } - - // Add GitHub server URL and API URL for dynamic domain extraction - // This allows the sanitization code to permit GitHub domains that vary by deployment yaml.WriteString(" GITHUB_SERVER_URL: ${{ github.server_url }}\n") yaml.WriteString(" GITHUB_API_URL: ${{ github.api_url }}\n") + writeOutputCollectionCommandEnv(yaml, data) + return nil +} + +func (c *Compiler) outputCollectionAllowedDomains(data *WorkflowData) (string, error) { + if data.SafeOutputs != nil && len(data.SafeOutputs.AllowedDomains) > 0 { + return c.computeExpandedAllowedDomainsForSanitization(data) + } + return c.computeAllowedDomainsForSanitization(data) +} - // Add command names for command trigger prevention in safe outputs +func writeOutputCollectionCommandEnv(yaml *strings.Builder, data *WorkflowData) { if len(data.Command) > 0 { if commandsJSON, err := json.Marshal(data.Command); err == nil { fmt.Fprintf(yaml, " GH_AW_COMMANDS: %q\n", string(commandsJSON)) @@ -301,15 +320,4 @@ func (c *Compiler) generateOutputCollectionStep(yaml *strings.Builder, data *Wor fmt.Fprintf(yaml, " GH_AW_LABEL_COMMANDS: %q\n", string(labelCommandsJSON)) } } - - yaml.WriteString(" with:\n") - yaml.WriteString(" script: |\n") - - // Load script from external file using require() - yaml.WriteString(" const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs');\n") - yaml.WriteString(" setupGlobals(core, github, context, exec, io, getOctokit);\n") - yaml.WriteString(" const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs');\n") - yaml.WriteString(" await main();\n") - - return nil } diff --git a/pkg/workflow/concurrency.go b/pkg/workflow/concurrency.go index 525fc7a82f3..e1d368d887a 100644 --- a/pkg/workflow/concurrency.go +++ b/pkg/workflow/concurrency.go @@ -238,112 +238,67 @@ func workflowCallBaseKey(workflowData *WorkflowData) string { // buildConcurrencyGroupKeys builds an array of keys for the concurrency group func buildConcurrencyGroupKeys(workflowData *WorkflowData, isCommandTrigger bool) []string { - // For workflow_call workers, github.workflow resolves to the *calling* workflow's - // name at runtime, not the callee's. Both the gateway and the worker would therefore - // share the same concurrency group, causing a deadlock. Hard-bake the compile-time - // workflow ID (filename without extension) when available and append github.run_id - // so parallel caller fan-out runs don't contend for a single static queue slot. - workflowKey := "${{ github.workflow }}" - if hasWorkflowCallTrigger(workflowData.On) { - workflowKey = workflowCallBaseKey(workflowData) + "-${{ github.run_id }}" - } - keys := []string{"gh-aw", workflowKey} - - // Whether this workflow exposes inputs.item_number via workflow_dispatch (label trigger shorthand). - // When true, include it in the concurrency key so that manual dispatches for different items - // use distinct groups and don't cancel each other. + keys := []string{"gh-aw", concurrencyWorkflowKey(workflowData)} hasItemNumber := workflowData.HasDispatchItemNumber - - // Detect whether the workflow is at risk of bot-self-cancellation. - // When safe-outputs uses a GitHub App token AND the workflow has issue_comment triggers, - // App-authored comments can re-trigger the workflow and collide with the primary run's - // concurrency group. When this risk is present we use botIsolatedConcurrencyKey so that - // bot-triggered runs always resolve to a unique github.run_id key instead. botRisk := hasBotSelfCancelRisk(workflowData) if botRisk { concurrencyLog.Print("Bot self-cancel risk detected: applying bot-actor isolation to concurrency group key") } + if entityKey := buildEntityConcurrencyKey(workflowData, isCommandTrigger, hasItemNumber, botRisk); entityKey != "" { + keys = append(keys, entityKey) + } + if hasItemNumber { + keys = append(keys, "${{ github.event.label.name || github.run_id }}") + } + return keys +} - // entityKey selects the correct concurrency key builder based on bot risk. - // It captures both botRisk and hasItemNumber from the outer scope, so call - // sites only need to supply the entity-specific parts. +func concurrencyWorkflowKey(workflowData *WorkflowData) string { + if hasWorkflowCallTrigger(workflowData.On) { + return workflowCallBaseKey(workflowData) + "-${{ github.run_id }}" + } + return "${{ github.workflow }}" +} + +func buildEntityConcurrencyKey(workflowData *WorkflowData, isCommandTrigger, hasItemNumber, botRisk bool) string { entityKey := func(primaryParts []string, tailParts []string) string { if botRisk { return botIsolatedConcurrencyKey(primaryParts, tailParts, hasItemNumber) } return entityConcurrencyKey(primaryParts, tailParts, hasItemNumber) } - - if isCommandTrigger || isSlashCommandWorkflow(workflowData.On) { - // For command/slash_command workflows: use issue/PR number; fall back to run_id when - // neither is available (e.g. manual workflow_dispatch of the outer workflow). - // When bot risk is detected, prepend the bot-actor isolation check. - concurrencyLog.Print("Building concurrency key for command/slash_command workflow") - if botRisk { - keys = append(keys, "${{ contains(github.actor, '[bot]') && github.run_id || github.event.issue.number || github.event.pull_request.number || github.run_id }}") - } else { - keys = append(keys, "${{ github.event.issue.number || github.event.pull_request.number || github.run_id }}") - } - } else if isPullRequestWorkflow(workflowData.On) && isIssueWorkflow(workflowData.On) { - // Mixed workflows with both issue and PR triggers + switch { + case isCommandTrigger || isSlashCommandWorkflow(workflowData.On): + return commandConcurrencyKey(botRisk) + case isPullRequestWorkflow(workflowData.On) && isIssueWorkflow(workflowData.On): concurrencyLog.Print("Building concurrency key for mixed PR+issue workflow") - keys = append(keys, entityKey( - []string{"github.event.issue.number", "github.event.pull_request.number"}, - []string{"github.run_id"}, - )) - } else if isPullRequestWorkflow(workflowData.On) && isDiscussionWorkflow(workflowData.On) { - // Mixed workflows with PR and discussion triggers - keys = append(keys, entityConcurrencyKey( - []string{"github.event.pull_request.number", "github.event.discussion.number"}, - []string{"github.run_id"}, - hasItemNumber, - )) - } else if isIssueWorkflow(workflowData.On) && isDiscussionWorkflow(workflowData.On) { - // Mixed workflows with issue and discussion triggers - keys = append(keys, entityKey( - []string{"github.event.issue.number", "github.event.discussion.number"}, - []string{"github.run_id"}, - )) - } else if isPullRequestWorkflow(workflowData.On) { - // PR workflows: use PR number, fall back to ref then run_id + return entityKey([]string{"github.event.issue.number", "github.event.pull_request.number"}, []string{"github.run_id"}) + case isPullRequestWorkflow(workflowData.On) && isDiscussionWorkflow(workflowData.On): + return entityConcurrencyKey([]string{"github.event.pull_request.number", "github.event.discussion.number"}, []string{"github.run_id"}, hasItemNumber) + case isIssueWorkflow(workflowData.On) && isDiscussionWorkflow(workflowData.On): + return entityKey([]string{"github.event.issue.number", "github.event.discussion.number"}, []string{"github.run_id"}) + case isPullRequestWorkflow(workflowData.On): concurrencyLog.Print("Building concurrency key for PR workflow") - keys = append(keys, entityConcurrencyKey( - []string{"github.event.pull_request.number"}, - []string{"github.ref", "github.run_id"}, - hasItemNumber, - )) - } else if isIssueWorkflow(workflowData.On) { - // Issue workflows: run_id is the fallback when no issue context is available - // (e.g. when a mixed-trigger workflow is started via workflow_dispatch). + return entityConcurrencyKey([]string{"github.event.pull_request.number"}, []string{"github.ref", "github.run_id"}, hasItemNumber) + case isIssueWorkflow(workflowData.On): concurrencyLog.Print("Building concurrency key for issue workflow") - keys = append(keys, entityKey( - []string{"github.event.issue.number"}, - []string{"github.run_id"}, - )) - } else if isDiscussionWorkflow(workflowData.On) { - // Discussion workflows: run_id is the fallback when no discussion context is available. - keys = append(keys, entityConcurrencyKey( - []string{"github.event.discussion.number"}, - []string{"github.run_id"}, - hasItemNumber, - )) - } else if isPushWorkflow(workflowData.On) { - // Push workflows: use ref to differentiate between branches + return entityKey([]string{"github.event.issue.number"}, []string{"github.run_id"}) + case isDiscussionWorkflow(workflowData.On): + return entityConcurrencyKey([]string{"github.event.discussion.number"}, []string{"github.run_id"}, hasItemNumber) + case isPushWorkflow(workflowData.On): concurrencyLog.Print("Building concurrency key for push workflow") - keys = append(keys, "${{ github.ref || github.run_id }}") + return "${{ github.ref || github.run_id }}" + default: + return "" } +} - // For label-triggered workflows (label trigger shorthand or label_command), include - // github.event.label.name as an additional segment so that runs triggered by different - // labels do not share a concurrency group. Without this, adding multiple labels to the - // same PR/issue simultaneously (which fires one labeled event per label) would cause all - // matching workflow runs to share a group, and with cancel-in-progress enabled the last - // surviving run could be for a different label than the workflow expects. - if hasItemNumber { - keys = append(keys, "${{ github.event.label.name || github.run_id }}") +func commandConcurrencyKey(botRisk bool) string { + concurrencyLog.Print("Building concurrency key for command/slash_command workflow") + if botRisk { + return "${{ contains(github.actor, '[bot]') && github.run_id || github.event.issue.number || github.event.pull_request.number || github.run_id }}" } - - return keys + return "${{ github.event.issue.number || github.event.pull_request.number || github.run_id }}" } // shouldEnableCancelInProgress determines if cancel-in-progress should be enabled diff --git a/pkg/workflow/copilot_engine_tools.go b/pkg/workflow/copilot_engine_tools.go index 44a8cf78764..7cf0e311059 100644 --- a/pkg/workflow/copilot_engine_tools.go +++ b/pkg/workflow/copilot_engine_tools.go @@ -66,209 +66,196 @@ func (e *CopilotEngine) computeCopilotToolArguments(tools map[string]any, safeOu var args []string hasRestrictedBashAllowlist := false - // Check if bash has wildcard - if so, use --allow-all-tools instead - if bashConfig, hasBash := tools["bash"]; hasBash { - if bashCommands, ok := bashConfig.([]any); ok { - // Check for :* or * wildcard - if present, allow all tools - for _, cmd := range bashCommands { - if cmdStr, ok := cmd.(string); ok { - if cmdStr == ":*" || cmdStr == "*" { - // Use --allow-all-tools flag instead of individual tool permissions - copilotEngineToolsLog.Print("Bash wildcard detected, using --allow-all-tools") - return []string{"--allow-all-tools"} - } - } - } - } + if copilotHasBashWildcard(tools) { + copilotEngineToolsLog.Print("Bash wildcard detected, using --allow-all-tools") + return []string{"--allow-all-tools"} } - // Handle bash/shell tools (when no wildcard) - if bashConfig, hasBash := tools["bash"]; hasBash { - if bashCommands, ok := bashConfig.([]any); ok { - hasRestrictedBashAllowlist = true - // Add specific shell commands - for _, cmd := range bashCommands { - if cmdStr, ok := cmd.(string); ok { - // Normalize trailing " *" wildcard (e.g. "jq *" → "jq") so that - // all engines emit the canonical prefix form (shell(jq)) regardless - // of whether the command was written with or without the wildcard. - cmdStr, _ = normalizeBashCommand(cmdStr) - // For stem commands (like dotnet, npm, cargo), Copilot CLI uses - // subcommand matching. When the user specifies just the base command - // (e.g., "dotnet"), append :* so "dotnet build", "dotnet test", etc. - // are all permitted. Skip if the command already has a colon (explicit - // matching) or a space (user already specified the subcommand). - if !strings.Contains(cmdStr, ":") && !strings.Contains(cmdStr, " ") && constants.CopilotStemCommands[cmdStr] { - args = append(args, "--allow-tool", fmt.Sprintf("shell(%s:*)", cmdStr)) - } else { - sanitized, wasSanitized := sanitizeCopilotShellCommand(cmdStr) - if wasSanitized { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage( - fmt.Sprintf("bash tool %q contains single quotes that crash Copilot CLI; "+ - "truncated to safe prefix %q for shell() prefix-matching. "+ - "Use %q in your workflow to silence this warning.", - cmdStr, sanitized, sanitized))) - } - args = append(args, "--allow-tool", fmt.Sprintf("shell(%s)", sanitized)) - } - } - } - } else { - // Bash with no specific commands or null value - allow all shell - args = append(args, "--allow-tool", "shell") - } - } + bashArgs, restrictedBash := copilotBashToolArgs(tools) + args = append(args, bashArgs...) + hasRestrictedBashAllowlist = restrictedBash // When MCP tools are mounted as CLI commands and bash uses a restricted allowlist, // ensure mounted MCP CLI commands are executable via shell(:*). // This avoids Copilot CLI permission blocks for mounted commands such as safeoutputs. if hasRestrictedBashAllowlist { - effectiveWorkflowData := buildCLIWorkflowDataForMounts(workflowData, tools, safeOutputs, mcpScripts) + args = append(args, e.copilotRestrictedBashMountArgs(tools, safeOutputs, mcpScripts, workflowData)...) + } - for _, serverName := range getMountedCLIServerNamesIfBashRestricted(effectiveWorkflowData, tools, safeOutputs, mcpScripts) { - args = append(args, "--allow-tool", fmt.Sprintf("shell(%s:*)", serverName)) - } - // When playwright is configured in CLI mode, playwright-cli must be executable. - // Automatically add shell(playwright-cli:*) to the restricted bash allowlist. - if workflowData != nil && isPlaywrightCLIMode(workflowData.Tools) { - args = append(args, "--allow-tool", "shell(playwright-cli:*)") + args = append(args, copilotBuiltinToolArgs(tools, safeOutputs, mcpScripts)...) + args = append(args, copilotMCPServerToolArgs(tools)...) + + args = sortAndDeduplicateCopilotToolArgs(args) + + copilotEngineToolsLog.Printf("Computed %d tool arguments", len(args)/2) + return args +} + +func copilotHasBashWildcard(tools map[string]any) bool { + bashCommands, ok := tools["bash"].([]any) + if !ok { + return false + } + for _, cmd := range bashCommands { + if cmdStr, ok := cmd.(string); ok && (cmdStr == ":*" || cmdStr == "*") { + return true } - // When GitHub CLI mode is enabled (tools.github.mode: gh-proxy), GitHub access - // goes through the gh CLI, so allow shell(gh:*). - if isGitHubCLIModeEnabled(effectiveWorkflowData) { - args = append(args, "--allow-tool", "shell(gh:*)") + } + return false +} + +func copilotBashToolArgs(tools map[string]any) ([]string, bool) { + bashConfig, hasBash := tools["bash"] + if !hasBash { + return nil, false + } + bashCommands, ok := bashConfig.([]any) + if !ok { + return []string{"--allow-tool", "shell"}, false + } + var args []string + for _, cmd := range bashCommands { + if cmdStr, ok := cmd.(string); ok { + args = append(args, "--allow-tool", copilotShellToolValue(cmdStr)) } } + return args, true +} - // Handle edit tools requirement for file write access - // Note: safe-outputs do not need write permission as they use MCP +func copilotShellToolValue(cmdStr string) string { + cmdStr, _ = normalizeBashCommand(cmdStr) + if !strings.Contains(cmdStr, ":") && !strings.Contains(cmdStr, " ") && constants.CopilotStemCommands[cmdStr] { + return fmt.Sprintf("shell(%s:*)", cmdStr) + } + sanitized, wasSanitized := sanitizeCopilotShellCommand(cmdStr) + if wasSanitized { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage( + fmt.Sprintf("bash tool %q contains single quotes that crash Copilot CLI; "+ + "truncated to safe prefix %q for shell() prefix-matching. "+ + "Use %q in your workflow to silence this warning.", + cmdStr, sanitized, sanitized))) + } + return fmt.Sprintf("shell(%s)", sanitized) +} + +func (e *CopilotEngine) copilotRestrictedBashMountArgs(tools map[string]any, safeOutputs *SafeOutputsConfig, mcpScripts *MCPScriptsConfig, workflowData *WorkflowData) []string { + effectiveWorkflowData := buildCLIWorkflowDataForMounts(workflowData, tools, safeOutputs, mcpScripts) + var args []string + for _, serverName := range getMountedCLIServerNamesIfBashRestricted(effectiveWorkflowData, tools, safeOutputs, mcpScripts) { + args = append(args, "--allow-tool", fmt.Sprintf("shell(%s:*)", serverName)) + } + if workflowData != nil && isPlaywrightCLIMode(workflowData.Tools) { + args = append(args, "--allow-tool", "shell(playwright-cli:*)") + } + if isGitHubCLIModeEnabled(effectiveWorkflowData) { + args = append(args, "--allow-tool", "shell(gh:*)") + } + return args +} + +func copilotBuiltinToolArgs(tools map[string]any, safeOutputs *SafeOutputsConfig, mcpScripts *MCPScriptsConfig) []string { + var args []string if _, hasEdit := tools["edit"]; hasEdit { copilotEngineToolsLog.Print("Edit tool enabled, adding write permission") args = append(args, "--allow-tool", "write") } - - // Handle safe_outputs MCP server - allow all tools if safe outputs are enabled - // This includes both safeOutputs config and safeOutputs.Jobs if HasSafeOutputsEnabled(safeOutputs) { copilotEngineToolsLog.Print("Safe-outputs enabled, adding MCP server permission") args = append(args, "--allow-tool", constants.SafeOutputsMCPServerID.String()) } - - // Handle mcp_scripts MCP server - allow the server if mcp-scripts are configured and feature flag is enabled if IsMCPScriptsEnabled(mcpScripts) { args = append(args, "--allow-tool", constants.MCPScriptsMCPServerID.String()) } - - // Handle web-fetch builtin tool (Copilot CLI uses web_fetch with underscore) if _, hasWebFetch := tools["web-fetch"]; hasWebFetch { copilotEngineToolsLog.Print("Web-fetch tool enabled, adding web_fetch permission") - // web-fetch -> web_fetch args = append(args, "--allow-tool", "web_fetch") } + return args +} - // Built-in tool names that should be skipped when processing MCP servers - // Note: GitHub is NOT included here because it needs MCP configuration in CLI mode - // Note: web-fetch is NOT included here because it needs explicit --allow-tool argument - builtInTools := map[string]struct { - }{ - "bash": {}, - "edit": {}, - "web-search": {}, - "playwright": {}, - } - - // Handle MCP server tools +func copilotMCPServerToolArgs(tools map[string]any) []string { + builtInTools := map[string]struct{}{"bash": {}, "edit": {}, "web-search": {}, "playwright": {}} + var args []string for toolName, toolConfig := range tools { - // Skip built-in tools we've already handled if setutil.Contains(builtInTools, toolName) { continue } - - // GitHub is a special case - it's an MCP server but doesn't have explicit MCP config in the workflow - // It gets MCP configuration through the parser's processBuiltinMCPTool if toolName == "github" { - if toolConfigMap, ok := toolConfig.(map[string]any); ok { - if allowed, hasAllowed := toolConfigMap["allowed"]; hasAllowed { - if allowedList, ok := allowed.([]any); ok { - // Process allowed list in a single pass - hasWildcard := false - for _, allowedTool := range allowedList { - if toolStr, ok := allowedTool.(string); ok { - if toolStr == "*" { - // Wildcard means allow entire GitHub MCP server - hasWildcard = true - } else { - // Add individual tool permission - args = append(args, "--allow-tool", fmt.Sprintf("github(%s)", toolStr)) - } - } - } - - // Add server-level permission only if wildcard was present - if hasWildcard { - args = append(args, "--allow-tool", "github") - } - } - } else { - // No allowed field specified - allow entire GitHub MCP server - args = append(args, "--allow-tool", "github") - } - } else { - // GitHub tool exists but is not a map (e.g., github: null) - allow entire server - args = append(args, "--allow-tool", "github") - } + args = append(args, copilotGitHubToolArgs(toolConfig)...) continue } + args = append(args, copilotCustomMCPToolArgs(toolName, toolConfig)...) + } + return args +} - // Check if this is an MCP server configuration - if toolConfigMap, ok := toolConfig.(map[string]any); ok { - if hasMcp, _ := hasMCPConfig(toolConfigMap); hasMcp { - copilotEngineToolsLog.Printf("Adding custom MCP server permission: %s", toolName) - // Allow the entire MCP server - args = append(args, "--allow-tool", toolName) - - // If it has specific allowed tools, add them individually - if allowed, hasAllowed := toolConfigMap["allowed"]; hasAllowed { - if allowedList, ok := allowed.([]any); ok { - for _, allowedTool := range allowedList { - if toolStr, ok := allowedTool.(string); ok { - args = append(args, "--allow-tool", fmt.Sprintf("%s(%s)", toolName, toolStr)) - } - } - } - } +func copilotGitHubToolArgs(toolConfig any) []string { + toolConfigMap, ok := toolConfig.(map[string]any) + if !ok { + return []string{"--allow-tool", "github"} + } + allowedList, ok := toolConfigMap["allowed"].([]any) + if !ok { + return []string{"--allow-tool", "github"} + } + var args []string + hasWildcard := false + for _, allowedTool := range allowedList { + if toolStr, ok := allowedTool.(string); ok { + if toolStr == "*" { + hasWildcard = true + } else { + args = append(args, "--allow-tool", fmt.Sprintf("github(%s)", toolStr)) } } } + if hasWildcard { + args = append(args, "--allow-tool", "github") + } + return args +} - // Sort and deduplicate values, then rebuild args. - // Deduplication is needed because sanitizeCopilotShellCommand can truncate - // multiple different commands to the same safe prefix (e.g. several jq filters - // all become "jq"), producing duplicate --allow-tool shell(jq) entries. - if len(args) > 0 { - var values []string - for i := 1; i < len(args); i += 2 { - values = append(values, args[i]) - } - sort.Strings(values) - - // Rebuild args with sorted, deduplicated values - newArgs := make([]string, 0, len(args)) - prev := "" - for _, value := range values { - if value == prev { - continue +func copilotCustomMCPToolArgs(toolName string, toolConfig any) []string { + toolConfigMap, ok := toolConfig.(map[string]any) + if !ok { + return nil + } + if hasMcp, _ := hasMCPConfig(toolConfigMap); !hasMcp { + return nil + } + copilotEngineToolsLog.Printf("Adding custom MCP server permission: %s", toolName) + args := []string{"--allow-tool", toolName} + if allowedList, ok := toolConfigMap["allowed"].([]any); ok { + for _, allowedTool := range allowedList { + if toolStr, ok := allowedTool.(string); ok { + args = append(args, "--allow-tool", fmt.Sprintf("%s(%s)", toolName, toolStr)) } - newArgs = append(newArgs, "--allow-tool", value) - prev = value } - args = newArgs } - - copilotEngineToolsLog.Printf("Computed %d tool arguments", len(args)/2) return args } +func sortAndDeduplicateCopilotToolArgs(args []string) []string { + if len(args) == 0 { + return args + } + var values []string + for i := 1; i < len(args); i += 2 { + values = append(values, args[i]) + } + sort.Strings(values) + newArgs := make([]string, 0, len(args)) + prev := "" + for _, value := range values { + if value == prev { + continue + } + newArgs = append(newArgs, "--allow-tool", value) + prev = value + } + return newArgs +} + // generateCopilotToolArgumentsComment generates a multi-line comment showing each tool argument. // This is used to document which tool permissions are being granted in the compiled workflow. func (e *CopilotEngine) generateCopilotToolArgumentsComment(tools map[string]any, safeOutputs *SafeOutputsConfig, mcpScripts *MCPScriptsConfig, workflowData *WorkflowData, indent string) string { diff --git a/pkg/workflow/copilot_logs.go b/pkg/workflow/copilot_logs.go index 1f86803f1ab..3f3f09adead 100644 --- a/pkg/workflow/copilot_logs.go +++ b/pkg/workflow/copilot_logs.go @@ -149,153 +149,163 @@ type SessionUsage struct { OutputTokens int `json:"output_tokens"` } +type sessionJSONLParseState struct { + metrics LogMetrics + totalTokenUsage int + toolCallMap map[string]*ToolCallInfo + currentSequence []string + turns int + assistantMessageCount int + foundSessionEntry bool + verbose bool +} + // parseSessionJSONL attempts to parse the log content as JSONL session format // Returns true if successful, false if the format is not recognized func (e *CopilotEngine) parseSessionJSONL(logContent string, verbose bool) (LogMetrics, bool) { - var metrics LogMetrics - var totalTokenUsage int - toolCallMap := make(map[string]*ToolCallInfo) - var currentSequence []string - turns := 0 - assistantMessageCount := 0 // fallback: count assistant messages when num_turns is absent - - lines := strings.Split(logContent, "\n") - foundSessionEntry := false - - for _, line := range lines { + state := sessionJSONLParseState{ + toolCallMap: make(map[string]*ToolCallInfo), + verbose: verbose, + } + for _, line := range strings.Split(logContent, "\n") { trimmedLine := strings.TrimSpace(line) - - // Skip empty lines and debug log lines if trimmedLine == "" || !strings.HasPrefix(trimmedLine, "{") { continue } - - // Try to parse as session entry var entry SessionEntry if err := json.Unmarshal([]byte(trimmedLine), &entry); err != nil { continue } + state.foundSessionEntry = true + state.processSessionEntry(entry) + } + return state.finalize() +} - foundSessionEntry = true - - // Handle different entry types - switch entry.Type { - case "system": - // System init entry - no action needed for metrics - if verbose { - copilotLogsLog.Printf("Found system init entry") - } - - case "assistant": - // Each assistant message represents one LLM turn - assistantMessageCount++ - - // Assistant message with potential tool calls - if entry.Message != nil { - for _, content := range entry.Message.Content { - if content.Type == "tool_use" { - toolName := content.Name - - // Track in sequence - currentSequence = append(currentSequence, toolName) - - // Calculate input size - inputSize := 0 - if content.Input != nil { - inputJSON, _ := json.Marshal(content.Input) //nolint:jsonmarshalignoredeerror // used only for len() size metric; failure yields len(nil)==0 which is acceptable - inputSize = len(inputJSON) - } - - // Update or create tool call info - if toolInfo, exists := toolCallMap[toolName]; exists { - toolInfo.CallCount++ - if inputSize > toolInfo.MaxInputSize { - toolInfo.MaxInputSize = inputSize - } - } else { - toolCallMap[toolName] = &ToolCallInfo{ - Name: toolName, - CallCount: 1, - MaxInputSize: inputSize, - MaxOutputSize: 0, - } - } +func (s *sessionJSONLParseState) processSessionEntry(entry SessionEntry) { + switch entry.Type { + case "system": + if s.verbose { + copilotLogsLog.Printf("Found system init entry") + } + case "assistant": + s.assistantMessageCount++ + s.processAssistantSessionMessage(entry.Message) + case "user": + s.processUserSessionMessage(entry.Message) + case "result": + s.processSessionResult(entry) + } +} - if verbose { - copilotLogsLog.Printf("Found tool call: %s with input size %d", toolName, inputSize) - } - } - } - } +func (s *sessionJSONLParseState) processAssistantSessionMessage(message *SessionMessage) { + if message == nil { + return + } + for _, content := range message.Content { + if content.Type != "tool_use" { + continue + } + toolName := content.Name + s.currentSequence = append(s.currentSequence, toolName) + inputSize := 0 + if content.Input != nil { + inputJSON, _ := json.Marshal(content.Input) //nolint:jsonmarshalignoredeerror // used only for len() size metric; failure yields len(nil)==0 which is acceptable + inputSize = len(inputJSON) + } + upsertToolInputSize(s.toolCallMap, toolName, inputSize) + if s.verbose { + copilotLogsLog.Printf("Found tool call: %s with input size %d", toolName, inputSize) + } + } +} - case "user": - // User message with tool results - if entry.Message != nil { - for _, content := range entry.Message.Content { - if content.Type == "tool_result" && content.ToolUseID != "" { - // Track output size - outputSize := len(content.Content) - - // Try to find the tool by matching recent tools in sequence - // Since we don't have the tool ID mapping, we'll update the most recent matching tool - for toolName, toolInfo := range toolCallMap { - if outputSize > toolInfo.MaxOutputSize { - toolInfo.MaxOutputSize = outputSize - if verbose { - copilotLogsLog.Printf("Updated %s MaxOutputSize to %d bytes", toolName, outputSize) - } - break // Update first matching tool - } - } - } - } - } +func upsertToolInputSize(toolCallMap map[string]*ToolCallInfo, toolName string, inputSize int) { + if toolInfo, exists := toolCallMap[toolName]; exists { + toolInfo.CallCount++ + if inputSize > toolInfo.MaxInputSize { + toolInfo.MaxInputSize = inputSize + } + return + } + toolCallMap[toolName] = &ToolCallInfo{ + Name: toolName, + CallCount: 1, + MaxInputSize: inputSize, + MaxOutputSize: 0, + } +} - case "result": - // Result entry with usage statistics - if entry.Usage != nil { - totalTokenUsage = entry.Usage.InputTokens + entry.Usage.OutputTokens - turns = entry.NumTurns +func (s *sessionJSONLParseState) processUserSessionMessage(message *SessionMessage) { + if message == nil { + return + } + for _, content := range message.Content { + if content.Type == "tool_result" && content.ToolUseID != "" { + s.updateFirstToolOutputSize(len(content.Content)) + } + } +} - if verbose { - copilotLogsLog.Printf("Found result entry: input_tokens=%d, output_tokens=%d, num_turns=%d", - entry.Usage.InputTokens, entry.Usage.OutputTokens, turns) - } +func (s *sessionJSONLParseState) updateFirstToolOutputSize(outputSize int) { + for toolName, toolInfo := range s.toolCallMap { + if outputSize > toolInfo.MaxOutputSize { + toolInfo.MaxOutputSize = outputSize + if s.verbose { + copilotLogsLog.Printf("Updated %s MaxOutputSize to %d bytes", toolName, outputSize) } + break } } +} - // If turns was not set from num_turns (0 or absent), fall back to counting assistant messages. - // The Copilot CLI may omit num_turns from the result entry; each assistant message represents - // one LLM conversation turn. - if turns == 0 && assistantMessageCount > 0 { - turns = assistantMessageCount - copilotLogsLog.Printf("num_turns not available in result entry, using assistant message count as turns: %d", turns) +func (s *sessionJSONLParseState) processSessionResult(entry SessionEntry) { + if entry.Usage == nil { + return } - - // If we found no session entries, return false to indicate fallback needed - if !foundSessionEntry { - return metrics, false + s.totalTokenUsage = entry.Usage.InputTokens + entry.Usage.OutputTokens + s.turns = entry.NumTurns + if s.verbose { + copilotLogsLog.Printf("Found result entry: input_tokens=%d, output_tokens=%d, num_turns=%d", + entry.Usage.InputTokens, entry.Usage.OutputTokens, s.turns) } +} - // Save current sequence before finalizing - if len(currentSequence) > 0 { - metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) +func (s *sessionJSONLParseState) finalize() (LogMetrics, bool) { + if s.turns == 0 && s.assistantMessageCount > 0 { + s.turns = s.assistantMessageCount + copilotLogsLog.Printf("num_turns not available in result entry, using assistant message count as turns: %d", s.turns) + } + if !s.foundSessionEntry { + return s.metrics, false + } + if len(s.currentSequence) > 0 { + s.metrics.ToolSequences = append(s.metrics.ToolSequences, s.currentSequence) } - - // Finalize metrics copilotLogsLog.Printf("Session JSONL parsing complete: totalTokenUsage=%d, turns=%d, toolCalls=%d", - totalTokenUsage, turns, len(toolCallMap)) - + s.totalTokenUsage, s.turns, len(s.toolCallMap)) FinalizeToolMetrics(FinalizeToolMetricsOptions{ - Metrics: &metrics, - ToolCallMap: toolCallMap, - CurrentSequence: currentSequence, - Turns: turns, - TokenUsage: totalTokenUsage, + Metrics: &s.metrics, + ToolCallMap: s.toolCallMap, + CurrentSequence: s.currentSequence, + Turns: s.turns, + TokenUsage: s.totalTokenUsage, }) + return s.metrics, true +} - return metrics, true +type copilotDebugLogParseState struct { + engine *CopilotEngine + metrics LogMetrics + totalTokenUsage int + toolCallMap map[string]*ToolCallInfo + currentSequence []string + turns int + inDataBlock bool + currentJSONLines []string + inWireBlock bool + currentWireLines []string + verbose bool } // ParseLogMetrics implements engine-specific log parsing for Copilot CLI. @@ -303,204 +313,153 @@ func (e *CopilotEngine) parseSessionJSONL(logContent string, verbose bool) (LogM // Parsing Strategy: // 1. First attempts to parse as JSONL session format (from ~/.copilot/session-state/*.jsonl) // 2. Falls back to debug log format if JSONL parsing fails or finds no entries -// -// Token Counting Behavior: -// Copilot CLI makes multiple API calls during a workflow run (one per turn). -// Each API call returns a response with usage statistics including token counts. -// This function accumulates token counts from ALL API responses to get the total -// token usage for the entire workflow run. -// -// Example: If a run has 3 turns with token counts [1000, 1500, 800], -// the total token usage will be 3300 (sum of all turns). -// -// This matches the behavior of the JavaScript parser in parse_copilot_log.cjs. -// -// Wire request block parsing (wireApi=responses format): -// When the Copilot CLI uses the OpenAI Responses API wire format, each turn is -// preceded by a [DEBUG] Wire request: block containing the full conversation -// history, including function_call_output items for completed tool calls. -// These blocks are parsed to extract tool output sizes and a response preview. func (e *CopilotEngine) ParseLogMetrics(logContent string, verbose bool) LogMetrics { - // Try parsing as JSONL session format first if metrics, success := e.parseSessionJSONL(logContent, verbose); success { copilotLogsLog.Printf("Successfully parsed session JSONL format") return metrics } - - // Fall back to debug log format parsing copilotLogsLog.Printf("JSONL parsing failed or no entries found, falling back to debug log format") - var metrics LogMetrics - var totalTokenUsage int - - lines := strings.Split(logContent, "\n") - toolCallMap := make(map[string]*ToolCallInfo) // Track tool calls - var currentSequence []string // Track tool sequence - turns := 0 - - // Track multi-line JSON blocks for token extraction - var inDataBlock bool - var currentJSONLines []string - - // Track Wire request blocks for tool output extraction - var inWireBlock bool - var currentWireLines []string + state := copilotDebugLogParseState{ + engine: e, + toolCallMap: make(map[string]*ToolCallInfo), + verbose: verbose, + } + for _, line := range strings.Split(logContent, "\n") { + state.processDebugLogLine(line) + } + return state.finalize() +} - // flushDataBlock processes and clears the accumulated data block. - flushDataBlock := func() { - if len(currentJSONLines) == 0 { - return - } - jsonStr := strings.Join(currentJSONLines, "\n") - copilotLogsLog.Printf("Parsing JSON block with %d lines (%d bytes)", len(currentJSONLines), len(jsonStr)) - jsonMetrics := ExtractJSONMetrics(jsonStr, verbose) - // Accumulate token usage from all responses (not just max) - // This matches the JavaScript parser behavior in parse_copilot_log.cjs - if jsonMetrics.TokenUsage > 0 { - copilotLogsLog.Printf("Extracted %d tokens from JSON block", jsonMetrics.TokenUsage) - totalTokenUsage += jsonMetrics.TokenUsage - } else { - copilotLogsLog.Printf("No tokens extracted from JSON block (possible format issue)") - } - if jsonMetrics.EstimatedCost > 0 { - metrics.EstimatedCost += jsonMetrics.EstimatedCost - } - e.extractToolCallSizes(jsonStr, toolCallMap, verbose) - inDataBlock = false - currentJSONLines = []string{} +func (s *copilotDebugLogParseState) processDebugLogLine(line string) { + if strings.TrimSpace(line) == "" { + return + } + if s.startDataBlock(line) || s.startWireBlock(line) { + return } + if s.inDataBlock { + s.accumulateDataBlockLine(line) + } + if s.inWireBlock { + s.accumulateWireBlockLine(line) + } + if toolName := s.engine.parseCopilotToolCallsWithSequence(line, s.toolCallMap); toolName != "" { + s.currentSequence = append(s.currentSequence, toolName) + } +} - // flushWireBlock processes and clears the accumulated wire request block. - flushWireBlock := func() { - if len(currentWireLines) > 0 { - wireStr := strings.Join(currentWireLines, "\n") - e.extractWireRequestOutputs(wireStr, toolCallMap, verbose) - } - inWireBlock = false - currentWireLines = []string{} +func (s *copilotDebugLogParseState) startDataBlock(line string) bool { + if !isTimestampedDebugLine(line, "[DEBUG] data:") { + return false } + if s.inWireBlock { + s.flushWireBlock() + } + s.inDataBlock = true + s.currentJSONLines = []string{} + s.turns++ + if len(s.currentSequence) > 0 { + s.metrics.ToolSequences = append(s.metrics.ToolSequences, s.currentSequence) + s.currentSequence = []string{} + } + return true +} - for _, line := range lines { - // Skip empty lines - if strings.TrimSpace(line) == "" { - continue - } +func (s *copilotDebugLogParseState) startWireBlock(line string) bool { + if !isTimestampedDebugLine(line, "[DEBUG] Wire request:") { + return false + } + if s.inDataBlock { + s.flushDataBlock() + } + if s.inWireBlock { + s.flushWireBlock() + } + s.inWireBlock = true + s.currentWireLines = []string{} + if idx := strings.Index(line, "{"); idx >= 0 { + s.currentWireLines = append(s.currentWireLines, line[idx:]) + } + return true +} - // Detect start of a JSON data block from Copilot debug logs. - // Format: "YYYY-MM-DDTHH:MM:SS.sssZ [DEBUG] data:" - if isTimestampedDebugLine(line, "[DEBUG] data:") { - // End any open wire block before starting a data block. - if inWireBlock { - flushWireBlock() - } - inDataBlock = true - currentJSONLines = []string{} - // Each API response data block represents one LLM conversation turn. - // Copilot CLI debug logs don't have "User:"/"Human:" patterns, so we - // count turns based on the number of API responses (data blocks). - turns++ - // Save previous sequence before starting new turn - if len(currentSequence) > 0 { - metrics.ToolSequences = append(metrics.ToolSequences, currentSequence) - currentSequence = []string{} - } - continue - } +func (s *copilotDebugLogParseState) accumulateDataBlockLine(line string) { + if !strings.Contains(line, "[DEBUG]") { + s.currentJSONLines = append(s.currentJSONLines, line) + return + } + _, after, ok := strings.Cut(line, "[DEBUG]") + if !ok { + return + } + cleanLine := strings.TrimSpace(after) + if startsJSONLine(cleanLine) { + s.currentJSONLines = append(s.currentJSONLines, cleanLine) + return + } + s.flushDataBlock() +} - // Detect start of a Wire request block (wireApi=responses format). - // Format: "YYYY-MM-DDTHH:MM:SS.sssZ [DEBUG] Wire request: {" - if isTimestampedDebugLine(line, "[DEBUG] Wire request:") { - // End any open data block before starting a wire block. - if inDataBlock { - flushDataBlock() - } - // End any open wire block before starting a new wire block. - if inWireBlock { - flushWireBlock() - } - inWireBlock = true - currentWireLines = []string{} - // Extract the opening { from the same line (Wire request: {) - if idx := strings.Index(line, "{"); idx >= 0 { - currentWireLines = append(currentWireLines, line[idx:]) - } - continue - } +func startsJSONLine(line string) bool { + return strings.HasPrefix(line, "{") || strings.HasPrefix(line, "}") || + strings.HasPrefix(line, "[") || strings.HasPrefix(line, "]") || strings.HasPrefix(line, "\"") +} - // While in a data block, accumulate lines - if inDataBlock { - // Check if this line has a [DEBUG] prefix (indicates it's a log line, not raw JSON) - hasDebug := strings.Contains(line, "[DEBUG]") - - if hasDebug { - // Strip the timestamp and [DEBUG] prefix to see what remains - // Format: "YYYY-MM-DDTHH:MM:SS.sssZ [DEBUG] {json content}" - _, after, ok := strings.Cut(line, "[DEBUG]") - if ok { - cleanLine := strings.TrimSpace(after) // Skip "[DEBUG]" - - // If after stripping, the line starts with JSON characters, it's part of JSON - // Otherwise, it's a new log entry and we should end the block - if strings.HasPrefix(cleanLine, "{") || strings.HasPrefix(cleanLine, "}") || - strings.HasPrefix(cleanLine, "[") || strings.HasPrefix(cleanLine, "]") || - strings.HasPrefix(cleanLine, "\"") { - // This is JSON content - add it - currentJSONLines = append(currentJSONLines, cleanLine) - } else { - // This is a new log line (not JSON content) - end of JSON block - flushDataBlock() - } - } - } else { - // Line has no [DEBUG] prefix — treat as raw JSON content. - // Note: [INFO] lines (e.g. "--- End of group ---") also land here but - // are harmless: sanitizeJSONBlock in extractToolCallSizes/ExtractJSONMetrics - // trims everything after the last closing brace. - currentJSONLines = append(currentJSONLines, line) - } - } +func (s *copilotDebugLogParseState) accumulateWireBlockLine(line string) { + if isTimestampedDebugOrInfoLine(line) { + s.flushWireBlock() + return + } + s.currentWireLines = append(s.currentWireLines, line) +} - // While in a wire request block, accumulate raw JSON lines. - // Wire request JSON is not prefixed per-line; the block ends when any - // timestamped log line ([DEBUG] or [INFO]) appears. - if inWireBlock { - if isTimestampedDebugOrInfoLine(line) { - flushWireBlock() - // The block-start checks above already evaluated this line (no match); - // fall through to the tool-call extraction below. - } else { - currentWireLines = append(currentWireLines, line) - } - } +func (s *copilotDebugLogParseState) flushDataBlock() { + if len(s.currentJSONLines) == 0 { + return + } + jsonStr := strings.Join(s.currentJSONLines, "\n") + copilotLogsLog.Printf("Parsing JSON block with %d lines (%d bytes)", len(s.currentJSONLines), len(jsonStr)) + jsonMetrics := ExtractJSONMetrics(jsonStr, s.verbose) + if jsonMetrics.TokenUsage > 0 { + copilotLogsLog.Printf("Extracted %d tokens from JSON block", jsonMetrics.TokenUsage) + s.totalTokenUsage += jsonMetrics.TokenUsage + } else { + copilotLogsLog.Printf("No tokens extracted from JSON block (possible format issue)") + } + if jsonMetrics.EstimatedCost > 0 { + s.metrics.EstimatedCost += jsonMetrics.EstimatedCost + } + s.engine.extractToolCallSizes(jsonStr, s.toolCallMap, s.verbose) + s.inDataBlock = false + s.currentJSONLines = []string{} +} - // Extract tool calls and add to sequence and toolCallMap - // "Executing tool: " lines confirm tool execution and are used to populate - // both the tool sequence and tool call statistics. This handles the common case where - // Copilot CLI JSON blocks have empty tool_calls arrays but emit execution log lines. - if toolName := e.parseCopilotToolCallsWithSequence(line, toolCallMap); toolName != "" { - currentSequence = append(currentSequence, toolName) - } +func (s *copilotDebugLogParseState) flushWireBlock() { + if len(s.currentWireLines) > 0 { + wireStr := strings.Join(s.currentWireLines, "\n") + s.engine.extractWireRequestOutputs(wireStr, s.toolCallMap, s.verbose) } + s.inWireBlock = false + s.currentWireLines = []string{} +} - // Process any remaining blocks at the end of file - if inDataBlock { - flushDataBlock() +func (s *copilotDebugLogParseState) finalize() LogMetrics { + if s.inDataBlock { + s.flushDataBlock() } - if inWireBlock { - flushWireBlock() + if s.inWireBlock { + s.flushWireBlock() } - - // Finalize metrics using shared helper - copilotLogsLog.Printf("Finalized metrics: totalTokenUsage=%d, turns=%d, toolCalls=%d", totalTokenUsage, turns, len(toolCallMap)) + copilotLogsLog.Printf("Finalized metrics: totalTokenUsage=%d, turns=%d, toolCalls=%d", s.totalTokenUsage, s.turns, len(s.toolCallMap)) FinalizeToolMetrics(FinalizeToolMetricsOptions{ - Metrics: &metrics, - ToolCallMap: toolCallMap, - CurrentSequence: currentSequence, - Turns: turns, - TokenUsage: totalTokenUsage, + Metrics: &s.metrics, + ToolCallMap: s.toolCallMap, + CurrentSequence: s.currentSequence, + Turns: s.turns, + TokenUsage: s.totalTokenUsage, }) - - return metrics + return s.metrics } // extractToolCallSizes extracts tool call input sizes from Copilot JSON responses. @@ -597,91 +556,92 @@ func isWireOutputStub(toolInfo *ToolCallInfo) bool { // extractWireRequestOutputs parses a [DEBUG] Wire request: JSON block and updates // MaxOutputSize and OutputSample for each tool that has a function_call_output entry. -// -// The wireApi=responses format includes the full conversation history in each request's -// "input" array. Completed tool calls appear as consecutive function_call / function_call_output -// pairs, letting us extract both the tool name (from function_call.name) and the tool -// response (from function_call_output.output) in a single pass. func (e *CopilotEngine) extractWireRequestOutputs(jsonStr string, toolCallMap map[string]*ToolCallInfo, verbose bool) { - clean := sanitizeJSONBlock(jsonStr) - if clean == "" { + inputs, ok := parseWireRequestInputs(jsonStr, verbose) + if !ok { return } + callIDToTool := wireRequestCallIDToTool(inputs) + for _, item := range inputs { + itemMap, ok := item.(map[string]any) + if !ok { + continue + } + toolName, output, ok := wireRequestOutput(itemMap, callIDToTool) + if ok { + updateWireRequestToolOutput(toolCallMap, toolName, output, verbose) + } + } +} +func parseWireRequestInputs(jsonStr string, verbose bool) ([]any, bool) { + clean := sanitizeJSONBlock(jsonStr) + if clean == "" { + return nil, false + } var data map[string]any if err := json.Unmarshal([]byte(clean), &data); err != nil { if verbose { copilotLogsLog.Printf("Failed to parse Wire request JSON: %v", err) } - return + return nil, false } - inputs, ok := data["input"].([]any) - if !ok { - return - } + return inputs, ok +} - // Build a local call_id → tool name map from function_call items in this request. - // The Wire request contains the full conversation history, so all historical - // function_call / function_call_output pairs are present in a single block. +func wireRequestCallIDToTool(inputs []any) map[string]string { callIDToTool := make(map[string]string, len(inputs)) for _, item := range inputs { itemMap, ok := item.(map[string]any) if !ok { continue } - if typ, _ := itemMap["type"].(string); typ == "function_call" { - callID, _ := itemMap["call_id"].(string) - name, _ := itemMap["name"].(string) - if callID != "" && name != "" { - callIDToTool[callID] = name - } - } - } - - // Extract output sizes and samples from function_call_output items. - for _, item := range inputs { - itemMap, ok := item.(map[string]any) - if !ok { + if typ, _ := itemMap["type"].(string); typ != "function_call" { continue } - if typ, _ := itemMap["type"].(string); typ != "function_call_output" { - continue - } - callID, _ := itemMap["call_id"].(string) - output, _ := itemMap["output"].(string) - if callID == "" || output == "" { - continue + name, _ := itemMap["name"].(string) + if callID != "" && name != "" { + callIDToTool[callID] = name } + } + return callIDToTool +} - toolName := callIDToTool[callID] - if toolName == "" { - continue - } +func wireRequestOutput(itemMap map[string]any, callIDToTool map[string]string) (string, string, bool) { + if typ, _ := itemMap["type"].(string); typ != "function_call_output" { + return "", "", false + } + callID, _ := itemMap["call_id"].(string) + output, _ := itemMap["output"].(string) + if callID == "" || output == "" { + return "", "", false + } + toolName := callIDToTool[callID] + return toolName, output, toolName != "" +} - outputSize := len(output) - if toolInfo, exists := toolCallMap[toolName]; exists { - if outputSize > toolInfo.MaxOutputSize { - toolInfo.MaxOutputSize = outputSize - toolInfo.OutputSample = truncateOutputSample(output) - if verbose { - copilotLogsLog.Printf("Updated %s MaxOutputSize to %d bytes with sample", toolName, outputSize) - } - } - } else { - // Tool entry not yet created by extractToolCallSizes — create a stub so the - // output sample is not lost (e.g. when wire-request ordering differs from data blocks). - toolCallMap[toolName] = &ToolCallInfo{ - Name: toolName, - CallCount: 1, - MaxOutputSize: outputSize, - OutputSample: truncateOutputSample(output), - } +func updateWireRequestToolOutput(toolCallMap map[string]*ToolCallInfo, toolName, output string, verbose bool) { + outputSize := len(output) + if toolInfo, exists := toolCallMap[toolName]; exists { + if outputSize > toolInfo.MaxOutputSize { + toolInfo.MaxOutputSize = outputSize + toolInfo.OutputSample = truncateOutputSample(output) if verbose { - copilotLogsLog.Printf("Created stub entry for %s from wire request output (%d bytes)", toolName, outputSize) + copilotLogsLog.Printf("Updated %s MaxOutputSize to %d bytes with sample", toolName, outputSize) } } + return + } + toolCallMap[toolName] = &ToolCallInfo{ + Name: toolName, + CallCount: 1, + MaxOutputSize: outputSize, + OutputSample: truncateOutputSample(output), + } + if verbose { + copilotLogsLog.Printf("Created stub entry for %s from wire request output (%d bytes)", toolName, outputSize) } } diff --git a/pkg/workflow/create_agent_session.go b/pkg/workflow/create_agent_session.go index df54858d347..af86cc4f65c 100644 --- a/pkg/workflow/create_agent_session.go +++ b/pkg/workflow/create_agent_session.go @@ -16,67 +16,40 @@ type CreateAgentSessionConfig struct { // parseAgentSessionConfig handles create-agent-session configuration func (c *Compiler) parseAgentSessionConfig(outputMap map[string]any) *CreateAgentSessionConfig { - // Try new key first if configData, exists := outputMap["create-agent-session"]; exists { createAgentSessionLog.Print("Parsing create-agent-session configuration") - agentSessionConfig := &CreateAgentSessionConfig{} - - if configMap, ok := configData.(map[string]any); ok { - // Parse base branch - if base, exists := configMap["base"]; exists { - if baseStr, ok := base.(string); ok { - agentSessionConfig.Base = baseStr - } - } - - // Parse target-repo using shared helper with validation - targetRepoSlug, isInvalid := parseTargetRepoWithValidation(configMap) - if isInvalid { - return nil // Invalid configuration, return nil to cause validation error - } - agentSessionConfig.TargetRepoSlug = targetRepoSlug - - // Parse common base fields with default max of 1 - c.parseBaseSafeOutputConfig(configMap, &agentSessionConfig.BaseSafeOutputConfig, 1) - } else { - // If configData is nil or not a map (e.g., "create-agent-session:" with no value), - // still set the default max - agentSessionConfig.Max = defaultIntStr(1) - } - - return agentSessionConfig + return c.parseAgentSessionConfigData(configData) } - // Fall back to deprecated key for backward compatibility if configData, exists := outputMap["create-agent-task"]; exists { createAgentSessionLog.Print("WARNING: Using deprecated 'create-agent-task' configuration. Please migrate to 'create-agent-session' using 'gh aw fix'") - agentSessionConfig := &CreateAgentSessionConfig{} + return c.parseAgentSessionConfigData(configData) + } - if configMap, ok := configData.(map[string]any); ok { - // Parse base branch - if base, exists := configMap["base"]; exists { - if baseStr, ok := base.(string); ok { - agentSessionConfig.Base = baseStr - } - } + return nil +} + +func (c *Compiler) parseAgentSessionConfigData(configData any) *CreateAgentSessionConfig { + agentSessionConfig := &CreateAgentSessionConfig{} - // Parse target-repo using shared helper with validation - targetRepoSlug, isInvalid := parseTargetRepoWithValidation(configMap) - if isInvalid { - return nil // Invalid configuration, return nil to cause validation error - } - agentSessionConfig.TargetRepoSlug = targetRepoSlug + configMap, ok := configData.(map[string]any) + if !ok { + agentSessionConfig.Max = defaultIntStr(1) + return agentSessionConfig + } - // Parse common base fields with default max of 1 - c.parseBaseSafeOutputConfig(configMap, &agentSessionConfig.BaseSafeOutputConfig, 1) - } else { - // If configData is nil or not a map (e.g., "create-agent-task:" with no value), - // still set the default max - agentSessionConfig.Max = defaultIntStr(1) + if base, exists := configMap["base"]; exists { + if baseStr, ok := base.(string); ok { + agentSessionConfig.Base = baseStr } + } - return agentSessionConfig + targetRepoSlug, isInvalid := parseTargetRepoWithValidation(configMap) + if isInvalid { + return nil } + agentSessionConfig.TargetRepoSlug = targetRepoSlug + c.parseBaseSafeOutputConfig(configMap, &agentSessionConfig.BaseSafeOutputConfig, 1) - return nil + return agentSessionConfig } diff --git a/pkg/workflow/create_code_scanning_alert.go b/pkg/workflow/create_code_scanning_alert.go index 09586f87fa2..3129b48bc89 100644 --- a/pkg/workflow/create_code_scanning_alert.go +++ b/pkg/workflow/create_code_scanning_alert.go @@ -84,27 +84,7 @@ func (c *Compiler) buildCodeScanningUploadJob(data *WorkflowData) (*Job, error) // We cannot pass tokens through job outputs (GitHub Actions masks secret references). // We must either compute the static token directly or mint a fresh GitHub App token. checkoutMgr := NewCheckoutManager(data.CheckoutConfigs) - - var restoreToken string - var tokenMintSteps []string - - // Check if the default checkout uses GitHub App auth. If so, mint a fresh token - // in this job — activation/safe_outputs app tokens have expired by this point. - defaultOverride := checkoutMgr.GetDefaultCheckoutOverride() - if defaultOverride != nil && defaultOverride.githubApp != nil { - permissions := NewPermissionsContentsReadSecurityEventsWrite() - for _, step := range c.buildGitHubAppTokenMintStep(defaultOverride.githubApp, permissions, "") { - tokenMintSteps = append(tokenMintSteps, - strings.ReplaceAll(step, "id: safe-outputs-app-token", "id: checkout-restore-app-token")) - } - //nolint:gosec // G101: False positive - this is a GitHub Actions expression template, not a hardcoded credential - restoreToken = "${{ steps.checkout-restore-app-token.outputs.token }}" - } else { - // No GitHub App configured for checkout — compute a static secret reference - // directly. This is safe because secret references are evaluated in the job's own - // context (not through job outputs which would be masked by GitHub Actions). - restoreToken = resolveStaticCheckoutToken(data.SafeOutputs, checkoutMgr) - } + restoreToken, tokenMintSteps := c.resolveCodeScanningRestoreToken(data, checkoutMgr) // Artifact prefix for workflow_call context (so the download name matches the upload name). agentArtifactPrefix := artifactPrefixExprForDownstreamJob(data) @@ -113,47 +93,12 @@ func (c *Compiler) buildCodeScanningUploadJob(data *WorkflowData) (*Job, error) // Prepend any token minting steps (needed when checkout uses GitHub App auth). steps = append(steps, tokenMintSteps...) - - // Step: Restore workspace to the triggering commit. - // The safe_outputs job may have checked out a different branch (e.g., the base branch for - // a PR) which would leave HEAD pointing at a different commit. The SARIF upload action - // requires HEAD to match the commit being scanned, otherwise it fails with "commit not found". - steps = append(steps, " - name: Restore checkout to triggering commit\n") - steps = append(steps, fmt.Sprintf(" uses: %s\n", getActionPin("actions/checkout"))) - steps = append(steps, " with:\n") - steps = append(steps, " ref: ${{ github.sha }}\n") - steps = append(steps, fmt.Sprintf(" token: %s\n", restoreToken)) - steps = append(steps, " persist-credentials: false\n") - steps = append(steps, " fetch-depth: 1\n") - - // Step: Download the SARIF artifact produced by safe_outputs. - // The SARIF file was written to the safe_outputs job workspace and uploaded as an artifact. - // This job runs in a fresh workspace so we must download the artifact before uploading - // to GitHub Code Scanning. - sarifDownloadSteps := buildArtifactDownloadSteps(ArtifactDownloadConfig{ - ArtifactName: agentArtifactPrefix + constants.SarifArtifactName, - DownloadPath: constants.SarifArtifactDownloadPath, - StepName: "Download SARIF artifact", - }, c.getActionPin) - steps = append(steps, sarifDownloadSteps...) + steps = append(steps, buildCodeScanningRestoreCheckoutSteps(restoreToken)...) + steps = append(steps, buildCodeScanningDownloadSteps(agentArtifactPrefix, c.getActionPin)...) // The local SARIF file path after the artifact download completes. localSarifPath := path.Join(constants.SarifArtifactDownloadPath, constants.SarifFileName) - - // Step: Upload SARIF file to GitHub Code Scanning. - steps = append(steps, " - name: Upload SARIF to GitHub Code Scanning\n") - steps = append(steps, fmt.Sprintf(" id: %s\n", constants.UploadCodeScanningJobName)) - steps = append(steps, fmt.Sprintf(" uses: %s\n", c.getActionPin("github/codeql-action/upload-sarif"))) - steps = append(steps, " with:\n") - // NOTE: github/codeql-action/upload-sarif uses 'token' as the input name, not 'github-token' - // Pass restoreToken as the fallback so GitHub App-minted tokens flow through consistently. - c.addUploadSARIFToken(&steps, data, data.SafeOutputs.CreateCodeScanningAlerts.GitHubToken, restoreToken) - // sarif_file now references the locally-downloaded artifact, not the path from safe_outputs - steps = append(steps, fmt.Sprintf(" sarif_file: %s\n", localSarifPath)) - // ref and sha pin the upload to the exact triggering commit regardless of local git state - steps = append(steps, " ref: ${{ github.ref }}\n") - steps = append(steps, " sha: ${{ github.sha }}\n") - steps = append(steps, " wait-for-processing: true\n") + steps = append(steps, c.buildCodeScanningUploadSteps(data, restoreToken, localSarifPath)...) // The job only runs when the safe_outputs job exported a non-empty SARIF file path. jobCondition := fmt.Sprintf("needs.%s.outputs.sarif_file != ''", constants.SafeOutputsJobName) @@ -177,6 +122,54 @@ func (c *Compiler) buildCodeScanningUploadJob(data *WorkflowData) (*Job, error) return job, nil } +func (c *Compiler) resolveCodeScanningRestoreToken(data *WorkflowData, checkoutMgr *CheckoutManager) (string, []string) { + defaultOverride := checkoutMgr.GetDefaultCheckoutOverride() + if defaultOverride == nil || defaultOverride.githubApp == nil { + return resolveStaticCheckoutToken(data.SafeOutputs, checkoutMgr), nil + } + permissions := NewPermissionsContentsReadSecurityEventsWrite() + var tokenMintSteps []string + for _, step := range c.buildGitHubAppTokenMintStep(defaultOverride.githubApp, permissions, "") { + tokenMintSteps = append(tokenMintSteps, strings.ReplaceAll(step, "id: safe-outputs-app-token", "id: checkout-restore-app-token")) + } + //nolint:gosec // G101: False positive - this is a GitHub Actions expression template, not a hardcoded credential + return "${{ steps.checkout-restore-app-token.outputs.token }}", tokenMintSteps +} + +func buildCodeScanningRestoreCheckoutSteps(restoreToken string) []string { + return []string{ + " - name: Restore checkout to triggering commit\n", + fmt.Sprintf(" uses: %s\n", getActionPin("actions/checkout")), + " with:\n", + " ref: ${{ github.sha }}\n", + fmt.Sprintf(" token: %s\n", restoreToken), + " persist-credentials: false\n", + " fetch-depth: 1\n", + } +} + +func buildCodeScanningDownloadSteps(agentArtifactPrefix string, getActionPin func(string) string) []string { + return buildArtifactDownloadSteps(ArtifactDownloadConfig{ + ArtifactName: agentArtifactPrefix + constants.SarifArtifactName, + DownloadPath: constants.SarifArtifactDownloadPath, + StepName: "Download SARIF artifact", + }, getActionPin) +} + +func (c *Compiler) buildCodeScanningUploadSteps(data *WorkflowData, restoreToken string, localSarifPath string) []string { + var steps []string + steps = append(steps, " - name: Upload SARIF to GitHub Code Scanning\n") + steps = append(steps, fmt.Sprintf(" id: %s\n", constants.UploadCodeScanningJobName)) + steps = append(steps, fmt.Sprintf(" uses: %s\n", c.getActionPin("github/codeql-action/upload-sarif"))) + steps = append(steps, " with:\n") + c.addUploadSARIFToken(&steps, data, data.SafeOutputs.CreateCodeScanningAlerts.GitHubToken, restoreToken) + steps = append(steps, fmt.Sprintf(" sarif_file: %s\n", localSarifPath)) + steps = append(steps, " ref: ${{ github.ref }}\n") + steps = append(steps, " sha: ${{ github.sha }}\n") + steps = append(steps, " wait-for-processing: true\n") + return steps +} + // addUploadSARIFToken adds the 'token' input for github/codeql-action/upload-sarif. // This action uses 'token' as the input name (not 'github-token' like other GitHub Actions). // This runs inside the upload_code_scanning_sarif job (a separate job from safe_outputs), so diff --git a/pkg/workflow/create_discussion.go b/pkg/workflow/create_discussion.go index 07f2b4e60fc..98372e5ab8e 100644 --- a/pkg/workflow/create_discussion.go +++ b/pkg/workflow/create_discussion.go @@ -75,7 +75,11 @@ func (c *Compiler) parseCreateDiscussionsConfig(outputMap map[string]any) *Creat // Normalize and validate category naming convention config.Category = normalizeDiscussionCategory(config.Category, discussionLog, c.markdownPath) - // Log configured values + logCreateDiscussionConfig(config) + return config +} + +func logCreateDiscussionConfig(config *CreateDiscussionsConfig) { if config.TitlePrefix != "" { discussionLog.Printf("Title prefix configured: %q", config.TitlePrefix) } @@ -106,8 +110,6 @@ func (c *Compiler) parseCreateDiscussionsConfig(outputMap map[string]any) *Creat if config.FallbackToIssue != nil { discussionLog.Printf("Fallback to issue configured: %t", *config.FallbackToIssue) } - - return config } // Returns normalized category (or original if it's a category ID) diff --git a/pkg/workflow/create_pull_request.go b/pkg/workflow/create_pull_request.go index b4c0171914c..f3468a5b449 100644 --- a/pkg/workflow/create_pull_request.go +++ b/pkg/workflow/create_pull_request.go @@ -97,62 +97,10 @@ func (c *Compiler) parseCreatePullRequestsConfig(outputMap map[string]any) *Crea return &CreatePullRequestsConfig{} }, func(configData map[string]any) bool { - coerceStringOrArrayFields(configData, createPRStringOrArrayFields, createPRLog) - - // Pre-process protected-files: supports string enum OR object form {policy, exclude}. - // Object form is preprocessed to extract the policy (stored back as string) and - // the exclude list (stored in a local variable and assigned to the config after unmarshaling). - protectedFilesExclude = preprocessProtectedFilesField(configData, createPRLog) - - // Validate protected-files string enum after object-form preprocessing. - validateStringEnumField(configData, "protected-files", []string{"blocked", "allowed", "fallback-to-issue", "request_review"}, createPRLog) - - // Pre-process patch-format: valid values are "bundle" (default) and "am". - validateStringEnumField(configData, "patch-format", []string{"am", "bundle"}, createPRLog) - - // Pre-process list fields that also accept a GitHub Actions expression string. - // An expression is wrapped in a single-element []string so the []string struct field - // can receive it after YAML unmarshaling; the handler config builder later re-emits it - // as a JSON string for runtime evaluation. - for _, field := range createPRExpressionArrayFields { - if err := preprocessStringArrayFieldAsTemplatable(configData, field, createPRLog); err != nil { - createPRLog.Printf("Invalid %s value: %v", field, err) - return false - } - } - - return true + return preprocessCreatePullRequestConfig(configData, &protectedFilesExclude) }, func(configData map[string]any, config *CreatePullRequestsConfig, expiresDisabled bool) { - if expiresDisabled { - createPRLog.Print("Pull request expiration disabled") - } - - // Log expires if configured - if config.Expires > 0 { - createPRLog.Printf("Pull request expiration configured: %d hours", config.Expires) - } - - // Apply the exclude list extracted from the object-form protected-files field. - config.ProtectedFilesExclude = protectedFilesExclude - - // Set default max if not explicitly configured (default is 1) - if config.Max == nil { - config.Max = defaultIntStr(1) - createPRLog.Print("Using default max count: 1") - } else { - createPRLog.Printf("Pull request max count configured: %s", *config.Max) - } - - // Parse head-github-app manually so that the app-id alias is honoured - // (YAML unmarshal would silently ignore app-id since GitHubAppConfig only - // declares the canonical client-id tag). - if headAppData, exists := configData["head-github-app"]; exists { - if headAppMap, ok := headAppData.(map[string]any); ok { - createPRLog.Print("Parsed head-github-app from config") - config.HeadGitHubApp = parseAppConfig(headAppMap) - } - } + finalizeCreatePullRequestConfig(configData, config, expiresDisabled, protectedFilesExclude) }, ) if config == nil { @@ -161,3 +109,39 @@ func (c *Compiler) parseCreatePullRequestsConfig(outputMap map[string]any) *Crea return config } + +func preprocessCreatePullRequestConfig(configData map[string]any, protectedFilesExclude *[]string) bool { + coerceStringOrArrayFields(configData, createPRStringOrArrayFields, createPRLog) + *protectedFilesExclude = preprocessProtectedFilesField(configData, createPRLog) + validateStringEnumField(configData, "protected-files", []string{"blocked", "allowed", "fallback-to-issue", "request_review"}, createPRLog) + validateStringEnumField(configData, "patch-format", []string{"am", "bundle"}, createPRLog) + for _, field := range createPRExpressionArrayFields { + if err := preprocessStringArrayFieldAsTemplatable(configData, field, createPRLog); err != nil { + createPRLog.Printf("Invalid %s value: %v", field, err) + return false + } + } + return true +} + +func finalizeCreatePullRequestConfig(configData map[string]any, config *CreatePullRequestsConfig, expiresDisabled bool, protectedFilesExclude []string) { + if expiresDisabled { + createPRLog.Print("Pull request expiration disabled") + } + if config.Expires > 0 { + createPRLog.Printf("Pull request expiration configured: %d hours", config.Expires) + } + config.ProtectedFilesExclude = protectedFilesExclude + if config.Max == nil { + config.Max = defaultIntStr(1) + createPRLog.Print("Using default max count: 1") + } else { + createPRLog.Printf("Pull request max count configured: %s", *config.Max) + } + if headAppData, exists := configData["head-github-app"]; exists { + if headAppMap, ok := headAppData.(map[string]any); ok { + createPRLog.Print("Parsed head-github-app from config") + config.HeadGitHubApp = parseAppConfig(headAppMap) + } + } +} diff --git a/pkg/workflow/dependabot.go b/pkg/workflow/dependabot.go index cf8863ecdb0..b02e52f5046 100644 --- a/pkg/workflow/dependabot.go +++ b/pkg/workflow/dependabot.go @@ -78,76 +78,14 @@ func (c *Compiler) GenerateDependabotManifests(workflowDataList []*WorkflowData, ecosystems := make(map[string]struct { }) - // Collect npm dependencies - npmDeps := c.collectNpmDependencies(workflowDataList) - if len(npmDeps) > 0 { - ecosystems["npm"] = struct { - }{} - dependabotLog.Printf("Found %d unique npm dependencies", len(npmDeps)) - if c.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d npm dependencies in workflows", len(npmDeps)))) - } - - // Generate package.json - packageJSONPath := filepath.Join(workflowDir, "package.json") - if err := c.generatePackageJSON(packageJSONPath, npmDeps, forceOverwrite); err != nil { - if c.strictMode { - return fmt.Errorf("failed to generate package.json: %w", err) - } - c.IncrementWarningCount() - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to generate package.json: %v", err))) - } else { - // Generate package-lock.json - if err := c.generatePackageLock(workflowDir); err != nil { - if c.strictMode { - return fmt.Errorf("failed to generate package-lock.json: %w", err) - } - c.IncrementWarningCount() - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to generate package-lock.json: %v", err))) - } - } + if err := c.generateDependabotNpmArtifacts(workflowDataList, workflowDir, forceOverwrite, ecosystems); err != nil { + return err } - - // Collect pip dependencies - pipDeps := c.collectPipDependencies(workflowDataList) - if len(pipDeps) > 0 { - ecosystems["pip"] = struct { - }{} - dependabotLog.Printf("Found %d unique pip dependencies", len(pipDeps)) - if c.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d pip dependencies in workflows", len(pipDeps)))) - } - - // Generate requirements.txt - requirementsPath := filepath.Join(workflowDir, "requirements.txt") - if err := c.generateRequirementsTxt(requirementsPath, pipDeps, forceOverwrite); err != nil { - if c.strictMode { - return fmt.Errorf("failed to generate requirements.txt: %w", err) - } - c.IncrementWarningCount() - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to generate requirements.txt: %v", err))) - } + if err := c.generateDependabotPipArtifacts(workflowDataList, workflowDir, forceOverwrite, ecosystems); err != nil { + return err } - - // Collect go dependencies - goDeps := c.collectGoDependencies(workflowDataList) - if len(goDeps) > 0 { - ecosystems["gomod"] = struct { - }{} - dependabotLog.Printf("Found %d unique go dependencies", len(goDeps)) - if c.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d go dependencies in workflows", len(goDeps)))) - } - - // Generate go.mod - goModPath := filepath.Join(workflowDir, "go.mod") - if err := c.generateGoMod(goModPath, goDeps, forceOverwrite); err != nil { - if c.strictMode { - return fmt.Errorf("failed to generate go.mod: %w", err) - } - c.IncrementWarningCount() - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to generate go.mod: %v", err))) - } + if err := c.generateDependabotGoArtifacts(workflowDataList, workflowDir, forceOverwrite, ecosystems); err != nil { + return err } // If no dependencies found at all, skip @@ -162,11 +100,7 @@ func (c *Compiler) GenerateDependabotManifests(workflowDataList []*WorkflowData, // Generate dependabot.yml with all detected ecosystems dependabotPath := filepath.Join(filepath.Dir(workflowDir), "dependabot.yml") if err := c.generateDependabotConfig(dependabotPath, ecosystems, forceOverwrite); err != nil { - if c.strictMode { - return fmt.Errorf("failed to generate dependabot.yml: %w", err) - } - c.IncrementWarningCount() - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to generate dependabot.yml: %v", err))) + return c.handleDependabotGenerationError("dependabot.yml", err) } if c.verbose { @@ -176,6 +110,69 @@ func (c *Compiler) GenerateDependabotManifests(workflowDataList []*WorkflowData, return nil } +func (c *Compiler) generateDependabotNpmArtifacts(workflowDataList []*WorkflowData, workflowDir string, forceOverwrite bool, ecosystems map[string]struct{}) error { + npmDeps := c.collectNpmDependencies(workflowDataList) + if len(npmDeps) == 0 { + return nil + } + ecosystems["npm"] = struct{}{} + dependabotLog.Printf("Found %d unique npm dependencies", len(npmDeps)) + if c.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d npm dependencies in workflows", len(npmDeps)))) + } + packageJSONPath := filepath.Join(workflowDir, "package.json") + if err := c.generatePackageJSON(packageJSONPath, npmDeps, forceOverwrite); err != nil { + return c.handleDependabotGenerationError("package.json", err) + } + if err := c.generatePackageLock(workflowDir); err != nil { + return c.handleDependabotGenerationError("package-lock.json", err) + } + return nil +} + +func (c *Compiler) generateDependabotPipArtifacts(workflowDataList []*WorkflowData, workflowDir string, forceOverwrite bool, ecosystems map[string]struct{}) error { + pipDeps := c.collectPipDependencies(workflowDataList) + if len(pipDeps) == 0 { + return nil + } + ecosystems["pip"] = struct{}{} + dependabotLog.Printf("Found %d unique pip dependencies", len(pipDeps)) + if c.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d pip dependencies in workflows", len(pipDeps)))) + } + requirementsPath := filepath.Join(workflowDir, "requirements.txt") + if err := c.generateRequirementsTxt(requirementsPath, pipDeps, forceOverwrite); err != nil { + return c.handleDependabotGenerationError("requirements.txt", err) + } + return nil +} + +func (c *Compiler) generateDependabotGoArtifacts(workflowDataList []*WorkflowData, workflowDir string, forceOverwrite bool, ecosystems map[string]struct{}) error { + goDeps := c.collectGoDependencies(workflowDataList) + if len(goDeps) == 0 { + return nil + } + ecosystems["gomod"] = struct{}{} + dependabotLog.Printf("Found %d unique go dependencies", len(goDeps)) + if c.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d go dependencies in workflows", len(goDeps)))) + } + goModPath := filepath.Join(workflowDir, "go.mod") + if err := c.generateGoMod(goModPath, goDeps, forceOverwrite); err != nil { + return c.handleDependabotGenerationError("go.mod", err) + } + return nil +} + +func (c *Compiler) handleDependabotGenerationError(artifact string, err error) error { + if c.strictMode { + return fmt.Errorf("failed to generate %s: %w", artifact, err) + } + c.IncrementWarningCount() + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to generate %s: %v", artifact, err))) + return nil +} + // collectNpmDependencies collects all npm dependencies from workflow data func (c *Compiler) collectNpmDependencies(workflowDataList []*WorkflowData) []NpmDependency { dependabotLog.Print("Collecting npm dependencies from workflows") @@ -256,33 +253,9 @@ func parseNpmPackage(pkg string) NpmDependency { func (c *Compiler) generatePackageJSON(path string, deps []NpmDependency, forceOverwrite bool) error { dependabotLog.Printf("Generating package.json at %s", path) - var pkgJSON PackageJSON - - // Check if package.json already exists - if _, err := os.Stat(path); err == nil { - // File exists - merge dependencies - dependabotLog.Print("Existing package.json found, merging dependencies") - - existingData, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read existing package.json: %w", err) - } - - if err := json.Unmarshal(existingData, &pkgJSON); err != nil { - return fmt.Errorf("failed to parse existing package.json: %w", err) - } - - if c.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Merging with existing package.json")) - } - } else { - // New package.json - dependabotLog.Print("Creating new package.json") - pkgJSON = PackageJSON{ - Name: "gh-aw-workflows-deps", - Private: true, - License: "MIT", - } + pkgJSON, err := c.loadPackageJSONForDependabot(path) + if err != nil { + return err } // Initialize dependencies map if nil @@ -295,17 +268,8 @@ func (c *Compiler) generatePackageJSON(path string, deps []NpmDependency, forceO pkgJSON.Dependencies[dep.Name] = dep.Version } - // Write package.json with nice formatting - jsonData, err := json.MarshalIndent(pkgJSON, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal package.json: %w", err) - } - - // Add newline at end for POSIX compliance - jsonData = append(jsonData, '\n') - - if err := os.WriteFile(path, jsonData, constants.FilePermPublic); err != nil { - return fmt.Errorf("failed to write package.json: %w", err) + if err := c.writePackageJSONForDependabot(path, pkgJSON); err != nil { + return err } dependabotLog.Printf("Successfully wrote package.json with %d dependencies", len(pkgJSON.Dependencies)) @@ -321,6 +285,38 @@ func (c *Compiler) generatePackageJSON(path string, deps []NpmDependency, forceO return nil } +func (c *Compiler) loadPackageJSONForDependabot(path string) (PackageJSON, error) { + if _, err := os.Stat(path); err == nil { + dependabotLog.Print("Existing package.json found, merging dependencies") + existingData, err := os.ReadFile(path) + if err != nil { + return PackageJSON{}, fmt.Errorf("failed to read existing package.json: %w", err) + } + var pkgJSON PackageJSON + if err := json.Unmarshal(existingData, &pkgJSON); err != nil { + return PackageJSON{}, fmt.Errorf("failed to parse existing package.json: %w", err) + } + if c.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Merging with existing package.json")) + } + return pkgJSON, nil + } + dependabotLog.Print("Creating new package.json") + return PackageJSON{Name: "gh-aw-workflows-deps", Private: true, License: "MIT"}, nil +} + +func (c *Compiler) writePackageJSONForDependabot(path string, pkgJSON PackageJSON) error { + jsonData, err := json.MarshalIndent(pkgJSON, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal package.json: %w", err) + } + jsonData = append(jsonData, '\n') + if err := os.WriteFile(path, jsonData, constants.FilePermPublic); err != nil { + return fmt.Errorf("failed to write package.json: %w", err) + } + return nil +} + // generatePackageLock runs npm install --package-lock-only to create package-lock.json func (c *Compiler) generatePackageLock(workflowDir string) error { dependabotLog.Printf("Generating package-lock.json in %s", workflowDir) @@ -368,47 +364,13 @@ func (c *Compiler) generateDependabotConfig(path string, ecosystems map[string]s }, forceOverwrite bool) error { dependabotLog.Printf("Generating dependabot.yml at %s", path) - var config DependabotConfig - - // Check if dependabot.yml already exists - if _, err := os.Stat(path); err == nil { - // File exists - read and merge configuration - dependabotLog.Print("Existing dependabot.yml found, merging configuration") - existingData, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read existing dependabot.yml: %w", err) - } - - if err := yaml.Unmarshal(existingData, &config); err != nil { - // If we can't parse it, start fresh - dependabotLog.Print("Could not parse existing dependabot.yml, creating new one") - config = DependabotConfig{Version: 2} - } - } else { - // New dependabot.yml - dependabotLog.Print("Creating new dependabot.yml") - config = DependabotConfig{Version: 2} + config, err := loadDependabotConfig(path) + if err != nil { + return err } // Add ecosystems that don't already exist for .github/workflows - for ecosystem := range ecosystems { - exists := false - for _, update := range config.Updates { - if update.PackageEcosystem == ecosystem && update.Directory == "/.github/workflows" { - exists = true - break - } - } - - if !exists { - entry := DependabotUpdateEntry{ - PackageEcosystem: ecosystem, - Directory: "/.github/workflows", - } - entry.Schedule.Interval = "weekly" - config.Updates = append(config.Updates, entry) - } - } + addDependabotEcosystems(&config, ecosystems) // Write dependabot.yml yamlData, err := yaml.Marshal(&config) @@ -433,6 +395,47 @@ func (c *Compiler) generateDependabotConfig(path string, ecosystems map[string]s return nil } +func loadDependabotConfig(path string) (DependabotConfig, error) { + if _, err := os.Stat(path); err == nil { + dependabotLog.Print("Existing dependabot.yml found, merging configuration") + existingData, err := os.ReadFile(path) + if err != nil { + return DependabotConfig{}, fmt.Errorf("failed to read existing dependabot.yml: %w", err) + } + var config DependabotConfig + if err := yaml.Unmarshal(existingData, &config); err != nil { + dependabotLog.Print("Could not parse existing dependabot.yml, creating new one") + return DependabotConfig{Version: 2}, nil + } + return config, nil + } + dependabotLog.Print("Creating new dependabot.yml") + return DependabotConfig{Version: 2}, nil +} + +func addDependabotEcosystems(config *DependabotConfig, ecosystems map[string]struct{}) { + for ecosystem := range ecosystems { + if hasDependabotUpdate(config.Updates, ecosystem, "/.github/workflows") { + continue + } + entry := DependabotUpdateEntry{ + PackageEcosystem: ecosystem, + Directory: "/.github/workflows", + } + entry.Schedule.Interval = "weekly" + config.Updates = append(config.Updates, entry) + } +} + +func hasDependabotUpdate(updates []DependabotUpdateEntry, ecosystem, directory string) bool { + for _, update := range updates { + if update.PackageEcosystem == ecosystem && update.Directory == directory { + return true + } + } + return false +} + // ReconcileManagedDependabotIgnores updates existing github-actions entries in .github/dependabot.yml // with compiler-managed ignore rules for compiler-emitted action refs. // This function is a no-op when dependabot.yml does not exist or has no github-actions update entries. @@ -441,106 +444,128 @@ func (c *Compiler) ReconcileManagedDependabotIgnores(path string) error { return nil } - original, err := os.ReadFile(path) + root, updates, original, err := readDependabotUpdatesRoot(path) + if err != nil { + return err + } + if updates == nil { + return nil + } + + managedPatterns := []string{c.effectiveActionsRepo()} + originalStr := string(original) + managedPatternsWithComment := managedPatternsWithInlineComment(originalStr, managedPatterns) + changed := reconcileManagedDependabotUpdateIgnores(updates, managedPatterns, managedPatternsWithComment) + + if !changed { + return nil + } + + root["updates"] = updates + updated, err := yaml.Marshal(root) if err != nil { - return fmt.Errorf("failed to read dependabot.yml: %w", err) + return fmt.Errorf("failed to encode dependabot.yml: %w", err) + } + updated = normalizeDependabotIgnoreEntries(updated, managedPatterns) + + if bytes.Equal(original, updated) { + return nil } + if err := os.WriteFile(path, updated, constants.FilePermPublic); err != nil { + return fmt.Errorf("failed to write dependabot.yml: %w", err) + } + return nil +} +func readDependabotUpdatesRoot(path string) (map[string]any, []any, []byte, error) { + original, err := os.ReadFile(path) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to read dependabot.yml: %w", err) + } var root map[string]any if err := yaml.Unmarshal(original, &root); err != nil { - return fmt.Errorf("failed to parse dependabot.yml: %w", err) + return nil, nil, nil, fmt.Errorf("failed to parse dependabot.yml: %w", err) } - updatesAny, ok := root["updates"] if !ok { - return nil + return root, nil, original, nil } updates, ok := dependabotToAnySlice(updatesAny) if !ok { - return nil + return root, nil, original, nil } + return root, updates, original, nil +} - managedPatterns := []string{c.effectiveActionsRepo()} +func reconcileManagedDependabotUpdateIgnores(updates []any, managedPatterns []string, managedPatternsWithComment map[string]struct{}) bool { changed := false - originalStr := string(original) - managedPatternsWithComment := managedPatternsWithInlineComment(originalStr, managedPatterns) - for i, updateAny := range updates { updateMap, ok := dependabotToStringAnyMap(updateAny) if !ok { continue } - ecosystem, _ := updateMap["package-ecosystem"].(string) if ecosystem != "github-actions" { continue } - - ignoreAny, hasIgnore := updateMap["ignore"] - if !hasIgnore || isYAMLNullOrEmptyScalar(ignoreAny) { - updateMap["ignore"] = []any{} - ignoreAny = updateMap["ignore"] + if reconcileManagedDependabotIgnoreEntry(updateMap, managedPatterns, managedPatternsWithComment) { changed = true } + updates[i] = updateMap + } + return changed +} - ignoreEntries, ok := dependabotToAnySlice(ignoreAny) - if !ok { +func reconcileManagedDependabotIgnoreEntry(updateMap map[string]any, managedPatterns []string, managedPatternsWithComment map[string]struct{}) bool { + ignoreEntries, changed, ok := dependabotIgnoreEntries(updateMap) + if !ok { + return changed + } + managedPresent := managedDependabotIgnorePresence(ignoreEntries, managedPatterns, managedPatternsWithComment, &changed) + for _, pattern := range managedPatterns { + if setutil.Contains(managedPresent, pattern) { continue } + ignoreEntries = append(ignoreEntries, map[string]any{"dependency-name": pattern}) + changed = true + } + updateMap["ignore"] = ignoreEntries + return changed +} - managedPresent := make(map[string]struct { - }, len(managedPatterns)) - for _, ignoreEntryAny := range ignoreEntries { - ignoreEntryMap, ok := dependabotToStringAnyMap(ignoreEntryAny) - if !ok { - continue - } - dependencyName, _ := ignoreEntryMap["dependency-name"].(string) - if dependencyName == "" { - continue - } +func dependabotIgnoreEntries(updateMap map[string]any) ([]any, bool, bool) { + ignoreAny, hasIgnore := updateMap["ignore"] + changed := false + if !hasIgnore || isYAMLNullOrEmptyScalar(ignoreAny) { + updateMap["ignore"] = []any{} + ignoreAny = updateMap["ignore"] + changed = true + } + ignoreEntries, ok := dependabotToAnySlice(ignoreAny) + return ignoreEntries, changed, ok +} - for _, pattern := range managedPatterns { - if dependencyName == pattern { - managedPresent[pattern] = struct { - }{} - if !setutil.Contains(managedPatternsWithComment, pattern) { - changed = true - } - } - } +func managedDependabotIgnorePresence(ignoreEntries []any, managedPatterns []string, managedPatternsWithComment map[string]struct{}, changed *bool) map[string]struct{} { + managedPresent := make(map[string]struct{}, len(managedPatterns)) + for _, ignoreEntryAny := range ignoreEntries { + ignoreEntryMap, ok := dependabotToStringAnyMap(ignoreEntryAny) + if !ok { + continue + } + dependencyName, _ := ignoreEntryMap["dependency-name"].(string) + if dependencyName == "" { + continue } - for _, pattern := range managedPatterns { - if setutil.Contains(managedPresent, pattern) { - continue + if dependencyName == pattern { + managedPresent[pattern] = struct{}{} + if !setutil.Contains(managedPatternsWithComment, pattern) { + *changed = true + } } - ignoreEntries = append(ignoreEntries, map[string]any{"dependency-name": pattern}) - changed = true } - - updateMap["ignore"] = ignoreEntries - updates[i] = updateMap } - - if !changed { - return nil - } - - root["updates"] = updates - updated, err := yaml.Marshal(root) - if err != nil { - return fmt.Errorf("failed to encode dependabot.yml: %w", err) - } - updated = normalizeDependabotIgnoreEntries(updated, managedPatterns) - - if bytes.Equal(original, updated) { - return nil - } - if err := os.WriteFile(path, updated, constants.FilePermPublic); err != nil { - return fmt.Errorf("failed to write dependabot.yml: %w", err) - } - return nil + return managedPresent } // DependabotConfigPath resolves the repository-local Dependabot config path. @@ -751,45 +776,12 @@ func (c *Compiler) generateRequirementsTxt(path string, deps []PipDependency, fo dependabotLog.Printf("Generating requirements.txt at %s", path) // Build requirements map for merging - reqMap := make(map[string]string) - for _, dep := range deps { - if dep.Version != "" { - reqMap[dep.Name] = dep.Version - } else { - reqMap[dep.Name] = "" - } - } + reqMap := buildRequirementsMap(deps) // Check if requirements.txt already exists if _, err := os.Stat(path); err == nil { - // File exists - merge dependencies - dependabotLog.Print("Existing requirements.txt found, merging dependencies") - - existingData, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read existing requirements.txt: %w", err) - } - - // Parse existing requirements - lines := strings.SplitSeq(string(existingData), "\n") - for line := range lines { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - dep := parsePipPackage(line) - // Only add if not already in our new deps - if _, exists := reqMap[dep.Name]; !exists { - if dep.Version != "" { - reqMap[dep.Name] = dep.Version - } else { - reqMap[dep.Name] = "" - } - } - } - - if c.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Merging with existing requirements.txt")) + if err := c.mergeExistingRequirements(path, reqMap); err != nil { + return err } } else { dependabotLog.Print("Creating new requirements.txt") @@ -828,6 +820,37 @@ func (c *Compiler) generateRequirementsTxt(path string, deps []PipDependency, fo return nil } +func buildRequirementsMap(deps []PipDependency) map[string]string { + reqMap := make(map[string]string) + for _, dep := range deps { + reqMap[dep.Name] = dep.Version + } + return reqMap +} + +func (c *Compiler) mergeExistingRequirements(path string, reqMap map[string]string) error { + dependabotLog.Print("Existing requirements.txt found, merging dependencies") + existingData, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read existing requirements.txt: %w", err) + } + lines := strings.SplitSeq(string(existingData), "\n") + for line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + dep := parsePipPackage(line) + if _, exists := reqMap[dep.Name]; !exists { + reqMap[dep.Name] = dep.Version + } + } + if c.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Merging with existing requirements.txt")) + } + return nil +} + // collectGoDependencies collects all Go dependencies from workflow data func (c *Compiler) collectGoDependencies(workflowDataList []*WorkflowData) []GoDependency { dependabotLog.Print("Collecting Go dependencies from workflows") @@ -903,56 +926,13 @@ func extractGoFromCommands(commands string) []string { func (c *Compiler) generateGoMod(path string, deps []GoDependency, forceOverwrite bool) error { dependabotLog.Printf("Generating go.mod at %s", path) - // Build module content - var lines []string - - // Check if go.mod already exists - if _, err := os.Stat(path); err == nil { - // File exists - read and preserve module declaration - dependabotLog.Print("Existing go.mod found, merging dependencies") - - existingData, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read existing go.mod: %w", err) - } - - existingLines := strings.SplitSeq(string(existingData), "\n") - // Keep module declaration and go version - for line := range existingLines { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "module ") || strings.HasPrefix(trimmed, "go ") { - lines = append(lines, line) - } - } - - if c.verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Merging with existing go.mod")) - } - } else { - // New go.mod - dependabotLog.Print("Creating new go.mod") - lines = append(lines, "module github.com/github/gh-aw-workflows-deps") - lines = append(lines, "") - lines = append(lines, "go 1.21") + lines, err := c.buildGoModBaseLines(path) + if err != nil { + return err } // Add require section if we have dependencies - if len(deps) > 0 { - lines = append(lines, "") - lines = append(lines, "require (") - for _, dep := range deps { - version := dep.Version - if version == "latest" || version == "" { - // Skip dependencies without explicit versions - they should be added manually - // or resolved using 'go get' or 'go mod tidy'. Using v0.0.0 as a placeholder - // can cause issues with Go module resolution. - dependabotLog.Printf("Skipping %s: no version specified (use 'go get %s@latest' to resolve)", dep.Path, dep.Path) - continue - } - lines = append(lines, fmt.Sprintf("\t%s %s", dep.Path, version)) - } - lines = append(lines, ")") - } + lines = appendGoModRequireSection(lines, deps) content := strings.Join(lines, "\n") + "\n" @@ -972,3 +952,42 @@ func (c *Compiler) generateGoMod(path string, deps []GoDependency, forceOverwrit return nil } + +func (c *Compiler) buildGoModBaseLines(path string) ([]string, error) { + if _, err := os.Stat(path); err == nil { + dependabotLog.Print("Existing go.mod found, merging dependencies") + existingData, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read existing go.mod: %w", err) + } + var lines []string + for line := range strings.SplitSeq(string(existingData), "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "module ") || strings.HasPrefix(trimmed, "go ") { + lines = append(lines, line) + } + } + if c.verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Merging with existing go.mod")) + } + return lines, nil + } + dependabotLog.Print("Creating new go.mod") + return []string{"module github.com/github/gh-aw-workflows-deps", "", "go 1.21"}, nil +} + +func appendGoModRequireSection(lines []string, deps []GoDependency) []string { + if len(deps) == 0 { + return lines + } + lines = append(lines, "", "require (") + for _, dep := range deps { + version := dep.Version + if version == "latest" || version == "" { + dependabotLog.Printf("Skipping %s: no version specified (use 'go get %s@latest' to resolve)", dep.Path, dep.Path) + continue + } + lines = append(lines, fmt.Sprintf("\t%s %s", dep.Path, version)) + } + return append(lines, ")") +} diff --git a/pkg/workflow/dispatch_repository.go b/pkg/workflow/dispatch_repository.go index 73bc6d1ad63..3f55ed868cc 100644 --- a/pkg/workflow/dispatch_repository.go +++ b/pkg/workflow/dispatch_repository.go @@ -29,98 +29,101 @@ type DispatchRepositoryConfig struct { // parseDispatchRepositoryConfig parses dispatch-repository configuration from the safe-outputs map. func (c *Compiler) parseDispatchRepositoryConfig(outputMap map[string]any) *DispatchRepositoryConfig { dispatchRepositoryLog.Print("Parsing dispatch-repository configuration") - - var configData any - var exists bool - - // dispatch-repository is canonical; keep underscore form as a backward-compatible alias. - if configData, exists = outputMap["dispatch-repository"]; !exists { - if configData, exists = outputMap["dispatch_repository"]; !exists { - return nil - } - dispatchRepositoryLog.Print("WARNING: safe-outputs.dispatch_repository is deprecated; rename to dispatch-repository or run `gh aw fix`") + configData, exists := dispatchRepositoryConfigData(outputMap) + if !exists { + return nil } - configMap, ok := configData.(map[string]any) if !ok { dispatchRepositoryLog.Print("dispatch-repository value is not a map, skipping") return nil } - dispatchRepositoryLog.Printf("Parsing dispatch-repository tools map with %d entries", len(configMap)) - - dispatchRepoConfig := &DispatchRepositoryConfig{ - Tools: make(map[string]*DispatchRepositoryToolConfig), - } - + dispatchRepoConfig := &DispatchRepositoryConfig{Tools: make(map[string]*DispatchRepositoryToolConfig)} for toolKey, toolValue := range configMap { - toolMap, ok := toolValue.(map[string]any) - if !ok { - dispatchRepositoryLog.Printf("Skipping tool %q: value is not a map", toolKey) - continue - } - - tool := &DispatchRepositoryToolConfig{} - - if desc, ok := toolMap["description"].(string); ok { - tool.Description = desc - } - - if workflow, ok := toolMap["workflow"].(string); ok { - tool.Workflow = workflow - } - - if eventType, ok := toolMap["event_type"].(string); ok { - tool.EventType = eventType - } - - if repo, ok := toolMap["repository"].(string); ok { - tool.Repository = repo - } - - // Parse allowed_repositories (list of repos) - if allowedReposRaw, exists := toolMap["allowed_repositories"]; exists { - if allowedReposList, ok := allowedReposRaw.([]any); ok { - for _, r := range allowedReposList { - if rStr, ok := r.(string); ok { - tool.AllowedRepositories = append(tool.AllowedRepositories, rStr) - } - } - } + tool := c.parseDispatchRepositoryTool(toolKey, toolValue) + if tool != nil { + dispatchRepoConfig.Tools[toolKey] = tool } + } + if len(dispatchRepoConfig.Tools) == 0 { + dispatchRepositoryLog.Print("No valid tools found in dispatch-repository config") + return nil + } + return dispatchRepoConfig +} - // Parse inputs (map of input definitions) - if inputsRaw, exists := toolMap["inputs"]; exists { - if inputsMap, ok := inputsRaw.(map[string]any); ok { - tool.Inputs = inputsMap - } - } +func dispatchRepositoryConfigData(outputMap map[string]any) (any, bool) { + configData, exists := outputMap["dispatch-repository"] + if exists { + return configData, true + } + configData, exists = outputMap["dispatch_repository"] + if exists { + dispatchRepositoryLog.Print("WARNING: safe-outputs.dispatch_repository is deprecated; rename to dispatch-repository or run `gh aw fix`") + } + return configData, exists +} - // Parse max (templatable int, default 1) - var baseCfg BaseSafeOutputConfig - c.parseBaseSafeOutputConfig(toolMap, &baseCfg, 1) - tool.Max = baseCfg.Max - tool.GitHubToken = baseCfg.GitHubToken - tool.Staged = baseCfg.Staged - - // Cap max at 50 - if maxVal := templatableIntValue(tool.Max); maxVal > 50 { - dispatchRepositoryLog.Printf("Tool %q: max value %d exceeds limit, capping at 50", toolKey, maxVal) - tool.Max = defaultIntStr(50) +func (c *Compiler) parseDispatchRepositoryTool(toolKey string, toolValue any) *DispatchRepositoryToolConfig { + toolMap, ok := toolValue.(map[string]any) + if !ok { + dispatchRepositoryLog.Printf("Skipping tool %q: value is not a map", toolKey) + return nil + } + tool := &DispatchRepositoryToolConfig{} + if desc, ok := toolMap["description"].(string); ok { + tool.Description = desc + } + if workflow, ok := toolMap["workflow"].(string); ok { + tool.Workflow = workflow + } + if eventType, ok := toolMap["event_type"].(string); ok { + tool.EventType = eventType + } + if repo, ok := toolMap["repository"].(string); ok { + tool.Repository = repo + } + tool.AllowedRepositories = parseDispatchRepositoryAllowedRepos(toolMap) + if inputsRaw, exists := toolMap["inputs"]; exists { + if inputsMap, ok := inputsRaw.(map[string]any); ok { + tool.Inputs = inputsMap } - - dispatchRepositoryLog.Printf("Parsed dispatch-repository tool %q: workflow=%s, event_type=%s, max=%v", - toolKey, tool.Workflow, tool.EventType, tool.Max) - - dispatchRepoConfig.Tools[toolKey] = tool } + c.applyDispatchRepositoryBaseConfig(toolKey, toolMap, tool) + dispatchRepositoryLog.Printf("Parsed dispatch-repository tool %q: workflow=%s, event_type=%s, max=%v", + toolKey, tool.Workflow, tool.EventType, tool.Max) + return tool +} - if len(dispatchRepoConfig.Tools) == 0 { - dispatchRepositoryLog.Print("No valid tools found in dispatch-repository config") +func parseDispatchRepositoryAllowedRepos(toolMap map[string]any) []string { + allowedReposRaw, exists := toolMap["allowed_repositories"] + if !exists { return nil } + allowedReposList, ok := allowedReposRaw.([]any) + if !ok { + return nil + } + var repos []string + for _, r := range allowedReposList { + if rStr, ok := r.(string); ok { + repos = append(repos, rStr) + } + } + return repos +} - return dispatchRepoConfig +func (c *Compiler) applyDispatchRepositoryBaseConfig(toolKey string, toolMap map[string]any, tool *DispatchRepositoryToolConfig) { + var baseCfg BaseSafeOutputConfig + c.parseBaseSafeOutputConfig(toolMap, &baseCfg, 1) + tool.Max = baseCfg.Max + tool.GitHubToken = baseCfg.GitHubToken + tool.Staged = baseCfg.Staged + if maxVal := templatableIntValue(tool.Max); maxVal > 50 { + dispatchRepositoryLog.Printf("Tool %q: max value %d exceeds limit, capping at 50", toolKey, maxVal) + tool.Max = defaultIntStr(50) + } } // generateDispatchRepositoryTool generates an MCP tool definition for a specific dispatch-repository tool. diff --git a/pkg/workflow/dispatch_repository_validation.go b/pkg/workflow/dispatch_repository_validation.go index 63b9d9ff9d1..21ec8c1e62d 100644 --- a/pkg/workflow/dispatch_repository_validation.go +++ b/pkg/workflow/dispatch_repository_validation.go @@ -35,65 +35,13 @@ func (c *Compiler) validateDispatchRepository(data *WorkflowData, workflowPath s for toolKey, tool := range config.Tools { dispatchRepositoryValidationLog.Printf("Validating dispatch_repository tool: %s", toolKey) - - // Validate workflow field is present - if strings.TrimSpace(tool.Workflow) == "" { - workflowErr := fmt.Errorf("dispatch_repository: tool %q must specify a 'workflow' field (target workflow name for traceability)\n\nExample:\n dispatch_repository:\n %s:\n workflow: ci.yml\n event_type: ci_trigger\n repository: org/target-repo", toolKey, toolKey) - if returnErr := collector.Add(workflowErr); returnErr != nil { - return returnErr - } - continue - } - - // Validate event_type field is present - if strings.TrimSpace(tool.EventType) == "" { - eventTypeErr := fmt.Errorf("dispatch_repository: tool %q must specify an 'event_type' field\n\nExample:\n dispatch_repository:\n %s:\n workflow: %s\n event_type: my_event\n repository: org/target-repo", toolKey, toolKey, tool.Workflow) - if returnErr := collector.Add(eventTypeErr); returnErr != nil { - return returnErr - } - continue + complete, err := validateDispatchRepositoryTool(toolKey, tool, collector) + if err != nil { + return err } - - // Validate that at least one repository target is specified - hasRepository := strings.TrimSpace(tool.Repository) != "" - hasAllowedRepos := len(tool.AllowedRepositories) > 0 - - if !hasRepository && !hasAllowedRepos { - repoErr := fmt.Errorf("dispatch_repository: tool %q must specify either 'repository' or 'allowed_repositories'\n\nExample with single repository:\n dispatch_repository:\n %s:\n workflow: %s\n event_type: %s\n repository: org/target-repo\n\nExample with multiple repositories:\n dispatch_repository:\n %s:\n workflow: %s\n event_type: %s\n allowed_repositories:\n - org/repo1\n - org/repo2", toolKey, toolKey, tool.Workflow, tool.EventType, toolKey, tool.Workflow, tool.EventType) - if returnErr := collector.Add(repoErr); returnErr != nil { - return returnErr - } - continue + if complete { + dispatchRepositoryValidationLog.Printf("Tool %q validation passed", toolKey) } - - // Validate single repository format (skip if it looks like a GitHub Actions expression) - if hasRepository && !hasExpressionMarker(tool.Repository) { - if !repoSlugPattern.MatchString(tool.Repository) { - repoFmtErr := fmt.Errorf("dispatch_repository: tool %q has invalid 'repository' format %q (expected 'owner/repo')", toolKey, tool.Repository) - if returnErr := collector.Add(repoFmtErr); returnErr != nil { - return returnErr - } - } - } - - // Validate allowed_repositories format - for _, repo := range tool.AllowedRepositories { - if hasExpressionMarker(repo) { - continue - } - // Allow glob patterns like "org/*" - if strings.Contains(repo, "*") { - continue - } - if !repoSlugPattern.MatchString(repo) { - allowedRepoErr := fmt.Errorf("dispatch_repository: tool %q has invalid repository %q in 'allowed_repositories' (expected 'owner/repo' format)", toolKey, repo) - if returnErr := collector.Add(allowedRepoErr); returnErr != nil { - return returnErr - } - } - } - - dispatchRepositoryValidationLog.Printf("Tool %q validation passed", toolKey) } dispatchRepositoryValidationLog.Printf("dispatch_repository validation completed: error_count=%d, total_tools=%d", @@ -101,3 +49,45 @@ func (c *Compiler) validateDispatchRepository(data *WorkflowData, workflowPath s return collector.FormattedError("dispatch_repository") } + +func validateDispatchRepositoryTool(toolKey string, tool *DispatchRepositoryToolConfig, collector *ErrorCollector) (bool, error) { + if strings.TrimSpace(tool.Workflow) == "" { + workflowErr := fmt.Errorf("dispatch_repository: tool %q must specify a 'workflow' field (target workflow name for traceability)\n\nExample:\n dispatch_repository:\n %s:\n workflow: ci.yml\n event_type: ci_trigger\n repository: org/target-repo", toolKey, toolKey) + return false, collector.Add(workflowErr) + } + if strings.TrimSpace(tool.EventType) == "" { + eventTypeErr := fmt.Errorf("dispatch_repository: tool %q must specify an 'event_type' field\n\nExample:\n dispatch_repository:\n %s:\n workflow: %s\n event_type: my_event\n repository: org/target-repo", toolKey, toolKey, tool.Workflow) + return false, collector.Add(eventTypeErr) + } + hasRepository := strings.TrimSpace(tool.Repository) != "" + hasAllowedRepos := len(tool.AllowedRepositories) > 0 + if !hasRepository && !hasAllowedRepos { + repoErr := fmt.Errorf("dispatch_repository: tool %q must specify either 'repository' or 'allowed_repositories'\n\nExample with single repository:\n dispatch_repository:\n %s:\n workflow: %s\n event_type: %s\n repository: org/target-repo\n\nExample with multiple repositories:\n dispatch_repository:\n %s:\n workflow: %s\n event_type: %s\n allowed_repositories:\n - org/repo1\n - org/repo2", toolKey, toolKey, tool.Workflow, tool.EventType, toolKey, tool.Workflow, tool.EventType) + return false, collector.Add(repoErr) + } + if err := validateDispatchRepositoryTarget(toolKey, tool.Repository, collector); err != nil { + return true, err + } + return true, validateDispatchAllowedRepositories(toolKey, tool.AllowedRepositories, collector) +} + +func validateDispatchRepositoryTarget(toolKey string, repo string, collector *ErrorCollector) error { + if strings.TrimSpace(repo) == "" || hasExpressionMarker(repo) || repoSlugPattern.MatchString(repo) { + return nil + } + repoFmtErr := fmt.Errorf("dispatch_repository: tool %q has invalid 'repository' format %q (expected 'owner/repo')", toolKey, repo) + return collector.Add(repoFmtErr) +} + +func validateDispatchAllowedRepositories(toolKey string, repos []string, collector *ErrorCollector) error { + for _, repo := range repos { + if hasExpressionMarker(repo) || strings.Contains(repo, "*") || repoSlugPattern.MatchString(repo) { + continue + } + allowedRepoErr := fmt.Errorf("dispatch_repository: tool %q has invalid repository %q in 'allowed_repositories' (expected 'owner/repo' format)", toolKey, repo) + if returnErr := collector.Add(allowedRepoErr); returnErr != nil { + return returnErr + } + } + return nil +} diff --git a/pkg/workflow/dispatch_workflow_validation.go b/pkg/workflow/dispatch_workflow_validation.go index b8a641cb138..1ff5ac4a826 100644 --- a/pkg/workflow/dispatch_workflow_validation.go +++ b/pkg/workflow/dispatch_workflow_validation.go @@ -40,111 +40,88 @@ func (c *Compiler) validateDispatchWorkflow(data *WorkflowData, workflowPath str collector := NewErrorCollector(c.failFast) for _, workflowName := range config.Workflows { - dispatchWorkflowValidationLog.Printf("Validating workflow: %s", workflowName) - if workflowName == currentWorkflowName { - selfRefErr := fmt.Errorf("dispatch-workflow: self-reference not allowed (workflow '%s' cannot dispatch itself)\n\nA workflow cannot trigger itself to prevent infinite loops.\nIf you need recurring execution, use a schedule trigger or workflow_dispatch instead", workflowName) - if returnErr := collector.Add(selfRefErr); returnErr != nil { - return returnErr - } - continue - } - fileResult, err := findWorkflowFile(workflowName, workflowPath) - if err != nil { - findErr := fmt.Errorf("dispatch-workflow: error finding workflow '%s': %w", workflowName, err) - if returnErr := collector.Add(findErr); returnErr != nil { - return returnErr - } - continue - } - - if !fileResult.mdExists && !fileResult.lockExists && !fileResult.ymlExists { - currentDir := filepath.Dir(workflowPath) - githubDir := filepath.Dir(currentDir) - repoRoot := filepath.Dir(githubDir) - workflowsDir := filepath.Join(repoRoot, constants.GetWorkflowDir()) - notFoundErr := fmt.Errorf("dispatch-workflow: workflow '%s' not found in %s\n\nChecked for: %s.md, %s.lock.yml, %s.yml\n\nTo fix:\n1. Verify the workflow file exists in %s/\n2. Ensure the filename matches exactly (case-sensitive)\n3. Use the filename without extension in your configuration", workflowName, workflowsDir, workflowName, workflowName, workflowName, workflowsDir) - if returnErr := collector.Add(notFoundErr); returnErr != nil { - return returnErr - } - continue + if err := validateDispatchWorkflowTarget(workflowName, currentWorkflowName, workflowPath, collector); err != nil { + return err } + } - var workflowContent []byte // #nosec G304 -- All file paths are validated via isPathWithinDir before use - var workflowFile string - var readErr error - - if fileResult.lockExists { - workflowFile = fileResult.lockPath - workflowContent, readErr = os.ReadFile(fileResult.lockPath) // #nosec G304 -- Path is validated via isPathWithinDir in findWorkflowFile - if readErr != nil { - fileReadErr := fmt.Errorf("dispatch-workflow: failed to read workflow file %s: %w", fileResult.lockPath, readErr) - if returnErr := collector.Add(fileReadErr); returnErr != nil { - return returnErr - } - continue - } - } else if fileResult.ymlExists { - workflowFile = fileResult.ymlPath - workflowContent, readErr = os.ReadFile(fileResult.ymlPath) // #nosec G304 -- Path is validated via isPathWithinDir in findWorkflowFile - if readErr != nil { - fileReadErr := fmt.Errorf("dispatch-workflow: failed to read workflow file %s: %w", fileResult.ymlPath, readErr) - if returnErr := collector.Add(fileReadErr); returnErr != nil { - return returnErr - } - continue - } - } else { - mdHasDispatch, checkErr := mdHasWorkflowDispatch(fileResult.mdPath) - if checkErr != nil { - readErr := fmt.Errorf("dispatch-workflow: failed to read workflow source %s: %w", fileResult.mdPath, checkErr) - if returnErr := collector.Add(readErr); returnErr != nil { - return returnErr - } - continue - } - if !mdHasDispatch { - dispatchErr := fmt.Errorf("dispatch-workflow: workflow '%s' does not support workflow_dispatch trigger (must include 'workflow_dispatch' in the 'on' section)", workflowName) - if returnErr := collector.Add(dispatchErr); returnErr != nil { - return returnErr - } - continue - } - dispatchWorkflowValidationLog.Printf("Workflow '%s' is valid for dispatch (found .md source at %s with workflow_dispatch trigger)", workflowName, fileResult.mdPath) - continue - } + dispatchWorkflowValidationLog.Printf("Dispatch workflow validation completed: error_count=%d, total_workflows=%d", collector.Count(), len(config.Workflows)) - var workflow map[string]any - if err := yaml.Unmarshal(workflowContent, &workflow); err != nil { - parseErr := fmt.Errorf("dispatch-workflow: failed to parse workflow file %s: %w", workflowFile, err) - if returnErr := collector.Add(parseErr); returnErr != nil { - return returnErr - } - continue - } + return collector.FormattedError("dispatch-workflow") +} - onSection, hasOn := workflow["on"] - if !hasOn { - onSectionErr := fmt.Errorf("dispatch-workflow: workflow '%s' does not have an 'on' trigger section", workflowName) - if returnErr := collector.Add(onSectionErr); returnErr != nil { - return returnErr - } - continue - } +func validateDispatchWorkflowTarget(workflowName, currentWorkflowName, workflowPath string, collector *ErrorCollector) error { + dispatchWorkflowValidationLog.Printf("Validating workflow: %s", workflowName) + if workflowName == currentWorkflowName { + selfRefErr := fmt.Errorf("dispatch-workflow: self-reference not allowed (workflow '%s' cannot dispatch itself)\n\nA workflow cannot trigger itself to prevent infinite loops.\nIf you need recurring execution, use a schedule trigger or workflow_dispatch instead", workflowName) + return collector.Add(selfRefErr) + } + fileResult, err := findWorkflowFile(workflowName, workflowPath) + if err != nil { + return collector.Add(fmt.Errorf("dispatch-workflow: error finding workflow '%s': %w", workflowName, err)) + } + if !fileResult.mdExists && !fileResult.lockExists && !fileResult.ymlExists { + return collector.Add(dispatchWorkflowNotFoundError(workflowName, workflowPath)) + } + workflowContent, workflowFile, handled, err := readDispatchWorkflowFile(workflowName, fileResult) + if err != nil { + return collector.Add(err) + } + if handled { + return nil + } + return validateDispatchWorkflowContent(workflowName, workflowFile, workflowContent, collector) +} - if !containsWorkflowDispatch(onSection) { - dispatchErr := fmt.Errorf("dispatch-workflow: workflow '%s' does not support workflow_dispatch trigger (must include 'workflow_dispatch' in the 'on' section)", workflowName) - if returnErr := collector.Add(dispatchErr); returnErr != nil { - return returnErr - } - continue - } +func dispatchWorkflowNotFoundError(workflowName, workflowPath string) error { + currentDir := filepath.Dir(workflowPath) + githubDir := filepath.Dir(currentDir) + repoRoot := filepath.Dir(githubDir) + workflowsDir := filepath.Join(repoRoot, constants.GetWorkflowDir()) + return fmt.Errorf("dispatch-workflow: workflow '%s' not found in %s\n\nChecked for: %s.md, %s.lock.yml, %s.yml\n\nTo fix:\n1. Verify the workflow file exists in %s/\n2. Ensure the filename matches exactly (case-sensitive)\n3. Use the filename without extension in your configuration", workflowName, workflowsDir, workflowName, workflowName, workflowName, workflowsDir) +} - dispatchWorkflowValidationLog.Printf("Workflow '%s' is valid for dispatch (found in %s)", workflowName, workflowFile) +func readDispatchWorkflowFile(workflowName string, fileResult *findWorkflowFileResult) ([]byte, string, bool, error) { + if fileResult.lockExists { + content, err := os.ReadFile(fileResult.lockPath) // #nosec G304 -- Path is validated via isPathWithinDir in findWorkflowFile + return content, fileResult.lockPath, false, dispatchWorkflowReadError(fileResult.lockPath, err) } + if fileResult.ymlExists { + content, err := os.ReadFile(fileResult.ymlPath) // #nosec G304 -- Path is validated via isPathWithinDir in findWorkflowFile + return content, fileResult.ymlPath, false, dispatchWorkflowReadError(fileResult.ymlPath, err) + } + mdHasDispatch, checkErr := mdHasWorkflowDispatch(fileResult.mdPath) + if checkErr != nil { + return nil, "", true, fmt.Errorf("dispatch-workflow: failed to read workflow source %s: %w", fileResult.mdPath, checkErr) + } + if !mdHasDispatch { + return nil, "", true, fmt.Errorf("dispatch-workflow: workflow '%s' does not support workflow_dispatch trigger (must include 'workflow_dispatch' in the 'on' section)", workflowName) + } + dispatchWorkflowValidationLog.Printf("Workflow '%s' is valid for dispatch (found .md source at %s with workflow_dispatch trigger)", workflowName, fileResult.mdPath) + return nil, "", true, nil +} - dispatchWorkflowValidationLog.Printf("Dispatch workflow validation completed: error_count=%d, total_workflows=%d", collector.Count(), len(config.Workflows)) +func dispatchWorkflowReadError(path string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("dispatch-workflow: failed to read workflow file %s: %w", path, err) +} - return collector.FormattedError("dispatch-workflow") +func validateDispatchWorkflowContent(workflowName, workflowFile string, workflowContent []byte, collector *ErrorCollector) error { + var workflow map[string]any + if err := yaml.Unmarshal(workflowContent, &workflow); err != nil { + return collector.Add(fmt.Errorf("dispatch-workflow: failed to parse workflow file %s: %w", workflowFile, err)) + } + onSection, hasOn := workflow["on"] + if !hasOn { + return collector.Add(fmt.Errorf("dispatch-workflow: workflow '%s' does not have an 'on' trigger section", workflowName)) + } + if !containsWorkflowDispatch(onSection) { + return collector.Add(fmt.Errorf("dispatch-workflow: workflow '%s' does not support workflow_dispatch trigger (must include 'workflow_dispatch' in the 'on' section)", workflowName)) + } + dispatchWorkflowValidationLog.Printf("Workflow '%s' is valid for dispatch (found in %s)", workflowName, workflowFile) + return nil } func (c *Compiler) shouldSkipLocalDispatchWorkflowValidation(targetRepoSlug string) bool { diff --git a/pkg/workflow/docker.go b/pkg/workflow/docker.go index ea006884905..6d83da41871 100644 --- a/pkg/workflow/docker.go +++ b/pkg/workflow/docker.go @@ -7,221 +7,143 @@ import ( "github.com/github/gh-aw/pkg/constants" "github.com/github/gh-aw/pkg/logger" + "github.com/github/gh-aw/pkg/parser" "github.com/github/gh-aw/pkg/setutil" ) var dockerLog = logger.New("workflow:docker") +type dockerImageCollector struct { + images []string + seen map[string]struct{} +} + // collectDockerImages collects all Docker images used in MCP configurations. // When workflowData.ActionCache contains container pins, the returned slice uses -// the pinned references (image:tag@sha256:…) instead of the bare tags, ensuring -// deterministic and supply-chain-safe image pulls. +// the pinned references (image:tag@sha256:…) instead of the bare tags. func collectDockerImages(tools map[string]any, workflowData *WorkflowData, actionMode ActionMode) []string { - var images []string - imageSet := make(map[string]struct{}) // Use a set to avoid duplicates + collector := dockerImageCollector{seen: make(map[string]struct{})} + collector.collectBuiltInToolImages(tools, workflowData, actionMode) + collector.collectFirewallImages(workflowData) + collector.collectSandboxMCPImage(workflowData) + collector.collectCustomMCPImages(tools) + + sort.Strings(collector.images) + dockerLog.Printf("Collected %d Docker images from tools", len(collector.images)) + pinnedImages, imagePins := applyContainerPins(collector.images, workflowData) + if workflowData != nil { + workflowData.DockerImages = mergeDockerImages(workflowData.DockerImages, pinnedImages) + workflowData.DockerImagePins = mergeDockerImagePins(workflowData.DockerImagePins, imagePins) + } + return pinnedImages +} - // Check for GitHub tool (uses Docker image) - if rawGithubTool, hasGitHub := tools["github"]; hasGitHub { - // Only proceed when the value is an actual config map; a boolean false - // means the tool is explicitly disabled. - if githubTool, ok := rawGithubTool.(map[string]any); ok { - githubType := getGitHubType(githubTool) - // Only add if using local (Docker) mode - if githubType == GitHubMCPModeLocal { - githubDockerImageVersion := getGitHubDockerImageVersion(githubTool) - image := "ghcr.io/github/github-mcp-server:" + githubDockerImageVersion - if !setutil.Contains(imageSet, image) { - images = append(images, image) - imageSet[image] = struct { - }{} - } - } - } +func (c *dockerImageCollector) add(image string) bool { + if setutil.Contains(c.seen, image) { + return false } + c.images = append(c.images, image) + c.seen[image] = struct{}{} + return true +} - // Check for Playwright tool (uses Docker image - no version tag, only one image) - // Only in MCP mode; CLI mode installs @playwright/cli via npm instead. - if _, hasPlaywright := tools["playwright"]; hasPlaywright { - if !isPlaywrightCLIMode(tools) { - image := "mcr.microsoft.com/playwright/mcp" - if !setutil.Contains(imageSet, image) { - images = append(images, image) - imageSet[image] = struct { - }{} - } +func (c *dockerImageCollector) collectBuiltInToolImages(tools map[string]any, workflowData *WorkflowData, actionMode ActionMode) { + if rawGithubTool, hasGitHub := tools["github"]; hasGitHub { + if githubTool, ok := rawGithubTool.(map[string]any); ok && getGitHubType(githubTool) == GitHubMCPModeLocal { + c.add("ghcr.io/github/github-mcp-server:" + getGitHubDockerImageVersion(githubTool)) } } - - // Check for safe-outputs MCP server. - // Safe outputs run in the published gh-aw node container and must be part of - // the default predownload set and lock-file manifest whenever enabled. + if _, hasPlaywright := tools["playwright"]; hasPlaywright && !isPlaywrightCLIMode(tools) { + c.add("mcr.microsoft.com/playwright/mcp") + } if workflowData != nil && HasSafeOutputsEnabled(workflowData.SafeOutputs) { - image := constants.DefaultGhAwNodeImage - if !setutil.Contains(imageSet, image) { - images = append(images, image) - imageSet[image] = struct { - }{} - dockerLog.Printf("Added safe-outputs MCP server container: %s", image) + if c.add(constants.DefaultGhAwNodeImage) { + dockerLog.Printf("Added safe-outputs MCP server container: %s", constants.DefaultGhAwNodeImage) } } - - // Check for agentic-workflows tool - // In dev mode, the image is built locally in the workflow, so don't add to pull list - // In release/script mode, use alpine:latest which needs to be pulled - if _, hasAgenticWorkflows := tools["agentic-workflows"]; hasAgenticWorkflows { - if !actionMode.IsDev() { - // Release/script mode: Use alpine:latest (needs to be pulled) - image := constants.DefaultAlpineImage - if !setutil.Contains(imageSet, image) { - images = append(images, image) - imageSet[image] = struct { - }{} - dockerLog.Printf("Added agentic-workflows MCP server container: %s", image) - } + if _, hasAgenticWorkflows := tools["agentic-workflows"]; hasAgenticWorkflows && !actionMode.IsDev() { + if c.add(constants.DefaultAlpineImage) { + dockerLog.Printf("Added agentic-workflows MCP server container: %s", constants.DefaultAlpineImage) } - // Dev mode: localhost/gh-aw:dev is built locally, not pulled } +} - // Collect AWF (firewall) container images when firewall is enabled - // AWF uses three containers: squid (proxy), agent, and api-proxy (for engines with LLM gateway support) - if isFirewallEnabled(workflowData) { - // Get the firewall version for image tags - firewallConfig := getFirewallConfig(workflowData) - awfImageTag := getAWFImageTag(firewallConfig) +func (c *dockerImageCollector) collectFirewallImages(workflowData *WorkflowData) { + if !isFirewallEnabled(workflowData) { + return + } + firewallConfig := getFirewallConfig(workflowData) + awfImageTag := getAWFImageTag(firewallConfig) + c.addLoggedAWFImage("squid", awfImageTag, "Added AWF squid (proxy) container: %s") + c.addLoggedAWFImage("agent", awfImageTag, "Added AWF agent container: %s") + if workflowData != nil && workflowData.AI != "" { + c.addLoggedAWFImage("api-proxy", awfImageTag, "Added AWF api-proxy sidecar container: %s") + } + if isCliProxyNeeded(workflowData) { + c.addLoggedAWFImage("cli-proxy", awfImageTag, "Added AWF cli-proxy sidecar container: %s") + } + if isArcDindTopology(workflowData) { + c.addLoggedAWFImage("build-tools", awfImageTag, "Added AWF build-tools sysroot container for arc-dind: %s") + } +} - // Add squid (proxy) container - squidImage := constants.DefaultFirewallRegistry + "/squid:" + awfImageTag - if !setutil.Contains(imageSet, squidImage) { - images = append(images, squidImage) - imageSet[squidImage] = struct { - }{} - dockerLog.Printf("Added AWF squid (proxy) container: %s", squidImage) - } +func (c *dockerImageCollector) addLoggedAWFImage(name, tag, logFormat string) { + image := constants.DefaultFirewallRegistry + "/" + name + ":" + tag + if c.add(image) { + dockerLog.Printf(logFormat, image) + } +} - // Add default agent container - agentImage := constants.DefaultFirewallRegistry + "/agent:" + awfImageTag - if !setutil.Contains(imageSet, agentImage) { - images = append(images, agentImage) - imageSet[agentImage] = struct { - }{} - dockerLog.Printf("Added AWF agent container: %s", agentImage) - } +func (c *dockerImageCollector) collectSandboxMCPImage(workflowData *WorkflowData) { + if workflowData == nil || workflowData.SandboxConfig == nil { + return + } + sandboxDisabled := workflowData.SandboxConfig.Agent != nil && workflowData.SandboxConfig.Agent.Disabled + if sandboxDisabled { + dockerLog.Print("Sandbox disabled, skipping MCP gateway container image") + return + } + if workflowData.SandboxConfig.MCP == nil || workflowData.SandboxConfig.MCP.Container == "" { + return + } + mcpGateway := workflowData.SandboxConfig.MCP + image := mcpGateway.Container + if mcpGateway.Version != "" { + image += ":" + mcpGateway.Version + } else { + image += ":" + string(constants.DefaultMCPGatewayVersion) + } + if c.add(image) { + dockerLog.Printf("Added sandbox.mcp container: %s", image) + } +} - // Add api-proxy sidecar container (required for all engines — LLM gateway is mandatory) - // The api-proxy holds LLM API keys securely and proxies requests through Squid - // Each engine uses its own dedicated port for communication - if workflowData != nil && workflowData.AI != "" { - apiProxyImage := constants.DefaultFirewallRegistry + "/api-proxy:" + awfImageTag - if !setutil.Contains(imageSet, apiProxyImage) { - images = append(images, apiProxyImage) - imageSet[apiProxyImage] = struct { - }{} - dockerLog.Printf("Added AWF api-proxy sidecar container: %s", apiProxyImage) - } +func (c *dockerImageCollector) collectCustomMCPImages(tools map[string]any) { + for toolName, toolValue := range tools { + mcpConfig, ok := toolValue.(map[string]any) + if !ok { + continue } - - // Add cli-proxy sidecar container when the cli-proxy is needed. - // Without this, --skip-pull causes AWF to fail because the cli-proxy image was never pulled. - if isCliProxyNeeded(workflowData) { - cliProxyImage := constants.DefaultFirewallRegistry + "/cli-proxy:" + awfImageTag - if !setutil.Contains(imageSet, cliProxyImage) { - images = append(images, cliProxyImage) - imageSet[cliProxyImage] = struct { - }{} - dockerLog.Printf("Added AWF cli-proxy sidecar container: %s", cliProxyImage) - } + if hasMcp, _ := hasMCPConfig(mcpConfig); !hasMcp { + continue } - - // Add build-tools sysroot image for ARC/DinD topology. - // AWF uses this as an init container to populate the sysroot volume with - // system binaries (gcc, make, libraries) that are invisible on the DinD daemon's FS. - if isArcDindTopology(workflowData) { - buildToolsImage := constants.DefaultFirewallRegistry + "/build-tools:" + awfImageTag - if !setutil.Contains(imageSet, buildToolsImage) { - images = append(images, buildToolsImage) - imageSet[buildToolsImage] = struct { - }{} - dockerLog.Printf("Added AWF build-tools sysroot container for arc-dind: %s", buildToolsImage) - } + if mcpConf, err := getMCPConfig(mcpConfig, toolName); err == nil { + c.addCustomMCPImage(mcpConf) } } +} - // Collect sandbox.mcp container (MCP gateway) - // Skip if sandbox is disabled (sandbox: false) - if workflowData != nil && workflowData.SandboxConfig != nil { - // Check if sandbox is disabled - sandboxDisabled := workflowData.SandboxConfig.Agent != nil && workflowData.SandboxConfig.Agent.Disabled - - if !sandboxDisabled && workflowData.SandboxConfig.MCP != nil { - mcpGateway := workflowData.SandboxConfig.MCP - if mcpGateway.Container != "" { - image := mcpGateway.Container - if mcpGateway.Version != "" { - image += ":" + mcpGateway.Version - } else { - // Use default version if not specified (consistent with mcp_servers.go) - image += ":" + string(constants.DefaultMCPGatewayVersion) - } - if !setutil.Contains(imageSet, image) { - images = append(images, image) - imageSet[image] = struct { - }{} - dockerLog.Printf("Added sandbox.mcp container: %s", image) - } - } - } else if sandboxDisabled { - dockerLog.Print("Sandbox disabled, skipping MCP gateway container image") - } +func (c *dockerImageCollector) addCustomMCPImage(mcpConf *parser.RegistryMCPServerConfig) { + if mcpConf.Container != "" { + c.add(mcpConf.Container) + return } - - // Collect images from custom MCP tools with container configurations - for toolName, toolValue := range tools { - if mcpConfig, ok := toolValue.(map[string]any); ok { - if hasMcp, _ := hasMCPConfig(mcpConfig); hasMcp { - // Check if this tool uses a container - if mcpConf, err := getMCPConfig(mcpConfig, toolName); err == nil { - // Check for direct container field - if mcpConf.Container != "" { - image := mcpConf.Container - if !setutil.Contains(imageSet, image) { - images = append(images, image) - imageSet[image] = struct { - }{} - } - } else if mcpConf.Command == "docker" && len(mcpConf.Args) > 0 { - // Extract container image from docker args - // Args format: ["run", "--rm", "-i", ... , "container-image"] - // The container image is the last arg - image := mcpConf.Args[len(mcpConf.Args)-1] - // Skip if it's a docker flag (starts with -) - if !strings.HasPrefix(image, "-") && !setutil.Contains(imageSet, image) { - images = append(images, image) - imageSet[image] = struct { - }{} - } - } - } - } + if mcpConf.Command == "docker" && len(mcpConf.Args) > 0 { + image := mcpConf.Args[len(mcpConf.Args)-1] + if !strings.HasPrefix(image, "-") { + c.add(image) } } - - // Sort for stable output - sort.Strings(images) - dockerLog.Printf("Collected %d Docker images from tools", len(images)) - - // Apply digest pins from the action cache when available. - // Each pinned ref replaces the bare tag with "tag@sha256:…" so that the pull - // is bound to a specific immutable manifest and not just to a mutable tag. - pinnedImages, imagePins := applyContainerPins(images, workflowData) - - // Store pinned image refs and full pin info in WorkflowData so they can be - // included in the compiled lock file header and gh-aw-manifest for auditability. - if workflowData != nil { - workflowData.DockerImages = mergeDockerImages(workflowData.DockerImages, pinnedImages) - workflowData.DockerImagePins = mergeDockerImagePins(workflowData.DockerImagePins, imagePins) - } - - return pinnedImages } // applyContainerPins substitutes cached digest-pinned references for any image diff --git a/pkg/workflow/docker_validation.go b/pkg/workflow/docker_validation.go index c3009eb2d10..524e25a8f7d 100644 --- a/pkg/workflow/docker_validation.go +++ b/pkg/workflow/docker_validation.go @@ -113,38 +113,16 @@ func validateDockerImage(image string, verbose bool, requireDocker bool) error { return fmt.Errorf("container image name '%s' is invalid: names must not start with '-'", image) } - // Check if docker CLI is available on PATH. - // If Docker is not installed, skip validation silently — compile is a source - // transformation step and should not require Docker at authoring time. - // When requireDocker is true, return an error instead of skipping. - _, err := exec.LookPath("docker") + available, err := ensureDockerValidationAvailable(image, requireDocker) if err != nil { - if requireDocker { - return fmt.Errorf("docker not installed - could not validate container image '%s'. Install Docker or omit the --validate-images flag to skip container image validation", image) - } - dockerValidationLog.Print("Docker not installed, skipping container image validation") - return nil + return err } - - // Check if Docker daemon is actually running (cached check with short timeout). - // If the daemon is not running (common on CI runners like ubuntu-slim, or when - // Docker Desktop is stopped), skip validation silently instead of emitting a - // warning. Image accessibility is a runtime concern, not a compile-time one. - // When requireDocker is true, return an error instead of skipping. - if !isDockerDaemonRunning() { - if requireDocker { - return fmt.Errorf("docker daemon not running - could not validate container image '%s'. Start the Docker daemon or omit the --validate-images flag to skip container image validation", image) - } - dockerValidationLog.Print("Docker daemon not running, skipping container image validation") + if !available { return nil } // Try to inspect the image (will succeed if image exists locally) - cmd := exec.Command("docker", "image", "inspect", image) - _, err = cmd.CombinedOutput() - - if err == nil { - // Image exists locally + if dockerImageExistsLocally(image) { dockerValidationLog.Printf("Docker image found locally: %s", image) return nil } @@ -158,58 +136,19 @@ func validateDockerImage(image string, verbose bool, requireDocker bool) error { var lastOutput string for attempt := 1; attempt <= maxAttempts; attempt++ { - dockerValidationLog.Printf("Attempt %d of %d: Pulling image %s", attempt, maxAttempts, image) - - pullCmd := exec.Command("docker", "pull", image) - pullOutput, pullErr := pullCmd.CombinedOutput() - outputStr := strings.TrimSpace(string(pullOutput)) - - if pullErr == nil { - // Successfully pulled - dockerValidationLog.Printf("Successfully pulled image %s on attempt %d", image, attempt) + outputStr, pulled := pullDockerImage(image, attempt, maxAttempts) + if pulled { return nil } - lastOutput = outputStr - - // Check if the error is due to authentication issues for existing private repositories - // We need to distinguish between: - // 1. "repository does not exist" - should fail validation immediately - // 2. "authentication required" for existing repos - should pass (private repo) - if (strings.Contains(outputStr, "denied") || - strings.Contains(outputStr, "unauthorized") || - strings.Contains(outputStr, "authentication required")) && - !strings.Contains(outputStr, "does not exist") && - !strings.Contains(outputStr, "not found") { - // This is likely a private image that requires authentication - // Don't fail validation for private/authenticated images + if isPrivateAuthenticatedImage(outputStr) { dockerValidationLog.Printf("Image %s appears to be private/authenticated, skipping validation", image) return nil } - - // Check if the error means the Docker daemon became unreachable mid-process. - // This can happen when `docker info` succeeded earlier but the daemon stopped - // (or was never fully operational) by the time we issue docker pull. - // Treat this as a daemon-unavailable condition: update the cached state so - // subsequent images skip immediately, emit a single warning, and return nil - // (or an error when requireDocker is true). - // Use case-insensitive matching to handle Docker version differences. - outputLower := strings.ToLower(outputStr) - if strings.Contains(outputLower, "cannot connect to the docker daemon") || - strings.Contains(outputLower, "is the docker daemon running") { - markDockerDaemonUnavailable() - if requireDocker { - return fmt.Errorf("docker daemon not running - could not validate container image '%s'. Start the Docker daemon or omit the --validate-images flag to skip container image validation", image) - } - dockerValidationLog.Printf("Docker daemon not reachable during pull of %s, skipping container image validation", image) - return nil + if err := handleDockerDaemonUnavailableDuringPull(image, outputStr, requireDocker); err != nil || isDockerDaemonUnavailableOutput(outputStr) { + return err } - - // Check for non-retryable errors (image doesn't exist) - if strings.Contains(outputStr, "does not exist") || - strings.Contains(outputStr, "not found") || - strings.Contains(outputStr, "manifest unknown") { - // These errors won't be resolved by retrying + if isDockerImageMissingOutput(outputStr) { dockerValidationLog.Printf("Image %s does not exist (non-retryable error)", image) return fmt.Errorf("container image '%s' not found and could not be pulled: %s. Please verify the image name and tag.\n\nExample:\ntools:\n my-tool:\n container: \"node:20\"\n\nOr:\ntools:\n my-tool:\n container: \"ghcr.io/owner/image:latest\"\n\nSee: %s", image, outputStr, constants.DocsToolsURL) } @@ -225,3 +164,71 @@ func validateDockerImage(image string, verbose bool, requireDocker bool) error { // All attempts failed with retryable errors return fmt.Errorf("container image '%s' not found and could not be pulled after %d attempts: %s. Please verify the image name and tag.\n\nExample:\ntools:\n my-tool:\n container: \"node:20\"\n\nOr:\ntools:\n my-tool:\n container: \"ghcr.io/owner/image:latest\"\n\nSee: %s", image, maxAttempts, lastOutput, constants.DocsToolsURL) } + +func ensureDockerValidationAvailable(image string, requireDocker bool) (bool, error) { + if _, err := exec.LookPath("docker"); err != nil { + if requireDocker { + return false, fmt.Errorf("docker not installed - could not validate container image '%s'. Install Docker or omit the --validate-images flag to skip container image validation", image) + } + dockerValidationLog.Print("Docker not installed, skipping container image validation") + return false, nil + } + if !isDockerDaemonRunning() { + if requireDocker { + return false, fmt.Errorf("docker daemon not running - could not validate container image '%s'. Start the Docker daemon or omit the --validate-images flag to skip container image validation", image) + } + dockerValidationLog.Print("Docker daemon not running, skipping container image validation") + return false, nil + } + return true, nil +} + +func dockerImageExistsLocally(image string) bool { + cmd := exec.Command("docker", "image", "inspect", image) + _, err := cmd.CombinedOutput() + return err == nil +} + +func pullDockerImage(image string, attempt, maxAttempts int) (string, bool) { + dockerValidationLog.Printf("Attempt %d of %d: Pulling image %s", attempt, maxAttempts, image) + pullCmd := exec.Command("docker", "pull", image) + pullOutput, pullErr := pullCmd.CombinedOutput() + outputStr := strings.TrimSpace(string(pullOutput)) + if pullErr == nil { + dockerValidationLog.Printf("Successfully pulled image %s on attempt %d", image, attempt) + return outputStr, true + } + return outputStr, false +} + +func isPrivateAuthenticatedImage(outputStr string) bool { + return (strings.Contains(outputStr, "denied") || + strings.Contains(outputStr, "unauthorized") || + strings.Contains(outputStr, "authentication required")) && + !strings.Contains(outputStr, "does not exist") && + !strings.Contains(outputStr, "not found") +} + +func isDockerDaemonUnavailableOutput(outputStr string) bool { + outputLower := strings.ToLower(outputStr) + return strings.Contains(outputLower, "cannot connect to the docker daemon") || + strings.Contains(outputLower, "is the docker daemon running") +} + +func handleDockerDaemonUnavailableDuringPull(image string, outputStr string, requireDocker bool) error { + if !isDockerDaemonUnavailableOutput(outputStr) { + return nil + } + markDockerDaemonUnavailable() + if requireDocker { + return fmt.Errorf("docker daemon not running - could not validate container image '%s'. Start the Docker daemon or omit the --validate-images flag to skip container image validation", image) + } + dockerValidationLog.Printf("Docker daemon not reachable during pull of %s, skipping container image validation", image) + return nil +} + +func isDockerImageMissingOutput(outputStr string) bool { + return strings.Contains(outputStr, "does not exist") || + strings.Contains(outputStr, "not found") || + strings.Contains(outputStr, "manifest unknown") +} diff --git a/pkg/workflow/engine.go b/pkg/workflow/engine.go index e337de5f3b0..d973d3e0a77 100644 --- a/pkg/workflow/engine.go +++ b/pkg/workflow/engine.go @@ -195,416 +195,432 @@ func (e *EngineConfig) GetMaxTurnCacheMisses() int { // ExtractEngineConfig extracts engine configuration from frontmatter, supporting both string and object formats func (c *Compiler) ExtractEngineConfig(frontmatter map[string]any) (string, *EngineConfig) { - topLevelMaxTurns := parseMaxTurnsValue(frontmatter["max-turns"]) - topLevelMaxToolDenials := parseMaxToolDenialsValue(frontmatter["max-tool-denials"]) - topLevelMaxAICredits := parseMaxAICreditsValue(frontmatter["max-ai-credits"]) - topLevelMaxTurnCacheMisses := parseMaxTurnCacheMissesValue(frontmatter["max-turn-cache-misses"]) - topLevelMaxRuns := parseMaxRunsValue(frontmatter["max-turns"]) - if topLevelMaxRuns == 0 { - topLevelMaxRuns = parseMaxRunsValue(frontmatter["max-runs"]) - } - + limits := extractTopLevelEngineLimits(frontmatter) if engine, exists := frontmatter["engine"]; exists { engineLog.Print("Extracting engine configuration from frontmatter") - - // Handle string format (backwards compatibility) - if engineStr, ok := engine.(string); ok { - engineLog.Printf("Found engine in string format: %s", engineStr) - return engineStr, &EngineConfig{ - ID: engineStr, - MaxTurns: topLevelMaxTurns, - MaxToolDenials: topLevelMaxToolDenials, - MaxRuns: topLevelMaxRuns, - MaxTurnCacheMisses: topLevelMaxTurnCacheMisses, - MaxAICredits: topLevelMaxAICredits, - } - } - - // Handle object format - if engineObj, ok := engine.(map[string]any); ok { + switch typed := engine.(type) { + case string: + engineLog.Printf("Found engine in string format: %s", typed) + return typed, limits.newConfig(typed) + case map[string]any: engineLog.Print("Found engine in object format, parsing configuration") - config := &EngineConfig{} - - // Detect inline definition: engine.runtime sub-object present instead of engine.id - if runtime, hasRuntime := engineObj["runtime"]; hasRuntime { - engineLog.Print("Found inline engine definition (engine.runtime sub-object)") - config.IsInlineDefinition = true - - if runtimeObj, ok := runtime.(map[string]any); ok { - if id, ok := runtimeObj["id"].(string); ok { - config.ID = id - engineLog.Printf("Inline engine runtime.id: %s", config.ID) - } - if version, hasVersion := runtimeObj["version"]; hasVersion { - config.Version = stringutil.ParseVersionValue(version) - } - } - - // Extract optional provider override: - // - string form: engine.provider: "openai" - // - object form: engine.provider.id / auth / request - if provider, hasProvider := engineObj["provider"]; hasProvider { - switch providerTyped := provider.(type) { - case string: - config.InlineProviderID = strings.ToLower(strings.TrimSpace(providerTyped)) - case map[string]any: - if id, ok := providerTyped["id"].(string); ok { - config.InlineProviderID = id - } - if model, ok := providerTyped["model"].(string); ok { - config.Model = model - } - if auth, hasAuth := providerTyped["auth"]; hasAuth { - if authObj, ok := auth.(map[string]any); ok { - authDef := parseAuthDefinition(authObj) - // Only store an AuthDefinition when the user actually provided - // at least one recognised field. An empty map (e.g. `auth: {}`) - // must not be treated as an explicit auth override. - if authDef.Strategy != "" || authDef.Secret != "" || - authDef.TokenURL != "" || authDef.ClientIDRef != "" || - authDef.ClientSecretRef != "" || authDef.HeaderName != "" || - authDef.TokenField != "" { - config.InlineProviderAuth = authDef - } - } - } - if request, hasRequest := providerTyped["request"]; hasRequest { - if requestObj, ok := request.(map[string]any); ok { - config.InlineProviderRequest = parseRequestShape(requestObj) - } - } - } - } - - // Extract optional 'bare' field (shared with non-inline path) - if bare, hasBare := engineObj["bare"]; hasBare { - if bareBool, ok := bare.(bool); ok { - config.Bare = bareBool - engineLog.Printf("Extracted bare mode (inline): %v", config.Bare) - } - } - // Extract optional 'permission-mode' field (shared with non-inline path) - if permissionMode, hasPermissionMode := engineObj["permission-mode"]; hasPermissionMode { - if permissionModeStr, ok := permissionMode.(string); ok { - config.PermissionMode = permissionModeStr - } - } - if topLevelMaxTurns != "" { - config.MaxTurns = topLevelMaxTurns - } - if topLevelMaxToolDenials != "" { - config.MaxToolDenials = topLevelMaxToolDenials - } - config.MaxRuns = topLevelMaxRuns - config.MaxTurnCacheMisses = topLevelMaxTurnCacheMisses - config.MaxAICredits = topLevelMaxAICredits - - engineLog.Printf("Extracted inline engine definition: runtimeID=%s, providerID=%s", config.ID, config.InlineProviderID) - return config.ID, config - } - - // Extract required 'id' field - if id, hasID := engineObj["id"]; hasID { - if idStr, ok := id.(string); ok { - config.ID = idStr - } - } + return extractEngineObjectConfig(typed, limits) + } + } + if limits.hasAny() { + return "", limits.newConfig("") + } + engineLog.Print("No engine configuration found in frontmatter") + return "", nil +} - // Extract optional 'version' field - if version, hasVersion := engineObj["version"]; hasVersion { - config.Version = stringutil.ParseVersionValue(version) - } +type topLevelEngineLimits struct { + maxTurns string + maxToolDenials string + maxRuns int + maxTurnCacheMisses int + maxAICredits int64 +} - // Extract optional 'model' field - if model, hasModel := engineObj["model"]; hasModel { - if modelStr, ok := model.(string); ok { - config.Model = modelStr - } - } +func extractTopLevelEngineLimits(frontmatter map[string]any) topLevelEngineLimits { + maxRuns := parseMaxRunsValue(frontmatter["max-turns"]) + if maxRuns == 0 { + maxRuns = parseMaxRunsValue(frontmatter["max-runs"]) + } + return topLevelEngineLimits{ + maxTurns: parseMaxTurnsValue(frontmatter["max-turns"]), + maxToolDenials: parseMaxToolDenialsValue(frontmatter["max-tool-denials"]), + maxRuns: maxRuns, + maxTurnCacheMisses: parseMaxTurnCacheMissesValue(frontmatter["max-turn-cache-misses"]), + maxAICredits: parseMaxAICreditsValue(frontmatter["max-ai-credits"]), + } +} - // Extract optional 'model-provider' field. - providerValue, hasProvider := engineObj["model-provider"] - if hasProvider { - if providerStr, ok := providerValue.(string); ok { - config.LLMProvider = strings.ToLower(strings.TrimSpace(providerStr)) - } - } - // Extract optional 'provider' shorthand override for built-in engines. - // Inline definitions are excluded here because they use engine.provider object - // semantics (id/auth/request) parsed in the engine.runtime branch above. - if providerValue, hasProvider := engineObj["provider"]; hasProvider && !config.IsInlineDefinition { - if providerStr, ok := providerValue.(string); ok { - config.LLMProvider = strings.ToLower(strings.TrimSpace(providerStr)) - } - } +func (l topLevelEngineLimits) hasAny() bool { + return l.maxTurns != "" || l.maxToolDenials != "" || l.maxAICredits != 0 || l.maxRuns > 0 || l.maxTurnCacheMisses > 0 +} - // Extract optional 'permission-mode' field - if permissionMode, hasPermissionMode := engineObj["permission-mode"]; hasPermissionMode { - if permissionModeStr, ok := permissionMode.(string); ok { - config.PermissionMode = permissionModeStr - } - } +func (l topLevelEngineLimits) newConfig(id string) *EngineConfig { + return &EngineConfig{ + ID: id, + MaxTurns: l.maxTurns, + MaxToolDenials: l.maxToolDenials, + MaxRuns: l.maxRuns, + MaxTurnCacheMisses: l.maxTurnCacheMisses, + MaxAICredits: l.maxAICredits, + } +} - // Extract optional 'max-turns' field (deprecated alias for top-level max-turns). - // Use parseMaxTurnsValue for consistent validation: rejects negative values and - // arbitrary strings while preserving valid integers and GitHub Actions expressions. - if maxTurns, hasMaxTurns := engineObj["max-turns"]; hasMaxTurns { - config.MaxTurns = parseMaxTurnsValue(maxTurns) - } - if topLevelMaxTurns != "" { - config.MaxTurns = topLevelMaxTurns - } - if topLevelMaxToolDenials != "" { - config.MaxToolDenials = topLevelMaxToolDenials - } +func extractEngineObjectConfig(engineObj map[string]any, limits topLevelEngineLimits) (string, *EngineConfig) { + config := &EngineConfig{} + if runtime, hasRuntime := engineObj["runtime"]; hasRuntime { + parseInlineEngineConfig(config, engineObj, runtime, limits) + engineLog.Printf("Extracted inline engine definition: runtimeID=%s, providerID=%s", config.ID, config.InlineProviderID) + return config.ID, config + } + parseEngineIdentityFields(config, engineObj) + parseEngineProviderFields(config, engineObj) + parseEngineLimitFields(config, engineObj, limits) + parseEngineConcurrency(config, engineObj) + parseEngineCommandFields(config, engineObj) + parseEngineEnvAndAuth(config, engineObj) + parseEngineArgsAndAgent(config, engineObj) + parseEngineMCPConfig(config, engineObj) + parseEngineExtensions(config, engineObj) + applyEngineFinalFields(config, engineObj, limits) + engineLog.Printf("Extracted engine configuration: ID=%s", config.ID) + return config.ID, config +} - // Extract optional 'max-continuations' field - if maxCont, hasMaxCont := engineObj["max-continuations"]; hasMaxCont { - if val, ok := typeutil.ParseIntValue(maxCont); ok { - config.MaxContinuations = val - } else if maxContStr, ok := maxCont.(string); ok { - if parsed, err := strconv.Atoi(maxContStr); err == nil { - config.MaxContinuations = parsed - } - } - } +func parseInlineEngineConfig(config *EngineConfig, engineObj map[string]any, runtime any, limits topLevelEngineLimits) { + engineLog.Print("Found inline engine definition (engine.runtime sub-object)") + config.IsInlineDefinition = true + if runtimeObj, ok := runtime.(map[string]any); ok { + if id, ok := runtimeObj["id"].(string); ok { + config.ID = id + engineLog.Printf("Inline engine runtime.id: %s", config.ID) + } + if version, hasVersion := runtimeObj["version"]; hasVersion { + config.Version = stringutil.ParseVersionValue(version) + } + } + parseInlineProviderConfig(config, engineObj) + parseInlineSharedFields(config, engineObj) + config.MaxTurns = limits.maxTurns + config.MaxToolDenials = limits.maxToolDenials + config.MaxRuns = limits.maxRuns + config.MaxTurnCacheMisses = limits.maxTurnCacheMisses + config.MaxAICredits = limits.maxAICredits +} - // Extract optional 'concurrency' field (string or object format) - if concurrency, hasConcurrency := engineObj["concurrency"]; hasConcurrency { - if concurrencyStr, ok := concurrency.(string); ok { - // Simple string format (group name) - config.Concurrency = fmt.Sprintf("concurrency:\n group: \"%s\"", concurrencyStr) - } else if concurrencyObj, ok := concurrency.(map[string]any); ok { - // Object format with group and optional cancel-in-progress - var parts []string - if group, hasGroup := concurrencyObj["group"]; hasGroup { - if groupStr, ok := group.(string); ok { - parts = append(parts, fmt.Sprintf("concurrency:\n group: \"%s\"", groupStr)) - } - } - if cancel, hasCancel := concurrencyObj["cancel-in-progress"]; hasCancel { - if cancelBool, ok := cancel.(bool); ok && cancelBool { - if len(parts) > 0 { - parts[0] += "\n cancel-in-progress: true" - } - } - } - if queue, hasQueue := concurrencyObj["queue"]; hasQueue { - if queueStr, ok := queue.(string); ok && queueStr != "" { - if len(parts) > 0 { - parts[0] += "\n queue: " + queueStr - } - } - } - if len(parts) > 0 { - config.Concurrency = parts[0] - } - } +func parseInlineProviderConfig(config *EngineConfig, engineObj map[string]any) { + provider, hasProvider := engineObj["provider"] + if !hasProvider { + return + } + switch providerTyped := provider.(type) { + case string: + config.InlineProviderID = strings.ToLower(strings.TrimSpace(providerTyped)) + case map[string]any: + if id, ok := providerTyped["id"].(string); ok { + config.InlineProviderID = id + } + if model, ok := providerTyped["model"].(string); ok { + config.Model = model + } + parseInlineProviderAuth(config, providerTyped) + if request, hasRequest := providerTyped["request"]; hasRequest { + if requestObj, ok := request.(map[string]any); ok { + config.InlineProviderRequest = parseRequestShape(requestObj) } + } + } +} - // Extract optional 'user-agent' field - if userAgent, hasUserAgent := engineObj["user-agent"]; hasUserAgent { - if userAgentStr, ok := userAgent.(string); ok { - config.UserAgent = userAgentStr - } - } +func parseInlineProviderAuth(config *EngineConfig, providerTyped map[string]any) { + auth, hasAuth := providerTyped["auth"] + if !hasAuth { + return + } + authObj, ok := auth.(map[string]any) + if !ok { + return + } + authDef := parseAuthDefinition(authObj) + if authDef.Strategy != "" || authDef.Secret != "" || authDef.TokenURL != "" || + authDef.ClientIDRef != "" || authDef.ClientSecretRef != "" || + authDef.HeaderName != "" || authDef.TokenField != "" { + config.InlineProviderAuth = authDef + } +} - // Extract optional 'command' field - if command, hasCommand := engineObj["command"]; hasCommand { - if commandStr, ok := command.(string); ok { - config.Command = commandStr - } - } +func parseInlineSharedFields(config *EngineConfig, engineObj map[string]any) { + if bare, hasBare := engineObj["bare"]; hasBare { + if bareBool, ok := bare.(bool); ok { + config.Bare = bareBool + engineLog.Printf("Extracted bare mode (inline): %v", config.Bare) + } + } + if permissionMode, hasPermissionMode := engineObj["permission-mode"]; hasPermissionMode { + if permissionModeStr, ok := permissionMode.(string); ok { + config.PermissionMode = permissionModeStr + } + } +} - // Extract optional 'harness' field: - // - string form (legacy): engine.harness: "custom.cjs" → sets HarnessScript - // - object form: engine.harness: { use: "custom.cjs", max-retries: N, ... } - if harness, hasHarness := engineObj["harness"]; hasHarness { - switch h := harness.(type) { - case string: - config.HarnessScript = h - case map[string]any: - if use, ok := h["use"].(string); ok { - config.HarnessScript = use - } - if v, ok := h["max-retries"]; ok { - config.HarnessMaxRetries = parseHarnessMaxRetriesValue(v) - } - if v, ok := h["initial-delay-ms"]; ok { - config.HarnessInitialDelayMs = parseMaxTurnsValue(v) - } - if v, ok := h["backoff-multiplier"]; ok { - config.HarnessBackoffMultiplier = parseMaxTurnsValue(v) - } - if v, ok := h["max-delay-ms"]; ok { - config.HarnessMaxDelayMs = parseMaxTurnsValue(v) - } - } - } +func parseEngineIdentityFields(config *EngineConfig, engineObj map[string]any) { + if id, hasID := engineObj["id"]; hasID { + if idStr, ok := id.(string); ok { + config.ID = idStr + } + } + if version, hasVersion := engineObj["version"]; hasVersion { + config.Version = stringutil.ParseVersionValue(version) + } + if model, hasModel := engineObj["model"]; hasModel { + if modelStr, ok := model.(string); ok { + config.Model = modelStr + } + } +} - // Extract optional 'driver' field (string - validated separately). - if driver, hasDriver := engineObj["driver"]; hasDriver { - if driverStr, ok := driver.(string); ok { - config.Driver = driverStr - engineLog.Printf("Extracted engine.driver: %s", driverStr) - } - } +func parseEngineProviderFields(config *EngineConfig, engineObj map[string]any) { + if providerValue, hasProvider := engineObj["model-provider"]; hasProvider { + if providerStr, ok := providerValue.(string); ok { + config.LLMProvider = strings.ToLower(strings.TrimSpace(providerStr)) + } + } + if providerValue, hasProvider := engineObj["provider"]; hasProvider && !config.IsInlineDefinition { + if providerStr, ok := providerValue.(string); ok { + config.LLMProvider = strings.ToLower(strings.TrimSpace(providerStr)) + } + } + if permissionMode, hasPermissionMode := engineObj["permission-mode"]; hasPermissionMode { + if permissionModeStr, ok := permissionMode.(string); ok { + config.PermissionMode = permissionModeStr + } + } +} - // Extract optional 'env' field (object/map of scalar values) - if env, hasEnv := engineObj["env"]; hasEnv { - if envMap, ok := env.(map[string]any); ok { - config.Env = make(map[string]string) - for key, value := range envMap { - if valueStr, ok := toEngineEnvValueString(value); ok { - config.Env[key] = valueStr - } - } - } +func parseEngineLimitFields(config *EngineConfig, engineObj map[string]any, limits topLevelEngineLimits) { + if maxTurns, hasMaxTurns := engineObj["max-turns"]; hasMaxTurns { + config.MaxTurns = parseMaxTurnsValue(maxTurns) + } + if limits.maxTurns != "" { + config.MaxTurns = limits.maxTurns + } + if limits.maxToolDenials != "" { + config.MaxToolDenials = limits.maxToolDenials + } + if maxCont, hasMaxCont := engineObj["max-continuations"]; hasMaxCont { + if val, ok := typeutil.ParseIntValue(maxCont); ok { + config.MaxContinuations = val + } else if maxContStr, ok := maxCont.(string); ok { + if parsed, err := strconv.Atoi(maxContStr); err == nil { + config.MaxContinuations = parsed } + } + } +} - // Extract optional 'auth' field (object) - if auth, hasAuth := engineObj["auth"]; hasAuth { - if authObj, ok := auth.(map[string]any); ok { - config.Auth = parseEngineAuthConfig(authObj) - applyEngineAuthEnv(config) - } - } +func parseEngineConcurrency(config *EngineConfig, engineObj map[string]any) { + concurrency, hasConcurrency := engineObj["concurrency"] + if !hasConcurrency { + return + } + if concurrencyStr, ok := concurrency.(string); ok { + config.Concurrency = fmt.Sprintf("concurrency:\n group: \"%s\"", concurrencyStr) + return + } + if concurrencyObj, ok := concurrency.(map[string]any); ok { + config.Concurrency = renderEngineConcurrencyObject(concurrencyObj) + } +} - // Extract optional 'config' field (additional TOML configuration) - if config_field, hasConfig := engineObj["config"]; hasConfig { - if configStr, ok := config_field.(string); ok { - config.Config = configStr - } - } +func renderEngineConcurrencyObject(concurrencyObj map[string]any) string { + var parts []string + if group, hasGroup := concurrencyObj["group"]; hasGroup { + if groupStr, ok := group.(string); ok { + parts = append(parts, fmt.Sprintf("concurrency:\n group: \"%s\"", groupStr)) + } + } + if cancel, hasCancel := concurrencyObj["cancel-in-progress"]; hasCancel { + if cancelBool, ok := cancel.(bool); ok && cancelBool && len(parts) > 0 { + parts[0] += "\n cancel-in-progress: true" + } + } + if queue, hasQueue := concurrencyObj["queue"]; hasQueue { + if queueStr, ok := queue.(string); ok && queueStr != "" && len(parts) > 0 { + parts[0] += "\n queue: " + queueStr + } + } + if len(parts) > 0 { + return parts[0] + } + return "" +} - // Extract optional 'args' field (array of strings) - if args, hasArgs := engineObj["args"]; hasArgs { - if argsArray, ok := args.([]any); ok { - config.Args = make([]string, 0, len(argsArray)) - for _, arg := range argsArray { - if argStr, ok := arg.(string); ok { - config.Args = append(config.Args, argStr) - } - } - } else if argsStrArray, ok := args.([]string); ok { - config.Args = argsStrArray - } - } +func parseEngineCommandFields(config *EngineConfig, engineObj map[string]any) { + if userAgent, hasUserAgent := engineObj["user-agent"]; hasUserAgent { + if userAgentStr, ok := userAgent.(string); ok { + config.UserAgent = userAgentStr + } + } + if command, hasCommand := engineObj["command"]; hasCommand { + if commandStr, ok := command.(string); ok { + config.Command = commandStr + } + } + parseEngineHarness(config, engineObj) + if driver, hasDriver := engineObj["driver"]; hasDriver { + if driverStr, ok := driver.(string); ok { + config.Driver = driverStr + engineLog.Printf("Extracted engine.driver: %s", driverStr) + } + } +} - // Extract optional 'agent' field (string - copilot engine only) - if agent, hasAgent := engineObj["agent"]; hasAgent { - if agentStr, ok := agent.(string); ok { - config.Agent = agentStr - engineLog.Printf("Extracted agent identifier: %s", agentStr) - } - } +func parseEngineHarness(config *EngineConfig, engineObj map[string]any) { + harness, hasHarness := engineObj["harness"] + if !hasHarness { + return + } + switch h := harness.(type) { + case string: + config.HarnessScript = h + case map[string]any: + if use, ok := h["use"].(string); ok { + config.HarnessScript = use + } + if v, ok := h["max-retries"]; ok { + config.HarnessMaxRetries = parseHarnessMaxRetriesValue(v) + } + if v, ok := h["initial-delay-ms"]; ok { + config.HarnessInitialDelayMs = parseMaxTurnsValue(v) + } + if v, ok := h["backoff-multiplier"]; ok { + config.HarnessBackoffMultiplier = parseMaxTurnsValue(v) + } + if v, ok := h["max-delay-ms"]; ok { + config.HarnessMaxDelayMs = parseMaxTurnsValue(v) + } + } +} - // Extract optional 'api-target' field (custom API endpoint for any engine) - if apiTarget, hasAPITarget := engineObj["api-target"]; hasAPITarget { - if apiTargetStr, ok := apiTarget.(string); ok && apiTargetStr != "" { - config.APITarget = apiTargetStr - engineLog.Printf("Extracted api-target: %s", apiTargetStr) +func parseEngineEnvAndAuth(config *EngineConfig, engineObj map[string]any) { + if env, hasEnv := engineObj["env"]; hasEnv { + if envMap, ok := env.(map[string]any); ok { + config.Env = make(map[string]string) + for key, value := range envMap { + if valueStr, ok := toEngineEnvValueString(value); ok { + config.Env[key] = valueStr } } + } + } + if auth, hasAuth := engineObj["auth"]; hasAuth { + if authObj, ok := auth.(map[string]any); ok { + config.Auth = parseEngineAuthConfig(authObj) + applyEngineAuthEnv(config) + } + } + if configField, hasConfig := engineObj["config"]; hasConfig { + if configStr, ok := configField.(string); ok { + config.Config = configStr + } + } +} - // Extract optional 'bare' field (disable automatic context/instruction loading) - if bare, hasBare := engineObj["bare"]; hasBare { - if bareBool, ok := bare.(bool); ok { - config.Bare = bareBool - engineLog.Printf("Extracted bare mode: %v", config.Bare) +func parseEngineArgsAndAgent(config *EngineConfig, engineObj map[string]any) { + if args, hasArgs := engineObj["args"]; hasArgs { + if argsArray, ok := args.([]any); ok { + config.Args = make([]string, 0, len(argsArray)) + for _, arg := range argsArray { + if argStr, ok := arg.(string); ok { + config.Args = append(config.Args, argStr) } } + } else if argsStrArray, ok := args.([]string); ok { + config.Args = argsStrArray + } + } + if agent, hasAgent := engineObj["agent"]; hasAgent { + if agentStr, ok := agent.(string); ok { + config.Agent = agentStr + engineLog.Printf("Extracted agent identifier: %s", agentStr) + } + } + parseEngineAPITargetAndBare(config, engineObj) +} - // Extract optional 'mcp' sub-object (engine-level MCP gateway configuration) - if mcpVal, hasMCP := engineObj["mcp"]; hasMCP { - if mcpObj, ok := mcpVal.(map[string]any); ok { - // Extract session-timeout (kebab-case only; camelCase is not supported) - if stVal, hasSessionTimeout := mcpObj["session-timeout"]; hasSessionTimeout { - if stStr, ok := stVal.(string); ok && stStr != "" { - config.MCPSessionTimeout = stStr - engineLog.Printf("Extracted engine.mcp.session-timeout: %s", config.MCPSessionTimeout) - } - } - // Extract tool-timeout (kebab-case only; camelCase is not supported) - if ttVal, hasToolTimeout := mcpObj["tool-timeout"]; hasToolTimeout { - if ttStr, ok := ttVal.(string); ok && ttStr != "" { - config.MCPToolTimeout = ttStr - engineLog.Printf("Extracted engine.mcp.tool-timeout: %s", config.MCPToolTimeout) - } - } - } - } +func parseEngineAPITargetAndBare(config *EngineConfig, engineObj map[string]any) { + if apiTarget, hasAPITarget := engineObj["api-target"]; hasAPITarget { + if apiTargetStr, ok := apiTarget.(string); ok && apiTargetStr != "" { + config.APITarget = apiTargetStr + engineLog.Printf("Extracted api-target: %s", apiTargetStr) + } + } + if bare, hasBare := engineObj["bare"]; hasBare { + if bareBool, ok := bare.(bool); ok { + config.Bare = bareBool + engineLog.Printf("Extracted bare mode: %v", config.Bare) + } + } +} - // Extract optional 'extensions' field (array of strings; used by the Pi engine) - if extVal, hasExt := engineObj["extensions"]; hasExt { - switch v := extVal.(type) { - case []any: - config.Extensions = make([]string, 0, len(v)) - for _, ext := range v { - if extStr, ok := ext.(string); ok && extStr != "" { - config.Extensions = append(config.Extensions, extStr) - } - } - engineLog.Printf("Extracted engine.extensions: %v", config.Extensions) - case []string: - config.Extensions = make([]string, 0, len(v)) - for _, ext := range v { - if ext != "" { - config.Extensions = append(config.Extensions, ext) - } - } - engineLog.Printf("Extracted engine.extensions ([]string): %v", config.Extensions) - default: - engineLog.Printf("Unexpected type for engine.extensions: %T, ignoring", extVal) - } - } +func parseEngineMCPConfig(config *EngineConfig, engineObj map[string]any) { + mcpVal, hasMCP := engineObj["mcp"] + if !hasMCP { + return + } + mcpObj, ok := mcpVal.(map[string]any) + if !ok { + return + } + if stVal, hasSessionTimeout := mcpObj["session-timeout"]; hasSessionTimeout { + if stStr, ok := stVal.(string); ok && stStr != "" { + config.MCPSessionTimeout = stStr + engineLog.Printf("Extracted engine.mcp.session-timeout: %s", config.MCPSessionTimeout) + } + } + if ttVal, hasToolTimeout := mcpObj["tool-timeout"]; hasToolTimeout { + if ttStr, ok := ttVal.(string); ok && ttStr != "" { + config.MCPToolTimeout = ttStr + engineLog.Printf("Extracted engine.mcp.tool-timeout: %s", config.MCPToolTimeout) + } + } +} - // Return the ID as the engineSetting for backwards compatibility - if topLevelMaxTurns != "" { - config.MaxTurns = topLevelMaxTurns - } - config.MaxRuns = topLevelMaxRuns - config.MaxTurnCacheMisses = topLevelMaxTurnCacheMisses - config.MaxAICredits = topLevelMaxAICredits - - // Extract optional 'copilot-sdk' field (bool; copilot engine only) - if sdkVal, hasSDK := engineObj["copilot-sdk"]; hasSDK { - if sdkBool, ok := sdkVal.(bool); ok { - config.CopilotSDK = sdkBool - engineLog.Printf("Extracted copilot-sdk: %v", config.CopilotSDK) - } - } - if config.Driver != "" && config.ID == "copilot" && !config.CopilotSDK { - config.CopilotSDK = true - engineLog.Print("Enabled copilot-sdk because driver is configured for copilot engine") +func parseEngineExtensions(config *EngineConfig, engineObj map[string]any) { + extVal, hasExt := engineObj["extensions"] + if !hasExt { + return + } + switch v := extVal.(type) { + case []any: + config.Extensions = make([]string, 0, len(v)) + for _, ext := range v { + if extStr, ok := ext.(string); ok && extStr != "" { + config.Extensions = append(config.Extensions, extStr) } - - // Extract optional 'cwd' field (templatable string for engine working directory) - if cwdVal, hasCwd := engineObj["cwd"]; hasCwd { - if cwdStr, ok := cwdVal.(string); ok && cwdStr != "" { - config.Cwd = cwdStr - engineLog.Printf("Extracted engine.cwd: %s", config.Cwd) - } + } + engineLog.Printf("Extracted engine.extensions: %v", config.Extensions) + case []string: + config.Extensions = make([]string, 0, len(v)) + for _, ext := range v { + if ext != "" { + config.Extensions = append(config.Extensions, ext) } - - engineLog.Printf("Extracted engine configuration: ID=%s", config.ID) - return config.ID, config } + engineLog.Printf("Extracted engine.extensions ([]string): %v", config.Extensions) + default: + engineLog.Printf("Unexpected type for engine.extensions: %T, ignoring", extVal) } +} - if topLevelMaxTurns != "" || topLevelMaxToolDenials != "" || topLevelMaxAICredits != 0 || topLevelMaxRuns > 0 || topLevelMaxTurnCacheMisses > 0 { - return "", &EngineConfig{ - MaxTurns: topLevelMaxTurns, - MaxToolDenials: topLevelMaxToolDenials, - MaxRuns: topLevelMaxRuns, - MaxTurnCacheMisses: topLevelMaxTurnCacheMisses, - MaxAICredits: topLevelMaxAICredits, +func applyEngineFinalFields(config *EngineConfig, engineObj map[string]any, limits topLevelEngineLimits) { + if limits.maxTurns != "" { + config.MaxTurns = limits.maxTurns + } + config.MaxRuns = limits.maxRuns + config.MaxTurnCacheMisses = limits.maxTurnCacheMisses + config.MaxAICredits = limits.maxAICredits + if sdkVal, hasSDK := engineObj["copilot-sdk"]; hasSDK { + if sdkBool, ok := sdkVal.(bool); ok { + config.CopilotSDK = sdkBool + engineLog.Printf("Extracted copilot-sdk: %v", config.CopilotSDK) + } + } + if config.Driver != "" && config.ID == "copilot" && !config.CopilotSDK { + config.CopilotSDK = true + engineLog.Print("Enabled copilot-sdk because driver is configured for copilot engine") + } + if cwdVal, hasCwd := engineObj["cwd"]; hasCwd { + if cwdStr, ok := cwdVal.(string); ok && cwdStr != "" { + config.Cwd = cwdStr + engineLog.Printf("Extracted engine.cwd: %s", config.Cwd) } } - - // No engine specified - engineLog.Print("No engine configuration found in frontmatter") - return "", nil } // getAgenticEngine returns the agentic engine for the given engine setting @@ -680,59 +696,24 @@ func applyEngineAuthEnv(config *EngineConfig) { config.Env = make(map[string]string) } - if config.Auth.Type != "" { - if _, exists := config.Env["AWF_AUTH_TYPE"]; !exists { - config.Env["AWF_AUTH_TYPE"] = config.Auth.Type - } - } - if config.Auth.Audience != "" { - if _, exists := config.Env["AWF_AUTH_OIDC_AUDIENCE"]; !exists { - config.Env["AWF_AUTH_OIDC_AUDIENCE"] = config.Auth.Audience - } - } - if config.Auth.AzureTenantID != "" { - if _, exists := config.Env["AWF_AUTH_AZURE_TENANT_ID"]; !exists { - config.Env["AWF_AUTH_AZURE_TENANT_ID"] = config.Auth.AzureTenantID - } - } - if config.Auth.AzureClientID != "" { - if _, exists := config.Env["AWF_AUTH_AZURE_CLIENT_ID"]; !exists { - config.Env["AWF_AUTH_AZURE_CLIENT_ID"] = config.Auth.AzureClientID - } - } - if config.Auth.AzureScope != "" { - if _, exists := config.Env["AWF_AUTH_AZURE_SCOPE"]; !exists { - config.Env["AWF_AUTH_AZURE_SCOPE"] = config.Auth.AzureScope - } - } - if config.Auth.AzureCloud != "" { - if _, exists := config.Env["AWF_AUTH_AZURE_CLOUD"]; !exists { - config.Env["AWF_AUTH_AZURE_CLOUD"] = config.Auth.AzureCloud - } - } - if config.Auth.Provider != "" { - if _, exists := config.Env["AWF_AUTH_PROVIDER"]; !exists { - config.Env["AWF_AUTH_PROVIDER"] = config.Auth.Provider - } - } - if config.Auth.AnthropicFederationRuleID != "" { - if _, exists := config.Env["AWF_AUTH_ANTHROPIC_FEDERATION_RULE_ID"]; !exists { - config.Env["AWF_AUTH_ANTHROPIC_FEDERATION_RULE_ID"] = config.Auth.AnthropicFederationRuleID - } - } - if config.Auth.AnthropicOrganizationID != "" { - if _, exists := config.Env["AWF_AUTH_ANTHROPIC_ORGANIZATION_ID"]; !exists { - config.Env["AWF_AUTH_ANTHROPIC_ORGANIZATION_ID"] = config.Auth.AnthropicOrganizationID - } - } - if config.Auth.AnthropicServiceAccountID != "" { - if _, exists := config.Env["AWF_AUTH_ANTHROPIC_SERVICE_ACCOUNT_ID"]; !exists { - config.Env["AWF_AUTH_ANTHROPIC_SERVICE_ACCOUNT_ID"] = config.Auth.AnthropicServiceAccountID - } + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_TYPE", config.Auth.Type) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_OIDC_AUDIENCE", config.Auth.Audience) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_AZURE_TENANT_ID", config.Auth.AzureTenantID) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_AZURE_CLIENT_ID", config.Auth.AzureClientID) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_AZURE_SCOPE", config.Auth.AzureScope) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_AZURE_CLOUD", config.Auth.AzureCloud) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_PROVIDER", config.Auth.Provider) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_ANTHROPIC_FEDERATION_RULE_ID", config.Auth.AnthropicFederationRuleID) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_ANTHROPIC_ORGANIZATION_ID", config.Auth.AnthropicOrganizationID) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_ANTHROPIC_SERVICE_ACCOUNT_ID", config.Auth.AnthropicServiceAccountID) + setEngineAuthEnvIfMissing(config.Env, "AWF_AUTH_ANTHROPIC_WORKSPACE_ID", config.Auth.AnthropicWorkspaceID) +} + +func setEngineAuthEnvIfMissing(env map[string]string, key, value string) { + if value == "" { + return } - if config.Auth.AnthropicWorkspaceID != "" { - if _, exists := config.Env["AWF_AUTH_ANTHROPIC_WORKSPACE_ID"]; !exists { - config.Env["AWF_AUTH_ANTHROPIC_WORKSPACE_ID"] = config.Auth.AnthropicWorkspaceID - } + if _, exists := env[key]; !exists { + env[key] = value } } diff --git a/pkg/workflow/engine_definition_loader.go b/pkg/workflow/engine_definition_loader.go index 904b844ebde..1851d5e75e3 100644 --- a/pkg/workflow/engine_definition_loader.go +++ b/pkg/workflow/engine_definition_loader.go @@ -113,57 +113,13 @@ func loadBuiltinEngineDefinitions() []*EngineDefinition { if err != nil { return err } - if d.IsDir() { + if !isBuiltinEngineMarkdownFile(path, d) { return nil } - if filepath.Ext(path) != ".md" { - return nil - } - - data, readErr := builtinEngineFS.ReadFile(path) + def, readErr := loadBuiltinEngineDefinition(path) if readErr != nil { - return fmt.Errorf("failed to read embedded engine file %s: %w", path, readErr) - } - - // Extract the frontmatter YAML from the Markdown file. - frontmatterYAML, fmErr := extractMarkdownFrontmatterYAML(data) - if fmErr != nil { - return fmt.Errorf("failed to extract frontmatter from %s: %w", path, fmErr) + return readErr } - - // Parse the engine definition from the frontmatter. - var wrapper engineDefinitionFile - if parseErr := yaml.Unmarshal(frontmatterYAML, &wrapper); parseErr != nil { - return fmt.Errorf("failed to parse embedded engine file %s: %w", path, parseErr) - } - - def := wrapper.Engine - - // Default runtime-id to engine id when omitted. - if def.RuntimeID == "" { - def.RuntimeID = def.ID - } - - // Register the full .md content in the parser's builtin virtual FS so the - // file can be resolved and read during import processing. - parser.RegisterBuiltinVirtualFile(builtinEnginePath(def.ID), data) - - // Register the JSON key that the import processor produces for this built-in - // so that parseEngineDefinitionFromJSON will cache it. The import processor - // marshals the raw "engine:" value (a map[string]any) to JSON; we replicate - // that here so the cache key matches exactly. - var rawFM map[string]any - if jsonErr := yaml.Unmarshal(frontmatterYAML, &rawFM); jsonErr != nil { - engineDefinitionLoaderLog.Printf("Warning: failed to unmarshal frontmatter for cache-key registration of %s: %v", path, jsonErr) - } else if engineVal, ok := rawFM["engine"]; ok { - if jsonKey, jsonErr := json.Marshal(engineVal); jsonErr != nil { - engineDefinitionLoaderLog.Printf("Warning: failed to marshal engine value to JSON for cache-key registration of %s: %v", path, jsonErr) - } else { - registerBuiltinEngineDefinitionJSON(string(jsonKey)) - } - } - - engineDefinitionLoaderLog.Printf("Loaded built-in engine definition: id=%s runtime-id=%s", def.ID, def.RuntimeID) definitions = append(definitions, &def) return nil }) @@ -175,3 +131,50 @@ func loadBuiltinEngineDefinitions() []*EngineDefinition { engineDefinitionLoaderLog.Printf("Loaded %d built-in engine definitions", len(definitions)) return definitions } + +func isBuiltinEngineMarkdownFile(path string, d fs.DirEntry) bool { + return !d.IsDir() && filepath.Ext(path) == ".md" +} + +func loadBuiltinEngineDefinition(path string) (EngineDefinition, error) { + data, err := builtinEngineFS.ReadFile(path) + if err != nil { + return EngineDefinition{}, fmt.Errorf("failed to read embedded engine file %s: %w", path, err) + } + frontmatterYAML, err := extractMarkdownFrontmatterYAML(data) + if err != nil { + return EngineDefinition{}, fmt.Errorf("failed to extract frontmatter from %s: %w", path, err) + } + + var wrapper engineDefinitionFile + if err := yaml.Unmarshal(frontmatterYAML, &wrapper); err != nil { + return EngineDefinition{}, fmt.Errorf("failed to parse embedded engine file %s: %w", path, err) + } + + def := wrapper.Engine + if def.RuntimeID == "" { + def.RuntimeID = def.ID + } + parser.RegisterBuiltinVirtualFile(builtinEnginePath(def.ID), data) + registerBuiltinEngineDefinitionCacheKey(path, frontmatterYAML) + engineDefinitionLoaderLog.Printf("Loaded built-in engine definition: id=%s runtime-id=%s", def.ID, def.RuntimeID) + return def, nil +} + +func registerBuiltinEngineDefinitionCacheKey(path string, frontmatterYAML []byte) { + var rawFM map[string]any + if err := yaml.Unmarshal(frontmatterYAML, &rawFM); err != nil { + engineDefinitionLoaderLog.Printf("Warning: failed to unmarshal frontmatter for cache-key registration of %s: %v", path, err) + return + } + engineVal, ok := rawFM["engine"] + if !ok { + return + } + jsonKey, err := json.Marshal(engineVal) + if err != nil { + engineDefinitionLoaderLog.Printf("Warning: failed to marshal engine value to JSON for cache-key registration of %s: %v", path, err) + return + } + registerBuiltinEngineDefinitionJSON(string(jsonKey)) +} diff --git a/pkg/workflow/engine_validation.go b/pkg/workflow/engine_validation.go index fccfddb71af..434e1ac83c2 100644 --- a/pkg/workflow/engine_validation.go +++ b/pkg/workflow/engine_validation.go @@ -392,28 +392,27 @@ func isModelOnlyEngineJSON(engineJSON string) bool { // validateSingleEngineSpecification validates that only one engine field exists across all files func (c *Compiler) validateSingleEngineSpecification(mainEngineSetting string, includedEnginesJSON []string) (string, error) { + allEngines, firstIncludedRealEngine := collectRealEngineSpecifications(mainEngineSetting, includedEnginesJSON) + if len(allEngines) == 0 { + return "", nil // No engine specification found anywhere; will use default + } + if len(allEngines) > 1 { + return "", fmt.Errorf("multiple engine fields found (%d engine specifications detected). Only one engine field is allowed across the main workflow and all included files. Remove duplicate engine specifications to keep only one.\n\nExample:\nengine: copilot\n\nSee: %s", len(allEngines), constants.DocsEnginesURL) + } + if mainEngineSetting != "" { + return mainEngineSetting, nil + } + return parseIncludedEngineSpecification(firstIncludedRealEngine) +} + +func collectRealEngineSpecifications(mainEngineSetting string, includedEnginesJSON []string) ([]string, string) { var allEngines []string - // firstIncludedRealEngine holds the raw JSON of the first non-model-only engine spec - // from included files. It is used below to extract the engine ID when the single - // engine specification originates from an included file rather than the main workflow. var firstIncludedRealEngine string - - // Add main engine if specified if mainEngineSetting != "" { allEngines = append(allEngines, mainEngineSetting) } - - // Add included engines — skip preference-only configs (objects with only 'model'/'mcp' - // keys and no 'id' or 'runtime'). These express a model or MCP preference without - // selecting an engine and must not be counted as engine specifications (avoids spurious - // "multiple engine fields" errors when a shared workflow only declares engine.model - // without engine.id). Objects with unknown keys or empty objects are not skipped - // and will continue through normal validation. for _, engineJSON := range includedEnginesJSON { - if engineJSON == "" { - continue - } - if isModelOnlyEngineJSON(engineJSON) { + if engineJSON == "" || isModelOnlyEngineJSON(engineJSON) { continue } allEngines = append(allEngines, engineJSON) @@ -421,50 +420,41 @@ func (c *Compiler) validateSingleEngineSpecification(mainEngineSetting string, i firstIncludedRealEngine = engineJSON } } + return allEngines, firstIncludedRealEngine +} - // Check count (only counting real engine specifications) - if len(allEngines) == 0 { - return "", nil // No engine specification found anywhere; will use default - } - - if len(allEngines) > 1 { - return "", fmt.Errorf("multiple engine fields found (%d engine specifications detected). Only one engine field is allowed across the main workflow and all included files. Remove duplicate engine specifications to keep only one.\n\nExample:\nengine: copilot\n\nSee: %s", len(allEngines), constants.DocsEnginesURL) - } - - // Exactly one engine found - parse and return it - if mainEngineSetting != "" { - return mainEngineSetting, nil - } - - // Must be from included file - parse the first real included engine specification +func parseIncludedEngineSpecification(firstIncludedRealEngine string) (string, error) { var firstEngine any if err := json.Unmarshal([]byte(firstIncludedRealEngine), &firstEngine); err != nil { return "", fmt.Errorf("failed to parse included engine configuration: %w. Expected string or object format.\n\nExample (string):\nengine: copilot\n\nExample (object):\nengine:\n id: copilot\n model: gpt-4\n\nSee: %s", err, constants.DocsEnginesURL) } - - // Handle string format if engineStr, ok := firstEngine.(string); ok { return engineStr, nil - } else if engineObj, ok := firstEngine.(map[string]any); ok { - // Handle object format: either engine.id (named engine) or engine.runtime.id (inline definition) - if id, hasID := engineObj["id"]; hasID { - if idStr, ok := id.(string); ok { - return idStr, nil - } + } + if engineObj, ok := firstEngine.(map[string]any); ok { + if idStr, ok := includedEngineID(engineObj); ok { + return idStr, nil } - // Handle inline definition with 'runtime' sub-object (engine.runtime.id) - if runtime, hasRuntime := engineObj["runtime"]; hasRuntime { - if runtimeObj, ok := runtime.(map[string]any); ok { - if id, hasID := runtimeObj["id"]; hasID { - if idStr, ok := id.(string); ok { - return idStr, nil - } + } + return "", fmt.Errorf("invalid engine configuration in included file, missing or invalid 'id' field. Expected string, object with 'id' field, or inline definition with 'runtime.id'.\n\nExample (string):\nengine: copilot\n\nExample (object with id):\nengine:\n id: copilot\n model: gpt-4\n\nExample (inline runtime definition):\nengine:\n runtime:\n id: codex\n\nSee: %s", constants.DocsEnginesURL) +} + +func includedEngineID(engineObj map[string]any) (string, bool) { + if id, hasID := engineObj["id"]; hasID { + if idStr, ok := id.(string); ok { + return idStr, true + } + } + if runtime, hasRuntime := engineObj["runtime"]; hasRuntime { + if runtimeObj, ok := runtime.(map[string]any); ok { + if id, hasID := runtimeObj["id"]; hasID { + if idStr, ok := id.(string); ok { + return idStr, true } } } } - - return "", fmt.Errorf("invalid engine configuration in included file, missing or invalid 'id' field. Expected string, object with 'id' field, or inline definition with 'runtime.id'.\n\nExample (string):\nengine: copilot\n\nExample (object with id):\nengine:\n id: copilot\n model: gpt-4\n\nExample (inline runtime definition):\nengine:\n runtime:\n id: codex\n\nSee: %s", constants.DocsEnginesURL) + return "", false } // EngineHasValidateSecretStep checks if the engine provides a validate-secret step. diff --git a/pkg/workflow/error_recovery.go b/pkg/workflow/error_recovery.go index 9fff8526f8d..76736b453c3 100644 --- a/pkg/workflow/error_recovery.go +++ b/pkg/workflow/error_recovery.go @@ -147,6 +147,23 @@ func BuildPrioritizedErrorReportFromMessages(messages []string, showAll bool) Pr } func prioritizeErrorMessages(messages []string) ([]PrioritizedError, int) { + candidates := classifyUniqueErrorMessages(messages) + prioritized, suppressedCount, hasCriticalSyntax := suppressCascadingErrors(candidates) + + if hasCriticalSyntax { + errorRecoveryLog.Printf("Critical syntax errors detected, suppressed %d cascading errors", suppressedCount) + } + + if len(prioritized) == 0 { + prioritized = candidates + suppressedCount = 0 + } + + sortPrioritizedErrors(prioritized) + return prioritized, suppressedCount +} + +func classifyUniqueErrorMessages(messages []string) []PrioritizedError { candidates := make([]PrioritizedError, 0, len(messages)) seen := make(map[string]struct{}, len(messages)) for _, message := range messages { @@ -161,17 +178,16 @@ func prioritizeErrorMessages(messages []string) ([]PrioritizedError, int) { seen[key] = struct{}{} candidates = append(candidates, classifyErrorMessage(trimmed)) } + return candidates +} + +func suppressCascadingErrors(candidates []PrioritizedError) ([]PrioritizedError, int, bool) { + hasCriticalSyntax := slices.ContainsFunc(candidates, func(candidate PrioritizedError) bool { + return candidate.Severity == SeverityCritical && candidate.Category == "syntax" + }) var prioritized []PrioritizedError suppressedCount := 0 - hasCriticalSyntax := false - for _, candidate := range candidates { - if candidate.Severity == SeverityCritical && candidate.Category == "syntax" { - hasCriticalSyntax = true - break - } - } - for _, candidate := range candidates { if shouldSuppressCascadingError(candidate, hasCriticalSyntax) { suppressedCount++ @@ -179,16 +195,10 @@ func prioritizeErrorMessages(messages []string) ([]PrioritizedError, int) { } prioritized = append(prioritized, candidate) } + return prioritized, suppressedCount, hasCriticalSyntax +} - if hasCriticalSyntax { - errorRecoveryLog.Printf("Critical syntax errors detected, suppressed %d cascading errors", suppressedCount) - } - - if len(prioritized) == 0 { - prioritized = candidates - suppressedCount = 0 - } - +func sortPrioritizedErrors(prioritized []PrioritizedError) { slices.SortStableFunc(prioritized, func(a, b PrioritizedError) int { if a.Severity != b.Severity { if a.Severity < b.Severity { @@ -211,8 +221,6 @@ func prioritizeErrorMessages(messages []string) ([]PrioritizedError, int) { return 0 } }) - - return prioritized, suppressedCount } func shouldSuppressCascadingError(candidate PrioritizedError, hasCriticalSyntax bool) bool { @@ -263,110 +271,63 @@ func classifyErrorMessage(message string) PrioritizedError { lower := normalizeErrorMessage(message) switch { - case strings.Contains(lower, "failed to parse frontmatter"), - strings.Contains(lower, "failed to parse yaml frontmatter"), - strings.Contains(lower, "no frontmatter found"), - strings.Contains(lower, "mapping values are not allowed"), - strings.Contains(lower, "did not find expected key"), - strings.Contains(lower, "missing ':' after key"), - strings.Contains(lower, "unexpected ':'"), - strings.Contains(lower, "tool config must be a mapping"): - return PrioritizedError{ - Message: message, - Severity: SeverityCritical, - Category: "syntax", - Suggestion: "Fix the YAML/frontmatter syntax first, then re-run `gh aw compile`.", - } + case isFrontmatterSyntaxMessage(lower): + return newPrioritizedError(message, SeverityCritical, "syntax", "Fix the YAML/frontmatter syntax first, then re-run `gh aw compile`.") case strings.Contains(lower, "mapping key") && strings.Contains(lower, "already defined"): - return PrioritizedError{ - Message: message, - Severity: SeverityCritical, - Category: "syntax", - Suggestion: buildDuplicateKeySuggestion(message), - } + return newPrioritizedError(message, SeverityCritical, "syntax", buildDuplicateKeySuggestion(message)) case strings.Contains(lower, "invalid engine"), strings.Contains(lower, "invalid engine value"): - return PrioritizedError{ - Message: message, - Severity: SeverityCritical, - Category: "configuration", - Suggestion: "Use a supported engine name in frontmatter, for example `engine: copilot`.", - } + return newPrioritizedError(message, SeverityCritical, "configuration", "Use a supported engine name in frontmatter, for example `engine: copilot`.") case strings.Contains(lower, "field 'engine'") && (strings.Contains(lower, "empty") || strings.Contains(lower, "required")): - return PrioritizedError{ - Message: message, - Severity: SeverityCritical, - Category: "configuration", - Suggestion: "Add an `engine:` value to the workflow frontmatter before fixing lower-priority issues.", - } + return newPrioritizedError(message, SeverityCritical, "configuration", "Add an `engine:` value to the workflow frontmatter before fixing lower-priority issues.") case strings.Contains(lower, "network.allowed"), (strings.Contains(lower, "network") && strings.Contains(lower, "strict mode")): - return PrioritizedError{ - Message: message, - Severity: SeverityHigh, - Category: "permissions", - Suggestion: "Either enable strict mode for the workflow or remove the unsupported network configuration.", - } + return newPrioritizedError(message, SeverityHigh, "permissions", "Either enable strict mode for the workflow or remove the unsupported network configuration.") case strings.Contains(lower, "mcp"), strings.Contains(lower, "tool configuration"), strings.Contains(lower, "tools."), strings.Contains(lower, "tools/"): - return PrioritizedError{ - Message: message, - Severity: SeverityHigh, - Category: "tools", - Suggestion: "Check the `tools:` and MCP server configuration for missing required fields or unsupported values.", - } + return newPrioritizedError(message, SeverityHigh, "tools", "Check the `tools:` and MCP server configuration for missing required fields or unsupported values.") case isUnknownPermissionScopeMessage(message): - return PrioritizedError{ - Message: message, - Severity: SeverityMedium, - Category: "permissions", - Suggestion: buildPermissionSuggestion(message), - } + return newPrioritizedError(message, SeverityMedium, "permissions", buildPermissionSuggestion(message)) case strings.Contains(lower, "event"), strings.Contains(lower, "workflow_dispatch"), strings.Contains(lower, "pull-request"), strings.Contains(lower, "pull_request"): - return PrioritizedError{ - Message: message, - Severity: SeverityMedium, - Category: "events", - Suggestion: "Correct the event or filter name, then re-run compilation.", - } + return newPrioritizedError(message, SeverityMedium, "events", "Correct the event or filter name, then re-run compilation.") case strings.Contains(lower, "permission"): - return PrioritizedError{ - Message: message, - Severity: SeverityMedium, - Category: "permissions", - Suggestion: "Adjust the permissions block to match the workflow's required scopes.", - } + return newPrioritizedError(message, SeverityMedium, "permissions", "Adjust the permissions block to match the workflow's required scopes.") case strings.Contains(lower, "runtime"), strings.Contains(lower, "node version"), strings.Contains(lower, "python version"), strings.Contains(lower, "version conflict"): - return PrioritizedError{ - Message: message, - Severity: SeverityMedium, - Category: "runtime", - Suggestion: "Resolve the runtime version conflict or choose a supported version.", - } + return newPrioritizedError(message, SeverityMedium, "runtime", "Resolve the runtime version conflict or choose a supported version.") case strings.Contains(lower, "deprecated"), strings.Contains(lower, "warning"), strings.Contains(lower, "recommend"): - return PrioritizedError{ - Message: message, - Severity: SeverityLow, - Category: "deprecation", - Suggestion: "Clean this up after the higher-priority errors are fixed.", - } + return newPrioritizedError(message, SeverityLow, "deprecation", "Clean this up after the higher-priority errors are fixed.") default: - return PrioritizedError{ - Message: message, - Severity: SeverityMedium, - Category: "configuration", - Suggestion: "Fix this configuration issue and re-run `gh aw compile`.", - } + return newPrioritizedError(message, SeverityMedium, "configuration", "Fix this configuration issue and re-run `gh aw compile`.") + } +} + +func isFrontmatterSyntaxMessage(lower string) bool { + return strings.Contains(lower, "failed to parse frontmatter") || + strings.Contains(lower, "failed to parse yaml frontmatter") || + strings.Contains(lower, "no frontmatter found") || + strings.Contains(lower, "mapping values are not allowed") || + strings.Contains(lower, "did not find expected key") || + strings.Contains(lower, "missing ':' after key") || + strings.Contains(lower, "unexpected ':'") || + strings.Contains(lower, "tool config must be a mapping") +} + +func newPrioritizedError(message string, severity ErrorSeverity, category string, suggestion string) PrioritizedError { + return PrioritizedError{ + Message: message, + Severity: severity, + Category: category, + Suggestion: suggestion, } } diff --git a/pkg/workflow/evals_job.go b/pkg/workflow/evals_job.go index 72aca454f38..c2bd249d700 100644 --- a/pkg/workflow/evals_job.go +++ b/pkg/workflow/evals_job.go @@ -30,57 +30,67 @@ func (c *Compiler) buildEvalsJob(data *WorkflowData) (*Job, error) { var steps []string - // Add setup action steps (installs the agentic engine helper scripts). - setupActionRef := c.resolveActionReference("./actions/setup", data) - if setupActionRef != "" || c.actionMode.IsScript() { - // For dev mode (local action path), checkout the actions folder first. - steps = append(steps, c.generateCheckoutActionsFolder(data)...) - // Reuse the activation job trace ID so all jobs share one OTLP trace. - evalsTraceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) - evalsParentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) - steps = append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, evalsTraceID, evalsParentSpanID)...) - } - - // Download agent output artifact to access output files (prompt.txt, agent_output.json). - // Use activation-derived prefix since this job always depends on activation. + steps = append(steps, c.buildEvalsSetupSteps(data)...) agentArtifactPrefix := artifactPrefixExprForDownstreamJob(data) steps = append(steps, buildAgentOutputDownloadSteps(agentArtifactPrefix, c.getActionPin)...) - - // Download experiment artifact so the evals agent can read the current variant assignments. steps = append(steps, buildExperimentArtifactDownloadSteps(data, c.getActionPin)...) - - // Add all evals steps: engine install, engine execution, parse, redact, upload. steps = append(steps, c.buildEvalsJobSteps(data)...) - // Determine job dependencies. - // Evals always depends on agent and activation, and additionally on detection when the detection job is enabled. - // This allows evals to run in parallel with safe_outputs. + needs := evalsJobNeeds(data) + evalsJobLog.Printf("Evals job dependencies resolved: needs=%v", needs) + + job := &Job{ + Name: string(constants.EvalsJobName), + Needs: needs, + If: evalsJobCondition(), + RunsOn: c.indentYAMLLines(evalsJobRunsOn(data), " "), + Environment: c.indentYAMLLines(data.Environment, " "), + Permissions: evalsJobPermissions(data), + Steps: steps, + } + + return job, nil +} + +func (c *Compiler) buildEvalsSetupSteps(data *WorkflowData) []string { + setupActionRef := c.resolveActionReference("./actions/setup", data) + if setupActionRef == "" && !c.actionMode.IsScript() { + return nil + } + var steps []string + steps = append(steps, c.generateCheckoutActionsFolder(data)...) + evalsTraceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) + evalsParentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) + return append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, evalsTraceID, evalsParentSpanID)...) +} + +func evalsJobNeeds(data *WorkflowData) []string { needs := []string{string(constants.AgentJobName), string(constants.ActivationJobName)} if IsDetectionJobEnabled(data.SafeOutputs) { needs = append(needs, string(constants.DetectionJobName)) } - evalsJobLog.Printf("Evals job dependencies resolved: needs=%v", needs) + return needs +} - // Evals job condition: always run but skip if the agent job was skipped. - // This matches the detection job pattern so conclusion still sees a non-skipped evals result. +func evalsJobCondition() string { alwaysFunc := BuildFunctionCall("always") upstreamNotSkipped := BuildNotEquals( BuildPropertyAccess(fmt.Sprintf("needs.%s.result", constants.AgentJobName)), BuildStringLiteral("skipped"), ) - jobConditionNode := BuildAnd(alwaysFunc, upstreamNotSkipped) - jobCondition := RenderCondition(jobConditionNode) + return RenderCondition(BuildAnd(alwaysFunc, upstreamNotSkipped)) +} - // Determine runs-on: use evals override if set, otherwise ubuntu-latest. - runsOn := "runs-on: ubuntu-latest" +func evalsJobRunsOn(data *WorkflowData) string { if data.Evals != nil && data.Evals.RunsOn != "" { - runsOn = normalizeRunsOnSnippet(data.Evals.RunsOn) + return normalizeRunsOnSnippet(data.Evals.RunsOn) } + return "runs-on: ubuntu-latest" +} - // Determine permissions for the evals job (same rationale as the detection job). - copilotRequestsEnabled := hasCopilotRequestsWritePermission(data) +func evalsJobPermissions(data *WorkflowData) string { perms := NewPermissionsContentsRead() - if copilotRequestsEnabled { + if hasCopilotRequestsWritePermission(data) { perms.Set(PermissionCopilotRequests, PermissionWrite) } if data.EngineConfig != nil && data.EngineConfig.Auth != nil && data.EngineConfig.Auth.Type == "github-oidc" { @@ -89,19 +99,7 @@ func (c *Compiler) buildEvalsJob(data *WorkflowData) (*Job, error) { if hasOTLPGitHubOIDCAuth(data.ParsedFrontmatter, data.RawFrontmatter) { perms.Set(PermissionIdToken, PermissionWrite) } - permissions := perms.RenderToYAML() - - job := &Job{ - Name: string(constants.EvalsJobName), - Needs: needs, - If: jobCondition, - RunsOn: c.indentYAMLLines(runsOn, " "), - Environment: c.indentYAMLLines(data.Environment, " "), - Permissions: permissions, - Steps: steps, - } - - return job, nil + return perms.RenderToYAML() } // buildPushEvalsStateJob creates a job that downloads the evals artifact and commits it to a @@ -115,36 +113,60 @@ func (c *Compiler) buildPushEvalsStateJob(data *WorkflowData) (*Job, error) { var steps []string - setupActionRef := c.resolveActionReference("./actions/setup", data) - if setupActionRef != "" || c.actionMode.IsScript() { - steps = append(steps, c.generateCheckoutActionsFolder(data)...) - traceID := fmt.Sprintf("${{ needs.%s.outputs.setup-trace-id }}", constants.ActivationJobName) - parentSpanID := setupParentSpanNeedsExpr(constants.ActivationJobName) - steps = append(steps, c.generateSetupStep(data, setupActionRef, SetupActionDestination, false, traceID, parentSpanID)...) + steps = append(steps, c.buildEvalsSetupSteps(data)...) + steps = append(steps, buildPushEvalsCheckoutSteps()...) + steps = append(steps, c.generateGitConfigurationSteps()...) + steps = append(steps, c.buildPushEvalsDownloadSteps(data)...) + branchName := evalsBranchName(data.WorkflowID) + steps = append(steps, buildPushEvalsStateSteps(data, branchName)...) + + if c.actionMode.IsDev() { + steps = append(steps, c.generateRestoreActionsSetupStep()) + } + + evalsFinished := BuildNotEquals( + BuildPropertyAccess(fmt.Sprintf("needs.%s.result", constants.EvalsJobName)), + BuildStringLiteral("skipped"), + ) + notCancelled := &NotNode{Child: BuildFunctionCall("cancelled")} + jobCondition := RenderCondition(BuildAnd(BuildAnd(BuildFunctionCall("always"), notCancelled), evalsFinished)) + + job := &Job{ + Name: pushEvalsStateJobName, + RunsOn: c.formatFrameworkJobRunsOn(data), + If: jobCondition, + Permissions: "permissions:\n contents: write", + Needs: []string{string(constants.EvalsJobName), string(constants.ActivationJobName)}, + Steps: steps, } - steps = append(steps, + return job, nil +} + +func buildPushEvalsCheckoutSteps() []string { + return []string{ " - name: Checkout repository\n", fmt.Sprintf(" uses: %s\n", getActionPin("actions/checkout")), " with:\n", " persist-credentials: false\n", " sparse-checkout: .\n", - ) - - steps = append(steps, c.generateGitConfigurationSteps()...) + } +} +func (c *Compiler) buildPushEvalsDownloadSteps(data *WorkflowData) []string { evalsArtifactName := artifactPrefixExprForDownstreamJob(data) + constants.EvalsArtifactName - steps = append(steps, + return []string{ " - name: Download evals artifact\n", fmt.Sprintf(" uses: %s\n", c.getActionPin("actions/download-artifact")), " continue-on-error: true\n", " with:\n", fmt.Sprintf(" name: %s\n", evalsArtifactName), fmt.Sprintf(" path: %s\n", evalsStateDir), - ) + } +} - branchName := evalsBranchName(data.WorkflowID) - steps = append(steps, +func buildPushEvalsStateSteps(data *WorkflowData, branchName string) []string { + return []string{ " - name: Push evals results to git\n", " id: push_evals_state\n", " if: always()\n", @@ -159,31 +181,9 @@ func (c *Compiler) buildPushEvalsStateJob(data *WorkflowData) (*Job, error) { " GH_AW_STATE_LABEL: evals results\n", " with:\n", " script: |\n", - " const { setupGlobals } = require('"+SetupActionDestination+"/setup_globals.cjs');\n", + " const { setupGlobals } = require('" + SetupActionDestination + "/setup_globals.cjs');\n", " setupGlobals(core, github, context, exec, io, getOctokit);\n", - " const { main } = require('"+SetupActionDestination+"/push_experiment_state.cjs');\n", + " const { main } = require('" + SetupActionDestination + "/push_experiment_state.cjs');\n", " await main();\n", - ) - - if c.actionMode.IsDev() { - steps = append(steps, c.generateRestoreActionsSetupStep()) } - - evalsFinished := BuildNotEquals( - BuildPropertyAccess(fmt.Sprintf("needs.%s.result", constants.EvalsJobName)), - BuildStringLiteral("skipped"), - ) - notCancelled := &NotNode{Child: BuildFunctionCall("cancelled")} - jobCondition := RenderCondition(BuildAnd(BuildAnd(BuildFunctionCall("always"), notCancelled), evalsFinished)) - - job := &Job{ - Name: pushEvalsStateJobName, - RunsOn: c.formatFrameworkJobRunsOn(data), - If: jobCondition, - Permissions: "permissions:\n contents: write", - Needs: []string{string(constants.EvalsJobName), string(constants.ActivationJobName)}, - Steps: steps, - } - - return job, nil } diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 0bdd199fd2b..c57395135d3 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -138,21 +138,6 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { // Determine engine ID (same resolution order as detection). engineID := c.getEvalsEngineID(data) - // Build the evals engine config by shallow-copying the main engine config. - // This preserves all fields including Auth (OIDC/Azure), LLMProvider, permissions, - // token weights, and other engine settings that should apply to eval runs. - var evalsEngineConfig *EngineConfig - if data.EngineConfig == nil { - evalsEngineConfig = &EngineConfig{ID: engineID} - } else { - // Shallow copy all fields from the main engine config - copy := *data.EngineConfig - evalsEngineConfig = © - if evalsEngineConfig.ID == "" { - evalsEngineConfig.ID = engineID - } - } - // Apply engine and enterprise default detection model (cost-effective for Q&A tasks). engine, err := c.getAgenticEngine(engineID) if err != nil { @@ -161,31 +146,44 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { } } - // Inherit APITarget from the main engine config for GHE/custom endpoints. - if evalsEngineConfig.APITarget == "" && data.EngineConfig != nil && data.EngineConfig.APITarget != "" { - evalsEngineConfig.APITarget = data.EngineConfig.APITarget - } - + evalsEngineConfig := c.buildEvalsEngineConfig(data, engineID) evalsEngineConfig.Model = c.resolveEvalsExecutionModel(data) + evalsData := buildEvalsWorkflowData(data, evalsEngineConfig, engineID) + + var steps []string + steps = append(steps, buildEvalsEngineInstallSteps(engine, evalsData)...) + steps = append(steps, c.buildEvalsMCPSetupSteps(engine, evalsData)...) + steps = append(steps, buildEvalsEngineExecutionSteps(engine, evalsData)...) + + return steps +} + +func (c *Compiler) buildEvalsEngineConfig(data *WorkflowData, engineID string) *EngineConfig { + if data.EngineConfig == nil { + return &EngineConfig{ID: engineID} + } + copy := *data.EngineConfig + if copy.ID == "" { + copy.ID = engineID + } + if copy.APITarget == "" && data.EngineConfig.APITarget != "" { + copy.APITarget = data.EngineConfig.APITarget + } + return © +} - // Build a minimal WorkflowData for evals engine execution. - // IsDetectionRun reuses detection-style network restrictions and MaxAI credits, - // which are appropriate for binary (YES/NO) evaluation tasks. - // RunnerConfig is propagated from the main workflow data so that arc-dind topology - // handling (daemon-visible Copilot staging step + daemon-visible spawn path) applies - // to the evals job the same way it applies to the agent job. - evalsData := &WorkflowData{ +func buildEvalsWorkflowData(data *WorkflowData, evalsEngineConfig *EngineConfig, engineID string) *WorkflowData { + return &WorkflowData{ Tools: map[string]any{ "bash": []any{"*"}, }, - SafeOutputs: nil, EngineConfig: evalsEngineConfig, AI: engineID, Features: data.Features, Permissions: data.Permissions, CachedPermissions: data.CachedPermissions, IsDetectionRun: true, - RunnerConfig: data.RunnerConfig, // propagate runner.topology (e.g. arc-dind) to the evals job + RunnerConfig: data.RunnerConfig, NetworkPermissions: &NetworkPermissions{ Allowed: getThreatDetectionAdditionalAllowedDomains(data), }, @@ -195,14 +193,11 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { }, }, } +} +func buildEvalsEngineInstallSteps(engine CodingAgentEngine, evalsData *WorkflowData) []string { var steps []string - - // Install the engine binary (fresh runner has no engine installed). installSteps := engine.GetInstallationSteps(evalsData) - - // Ensure Node.js is on PATH when the engine harness requires it. - // Guard against engines whose install steps already bundle Setup Node.js. if engineRequiresNodeHarness(engine) && !installStepsContainNodeSetup(installSteps) { for _, line := range GenerateNodeJsSetupStep() { steps = append(steps, line+"\n") @@ -213,8 +208,11 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { steps = append(steps, line+"\n") } } + return steps +} - // Codex requires MCP gateway config (OpenAI proxy provider in config.toml). +func (c *Compiler) buildEvalsMCPSetupSteps(engine CodingAgentEngine, evalsData *WorkflowData) []string { + var steps []string if engine.GetID() == "codex" { var mcpSetup strings.Builder if err := c.generateMCPSetup(&mcpSetup, evalsData.Tools, engine, evalsData); err == nil { @@ -227,32 +225,30 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { evalsStepsLog.Printf("Failed to generate MCP setup for Codex evals; OpenAI proxy configuration may be incomplete: %v", err) } } + return steps +} - // Execute the engine through AWF; output is written to evalsLogPath. +func buildEvalsEngineExecutionSteps(engine CodingAgentEngine, evalsData *WorkflowData) []string { + var steps []string executionSteps := engine.GetExecutionSteps(evalsData, evalsLogPath) for _, step := range executionSteps { - // Track whether we've injected the if/continue-on-error fields yet - injected := false - for _, line := range step { - // Prefix the agentic_execution step ID to avoid collisions with the agent job step - // IDs — job managers validate for duplicate step IDs across the compiled YAML. - // This mirrors the same pattern used in buildDetectionEngineExecutionStep (see - // threat_detection_inline_engine.go), where the ID is also a well-known literal - // produced by every engine's GetExecutionSteps implementation. - prefixed := strings.Replace(line, "id: agentic_execution", "id: evals_agentic_execution", 1) - steps = append(steps, prefixed+"\n") - // Inject always() condition and continue-on-error after the "- name:" line - // so that infrastructure failures do not block the parse step that follows. - // Search for the name field instead of assuming it's always at index 0 to handle - // engines that might emit comments or other fields before the name. - if !injected && strings.Contains(strings.TrimSpace(line), "- name:") { - steps = append(steps, " if: always()\n") - steps = append(steps, " continue-on-error: true\n") - injected = true - } - } + steps = append(steps, buildSingleEvalsExecutionStep(step)...) } + return steps +} +func buildSingleEvalsExecutionStep(step []string) []string { + var steps []string + injected := false + for _, line := range step { + prefixed := strings.Replace(line, "id: agentic_execution", "id: evals_agentic_execution", 1) + steps = append(steps, prefixed+"\n") + if !injected && strings.Contains(strings.TrimSpace(line), "- name:") { + steps = append(steps, " if: always()\n") + steps = append(steps, " continue-on-error: true\n") + injected = true + } + } return steps } diff --git a/pkg/workflow/event_validation.go b/pkg/workflow/event_validation.go index 7480e2c26d8..4620bf3b652 100644 --- a/pkg/workflow/event_validation.go +++ b/pkg/workflow/event_validation.go @@ -112,12 +112,24 @@ func ValidateEventTypes(frontmatter map[string]any) error { eventValidationLog.Print("No 'on' section found, skipping event type validation") return nil } + eventNames, ok := eventNamesFromOnSection(on) + if !ok { + eventValidationLog.Printf("'on' section has unexpected type %T, skipping event type validation", on) + return nil + } + for _, eventName := range eventNames { + if err := validateOneEventType(eventName); err != nil { + return err + } + } + return nil +} - // Extract event names from the on: section (handles string, []any, map formats) +func eventNamesFromOnSection(on any) ([]string, bool) { var eventNames []string switch v := on.(type) { case string: - eventNames = []string{v} + return []string{v}, true case []any: for _, item := range v { if s, ok := item.(string); ok { @@ -129,51 +141,39 @@ func ValidateEventTypes(frontmatter map[string]any) error { eventNames = append(eventNames, key) } default: - eventValidationLog.Printf("'on' section has unexpected type %T, skipping event type validation", on) - return nil + return nil, false } + return eventNames, true +} - // Check each event name against the list of known valid events - for _, eventName := range eventNames { - if isKnownGitHubEvent(eventName) { - continue - } - - // Skip gh-aw-specific on: section extensions - if ghAwOnSectionKeys[eventName] { - eventValidationLog.Printf("Skipping gh-aw extension key: %q", eventName) - continue - } - - eventValidationLog.Printf("Unknown event type: %q", eventName) - - // Check for a case-only difference first (e.g. "Push" → "push") - lowerEventName := strings.ToLower(eventName) - if lowerEventName != eventName && isKnownGitHubEvent(lowerEventName) { - return fmt.Errorf( - "unknown event type %q in 'on:' section.\n\nDid you mean: %s?\n\nValid event types include: %s\n\nSee: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows", - eventName, - lowerEventName, - strings.Join(validGitHubEventTypes[:10], ", ")+"...", - ) - } - - // Only flag as a typo when there is a close match - suggestions := stringutil.FindClosestMatches(eventName, validGitHubEventTypes, 3) - if len(suggestions) == 0 { - eventValidationLog.Printf("No close matches found for unknown event %q, skipping", eventName) - continue - } - - return fmt.Errorf( - "unknown event type %q in 'on:' section.\n\nDid you mean: %s?\n\nValid event types include: %s\n\nSee: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows", - eventName, - strings.Join(suggestions, ", "), - strings.Join(validGitHubEventTypes[:10], ", ")+"...", - ) +func validateOneEventType(eventName string) error { + if isKnownGitHubEvent(eventName) { + return nil } + if ghAwOnSectionKeys[eventName] { + eventValidationLog.Printf("Skipping gh-aw extension key: %q", eventName) + return nil + } + eventValidationLog.Printf("Unknown event type: %q", eventName) + lowerEventName := strings.ToLower(eventName) + if lowerEventName != eventName && isKnownGitHubEvent(lowerEventName) { + return unknownEventTypeError(eventName, lowerEventName) + } + suggestions := stringutil.FindClosestMatches(eventName, validGitHubEventTypes, 3) + if len(suggestions) == 0 { + eventValidationLog.Printf("No close matches found for unknown event %q, skipping", eventName) + return nil + } + return unknownEventTypeError(eventName, strings.Join(suggestions, ", ")) +} - return nil +func unknownEventTypeError(eventName, suggestion string) error { + return fmt.Errorf( + "unknown event type %q in 'on:' section.\n\nDid you mean: %s?\n\nValid event types include: %s\n\nSee: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows", + eventName, + suggestion, + strings.Join(validGitHubEventTypes[:10], ", ")+"...", + ) } // isKnownGitHubEvent returns true if the event name is in the list of valid GitHub Actions event types. diff --git a/pkg/workflow/expression_optimizer.go b/pkg/workflow/expression_optimizer.go index 60c976f1fcb..bd145ec2878 100644 --- a/pkg/workflow/expression_optimizer.go +++ b/pkg/workflow/expression_optimizer.go @@ -207,21 +207,14 @@ func termSubsumedBy(cand, sub ConditionNode) bool { // --- node-specific optimisers ------------------------------------------------ func optimizeAndNode(n *AndNode) ConditionNode { - // Bottom-up: optimise children first. left := optimizeNode(n.Left) right := optimizeNode(n.Right) - - // Annihilation: A && false → false (before flattening for early exit). if isBoolLiteral(left, false) || isBoolLiteral(right, false) { expressionOptimizerLog.Printf("AND annihilation: %s && %s → false", left.Render(), right.Render()) return &BooleanLiteralNode{Value: false} } - // Flatten the entire AND chain so that rules can operate across nesting levels. - // e.g. A && (A && B) → [A, A, B] → dedup → [A, B] → A && B terms := collectAndTerms(&AndNode{Left: left, Right: right}) - - // Annihilation within the flat list (covers cases after child optimisation). for _, t := range terms { if isBoolLiteral(t, false) { expressionOptimizerLog.Printf("AND annihilation (flatten): false term → false") @@ -229,9 +222,27 @@ func optimizeAndNode(n *AndNode) ConditionNode { } } - // Identity: filter out `true` literals, but keep them when any term is a - // status function (to preserve status-function semantics). hasStatusFuncInTerms := slices.ContainsFunc(terms, containsStatusFunc) + filtered := filterAndIdentityTerms(terms, hasStatusFuncInTerms) + if len(filtered) == 0 { + return &BooleanLiteralNode{Value: true} + } + deduped := dedupeRenderedTerms(filtered, "AND") + if len(deduped) == 1 { + return deduped[0] + } + if !hasStatusFuncInTerms && hasComplementTerms(deduped, "AND complement (flatten)") { + return &BooleanLiteralNode{Value: false} + } + if !hasStatusFuncInTerms { + if absorbed := optimizeAndAbsorption(deduped); absorbed != nil { + return absorbed + } + } + return rebuildAndChain(deduped) +} + +func filterAndIdentityTerms(terms []ConditionNode, hasStatusFuncInTerms bool) []ConditionNode { filtered := make([]ConditionNode, 0, len(terms)) for _, t := range terms { if isBoolLiteral(t, true) && !hasStatusFuncInTerms { @@ -240,148 +251,137 @@ func optimizeAndNode(n *AndNode) ConditionNode { } filtered = append(filtered, t) } - if len(filtered) == 0 { - return &BooleanLiteralNode{Value: true} - } + return filtered +} - // Deduplicate terms by rendered form (safe even for status functions). - seen := make(map[string]struct{}, len(filtered)) - deduped := make([]ConditionNode, 0, len(filtered)) - for _, t := range filtered { - key := t.Render() +func dedupeRenderedTerms(terms []ConditionNode, label string) []ConditionNode { + seen := make(map[string]struct{}, len(terms)) + deduped := make([]ConditionNode, 0, len(terms)) + for _, term := range terms { + key := term.Render() if _, exists := seen[key]; !exists { seen[key] = struct{}{} - deduped = append(deduped, t) + deduped = append(deduped, term) } else { - expressionOptimizerLog.Printf("AND dedup: removing duplicate term %q", key) + expressionOptimizerLog.Printf("%s dedup: removing duplicate term %q", label, key) } } - if len(deduped) == 1 { - return deduped[0] - } + return deduped +} - // Complement: A && !A → false (skip when status functions present). - if !hasStatusFuncInTerms { - for i := range deduped { - for j := i + 1; j < len(deduped); j++ { - if isNegationOf(deduped[i], deduped[j]) { - expressionOptimizerLog.Printf("AND complement (flatten): %s && %s → false", deduped[i].Render(), deduped[j].Render()) - return &BooleanLiteralNode{Value: false} - } +func hasComplementTerms(terms []ConditionNode, label string) bool { + for i := range terms { + for j := i + 1; j < len(terms); j++ { + if isNegationOf(terms[i], terms[j]) { + expressionOptimizerLog.Printf("%s: %s && %s → false", label, terms[i].Render(), terms[j].Render()) + return true } } } + return false +} - // Absorption (AND): A && (A || B) → A - // If any term is an OR/Disjunction that contains another conjunct as a - // sub-term, the OR term is absorbed: the simpler conjunct subsumes it. - if !hasStatusFuncInTerms { - absorbed := make([]bool, len(deduped)) - for i, ti := range deduped { - orTerms := collectOrTerms(ti) - if len(orTerms) < 2 { - continue // ti is not an OR expression - } - for j, tj := range deduped { - if i == j || absorbed[i] { - continue - } - for _, ot := range orTerms { - if nodesEqual(ot, tj) { - expressionOptimizerLog.Printf("AND absorption: (%s) && (%s) → %s (absorbed)", - tj.Render(), ti.Render(), tj.Render()) - absorbed[i] = true - break - } - } - } +func optimizeAndAbsorption(deduped []ConditionNode) ConditionNode { + absorbed := make([]bool, len(deduped)) + for i, ti := range deduped { + orTerms := collectOrTerms(ti) + if len(orTerms) < 2 { + continue } - anyAbsorbed := slices.Contains(absorbed, true) - if anyAbsorbed { - surviving := make([]ConditionNode, 0, len(deduped)) - for i, t := range deduped { - if !absorbed[i] { - surviving = append(surviving, t) - } - } - if len(surviving) == 0 { - return &BooleanLiteralNode{Value: true} + markAndAbsorbedTerm(absorbed, deduped, orTerms, i, ti) + } + if !slices.Contains(absorbed, true) { + return nil + } + surviving := survivingTerms(deduped, absorbed) + if len(surviving) == 0 { + return &BooleanLiteralNode{Value: true} + } + return optimizeNode(rebuildAndChain(surviving)) +} + +func markAndAbsorbedTerm(absorbed []bool, deduped, orTerms []ConditionNode, i int, ti ConditionNode) { + for j, tj := range deduped { + if i == j || absorbed[i] { + continue + } + for _, ot := range orTerms { + if nodesEqual(ot, tj) { + expressionOptimizerLog.Printf("AND absorption: (%s) && (%s) → %s (absorbed)", tj.Render(), ti.Render(), tj.Render()) + absorbed[i] = true + break } - return optimizeNode(rebuildAndChain(surviving)) } } - - return rebuildAndChain(deduped) } func optimizeOrNode(n *OrNode) ConditionNode { - // Bottom-up: optimise children first. left := optimizeNode(n.Left) right := optimizeNode(n.Right) - - // Flatten OR chains: when either child is already an OrNode or DisjunctionNode, - // collect all terms and delegate to the DisjunctionNode optimiser, which - // performs dedup, false-filtering and true short-circuit across the whole chain. - _, leftIsOr := left.(*OrNode) - _, leftIsDisj := left.(*DisjunctionNode) - _, rightIsOr := right.(*OrNode) - _, rightIsDisj := right.(*DisjunctionNode) - if leftIsOr || leftIsDisj || rightIsOr || rightIsDisj { - terms := append(collectOrTerms(left), collectOrTerms(right)...) - expressionOptimizerLog.Printf("OR flatten: collected %d terms", len(terms)) - return optimizeDisjunctionNode(&DisjunctionNode{Terms: terms}) + if flattened := optimizeFlattenedOr(left, right); flattened != nil { + return flattened } - - // Annihilation: A || true → true if isBoolLiteral(left, true) || isBoolLiteral(right, true) { expressionOptimizerLog.Printf("OR annihilation: %s || %s → true", left.Render(), right.Render()) return &BooleanLiteralNode{Value: true} } - - // Identity: A || false → A - if isBoolLiteral(right, false) { - expressionOptimizerLog.Printf("OR identity (right false): %s || false → %s", left.Render(), left.Render()) - return left - } - if isBoolLiteral(left, false) { - expressionOptimizerLog.Printf("OR identity (left false): false || %s → %s", right.Render(), right.Render()) - return right + if identity := optimizeOrIdentity(left, right); identity != nil { + return identity } - - // Skip idempotent / complement rules when status functions are present. if containsStatusFunc(left) || containsStatusFunc(right) { return &OrNode{Left: left, Right: right} } - - // Idempotent: A || A → A if nodesEqual(left, right) { expressionOptimizerLog.Printf("OR idempotent: %s || %s → %s", left.Render(), right.Render(), left.Render()) return left } - - // Complement: A || !A → true if isNegationOf(left, right) { expressionOptimizerLog.Printf("OR complement: %s || %s → true", left.Render(), right.Render()) return &BooleanLiteralNode{Value: true} } + if absorbed := optimizeOrAbsorption(left, right); absorbed != nil { + return absorbed + } + return &OrNode{Left: left, Right: right} +} + +func optimizeFlattenedOr(left, right ConditionNode) ConditionNode { + _, leftIsOr := left.(*OrNode) + _, leftIsDisj := left.(*DisjunctionNode) + _, rightIsOr := right.(*OrNode) + _, rightIsDisj := right.(*DisjunctionNode) + if !leftIsOr && !leftIsDisj && !rightIsOr && !rightIsDisj { + return nil + } + terms := append(collectOrTerms(left), collectOrTerms(right)...) + expressionOptimizerLog.Printf("OR flatten: collected %d terms", len(terms)) + return optimizeDisjunctionNode(&DisjunctionNode{Terms: terms}) +} + +func optimizeOrIdentity(left, right ConditionNode) ConditionNode { + if isBoolLiteral(right, false) { + expressionOptimizerLog.Printf("OR identity (right false): %s || false → %s", left.Render(), left.Render()) + return left + } + if isBoolLiteral(left, false) { + expressionOptimizerLog.Printf("OR identity (left false): false || %s → %s", right.Render(), right.Render()) + return right + } + return nil +} - // Absorption (OR): A || (A && B) → A - // If one side is an AND-chain that contains the other side as a conjunct, - // the AND-chain is absorbed by the simpler operand. +func optimizeOrAbsorption(left, right ConditionNode) ConditionNode { for _, pair := range [][2]ConditionNode{{left, right}, {right, left}} { simple, complex := pair[0], pair[1] if termSubsumedBy(complex, simple) { - expressionOptimizerLog.Printf("OR absorption: %s || (%s) → %s (absorbed)", - simple.Render(), complex.Render(), simple.Render()) + expressionOptimizerLog.Printf("OR absorption: %s || (%s) → %s (absorbed)", simple.Render(), complex.Render(), simple.Render()) return simple } } - - return &OrNode{Left: left, Right: right} + return nil } func optimizeNotNode(n *NotNode) ConditionNode { - // Bottom-up: optimise child first. child := optimizeNode(n.Child) // Constant folding: !true → false, !false → true @@ -446,96 +446,108 @@ func optimizeDisjunctionNode(n *DisjunctionNode) ConditionNode { if len(n.Terms) == 0 { return n } + optimised := optimizeDisjunctionTerms(n.Terms) + if hasTrueDisjunctionTerm(optimised) { + return &BooleanLiteralNode{Value: true} + } + filtered := filterFalseDisjunctionTerms(optimised) + if len(filtered) == 0 { + expressionOptimizerLog.Printf("Disjunction all-false → false") + return &BooleanLiteralNode{Value: false} + } + deduped := dedupeRenderedTerms(filtered, "Disjunction") + if len(deduped) == 1 { + return deduped[0] + } + if !slices.ContainsFunc(deduped, containsStatusFunc) { + if hasDisjunctionComplement(deduped) { + return &BooleanLiteralNode{Value: true} + } + if absorbed := optimizeDisjunctionSubsumption(deduped, n.Multiline); absorbed != nil { + return absorbed + } + } + return &DisjunctionNode{Terms: deduped, Multiline: n.Multiline} +} - // Bottom-up: optimise each term first. - optimised := make([]ConditionNode, 0, len(n.Terms)) - for _, term := range n.Terms { +func optimizeDisjunctionTerms(terms []ConditionNode) []ConditionNode { + optimised := make([]ConditionNode, 0, len(terms)) + for _, term := range terms { optimised = append(optimised, optimizeNode(term)) } + return optimised +} - // Short-circuit: if any term is true the whole disjunction is true. - for _, term := range optimised { +func hasTrueDisjunctionTerm(terms []ConditionNode) bool { + for _, term := range terms { if isBoolLiteral(term, true) { expressionOptimizerLog.Printf("Disjunction short-circuit on true") - return &BooleanLiteralNode{Value: true} + return true } } + return false +} - // Filter out false terms (identity: A || false → A). - filtered := make([]ConditionNode, 0, len(optimised)) - for _, term := range optimised { +func filterFalseDisjunctionTerms(terms []ConditionNode) []ConditionNode { + filtered := make([]ConditionNode, 0, len(terms)) + for _, term := range terms { if !isBoolLiteral(term, false) { filtered = append(filtered, term) } } - if len(filtered) == 0 { - expressionOptimizerLog.Printf("Disjunction all-false → false") - return &BooleanLiteralNode{Value: false} - } + return filtered +} - // Deduplicate terms by rendered form. - seen := make(map[string]struct{}, len(filtered)) - deduped := make([]ConditionNode, 0, len(filtered)) - for _, term := range filtered { - key := term.Render() - if _, exists := seen[key]; !exists { - seen[key] = struct{}{} - deduped = append(deduped, term) - } else { - expressionOptimizerLog.Printf("Disjunction dedup: removing duplicate term %q", key) +func hasDisjunctionComplement(terms []ConditionNode) bool { + for i := range terms { + for j := i + 1; j < len(terms); j++ { + if isNegationOf(terms[i], terms[j]) { + expressionOptimizerLog.Printf("Disjunction complement: %s || %s → true", terms[i].Render(), terms[j].Render()) + return true + } } } + return false +} - if len(deduped) == 1 { - return deduped[0] +func optimizeDisjunctionSubsumption(deduped []ConditionNode, multiline bool) ConditionNode { + subsumed := markDisjunctionSubsumedTerms(deduped) + if !slices.Contains(subsumed, true) { + return nil } + surviving := survivingTerms(deduped, subsumed) + if len(surviving) == 0 { + return &BooleanLiteralNode{Value: false} + } + if len(surviving) == 1 { + return surviving[0] + } + return optimizeNode(&DisjunctionNode{Terms: surviving, Multiline: multiline}) +} - // Complement: A || !A → true (mirrors the OrNode complement rule). - // Guard: skip when any term contains a status function so that - // status-function expressions are never silently eliminated. - if !slices.ContainsFunc(deduped, containsStatusFunc) { - for i := range deduped { - for j := i + 1; j < len(deduped); j++ { - if isNegationOf(deduped[i], deduped[j]) { - expressionOptimizerLog.Printf("Disjunction complement: %s || %s → true", deduped[i].Render(), deduped[j].Render()) - return &BooleanLiteralNode{Value: true} - } - } - } - - // Subsumption: disj(A, A&&B, …) → disj(A, …) - // A term cand is subsumed (and thus redundant in a disjunction) when - // a simpler term sub is also present such that sub |= cand, i.e. every - // assignment that satisfies sub also satisfies cand. - subsumed := make([]bool, len(deduped)) - for i, cand := range deduped { - for j, sub := range deduped { - if i == j { - continue - } - if termSubsumedBy(cand, sub) { - expressionOptimizerLog.Printf("Disjunction subsumption: %s subsumed by %s", cand.Render(), sub.Render()) - subsumed[i] = true - break - } - } - } - if slices.Contains(subsumed, true) { - surviving := make([]ConditionNode, 0, len(deduped)) - for i, t := range deduped { - if !subsumed[i] { - surviving = append(surviving, t) - } - } - if len(surviving) == 0 { - return &BooleanLiteralNode{Value: false} +func markDisjunctionSubsumedTerms(deduped []ConditionNode) []bool { + subsumed := make([]bool, len(deduped)) + for i, cand := range deduped { + for j, sub := range deduped { + if i == j { + continue } - if len(surviving) == 1 { - return surviving[0] + if termSubsumedBy(cand, sub) { + expressionOptimizerLog.Printf("Disjunction subsumption: %s subsumed by %s", cand.Render(), sub.Render()) + subsumed[i] = true + break } - return optimizeNode(&DisjunctionNode{Terms: surviving, Multiline: n.Multiline}) } } + return subsumed +} - return &DisjunctionNode{Terms: deduped, Multiline: n.Multiline} +func survivingTerms(terms []ConditionNode, removed []bool) []ConditionNode { + surviving := make([]ConditionNode, 0, len(terms)) + for i, term := range terms { + if !removed[i] { + surviving = append(surviving, term) + } + } + return surviving } diff --git a/pkg/workflow/expression_parser.go b/pkg/workflow/expression_parser.go index f84125d9ef5..43395a57c3f 100644 --- a/pkg/workflow/expression_parser.go +++ b/pkg/workflow/expression_parser.go @@ -89,101 +89,101 @@ func (p *ExpressionParser) tokenize(expression string) ([]token, error) { i := 0 for i < len(expression) { - // Skip whitespace if unicode.IsSpace(rune(expression[i])) { i++ continue } + if tok, next, ok := tokenizeSimpleOperator(expression, i); ok { + tokens = append(tokens, tok) + i = next + continue + } + literal, next, err := p.tokenizeLiteral(expression, i) + if err != nil { + return nil, err + } + tokens = append(tokens, literal) + i = next + } - switch { - case i+1 < len(expression) && expression[i:i+2] == "&&": - tokens = append(tokens, token{tokenAnd, "&&", i}) - i += 2 - case i+1 < len(expression) && expression[i:i+2] == "||": - tokens = append(tokens, token{tokenOr, "||", i}) - i += 2 - case expression[i] == '!' && (i+1 >= len(expression) || expression[i+1] != '='): - // Only treat ! as NOT if not followed by = (to avoid conflicting with !=) - tokens = append(tokens, token{tokenNot, "!", i}) - i++ - case expression[i] == '(': - tokens = append(tokens, token{tokenLeftParen, "(", i}) - i++ - case expression[i] == ')': - tokens = append(tokens, token{tokenRightParen, ")", i}) - i++ - default: - // Parse literal expression - everything until we hit a logical operator or paren - start := i - parenCount := 0 - - for i < len(expression) { - ch := expression[i] - - // Handle quoted strings - skip everything inside quotes - // Support single quotes ('), double quotes ("), and backticks (`) - if ch == '\'' || ch == '"' || ch == '`' { - quote := ch - i++ // skip opening quote - for i < len(expression) { - if expression[i] == quote { - i++ // skip closing quote - break - } - if expression[i] == '\\' && i+1 < len(expression) { - i += 2 // skip escaped character - } else { - i++ - } - } - continue - } - - // Track parentheses that are part of the expression (e.g., function calls) - if ch == '(' { - parenCount++ - i++ - continue - } else if ch == ')' { - if parenCount > 0 { - parenCount-- - i++ - continue - } else { - // This closes our group expression, stop here - break - } - } - - // Check for logical operators when not inside parentheses - if parenCount == 0 { - // Check for && or || - if i+1 < len(expression) { - next := expression[i : i+2] - if next == "&&" || next == "||" { - break - } - } + tokens = append(tokens, token{tokenEOF, "", i}) + return tokens, nil +} - // Check for logical NOT that's not part of != - if ch == '!' && (i+1 >= len(expression) || expression[i+1] != '=') { - break - } - } +func tokenizeSimpleOperator(expression string, i int) (token, int, bool) { + switch { + case i+1 < len(expression) && expression[i:i+2] == "&&": + return token{tokenAnd, "&&", i}, i + 2, true + case i+1 < len(expression) && expression[i:i+2] == "||": + return token{tokenOr, "||", i}, i + 2, true + case expression[i] == '!' && (i+1 >= len(expression) || expression[i+1] != '='): + return token{tokenNot, "!", i}, i + 1, true + case expression[i] == '(': + return token{tokenLeftParen, "(", i}, i + 1, true + case expression[i] == ')': + return token{tokenRightParen, ")", i}, i + 1, true + default: + return token{}, i, false + } +} - i++ +func (p *ExpressionParser) tokenizeLiteral(expression string, start int) (token, int, error) { + i := start + parenCount := 0 + for i < len(expression) { + ch := expression[i] + if ch == '\'' || ch == '"' || ch == '`' { + i = skipQuotedExpression(expression, i) + continue + } + if ch == '(' { + parenCount++ + i++ + continue + } else if ch == ')' { + if parenCount == 0 { + break } + parenCount-- + i++ + continue + } + if parenCount == 0 && isExpressionTokenBoundary(expression, i) { + break + } + i++ + } + literal := strings.TrimSpace(expression[start:i]) + if literal == "" { + return token{}, i, fmt.Errorf("unexpected empty literal at position %d", start) + } + return token{tokenLiteral, literal, start}, i, nil +} - literal := strings.TrimSpace(expression[start:i]) - if literal == "" { - return nil, fmt.Errorf("unexpected empty literal at position %d", start) - } - tokens = append(tokens, token{tokenLiteral, literal, start}) +func skipQuotedExpression(expression string, i int) int { + quote := expression[i] + i++ + for i < len(expression) { + if expression[i] == quote { + return i + 1 + } + if expression[i] == '\\' && i+1 < len(expression) { + i += 2 + } else { + i++ } } + return i +} - tokens = append(tokens, token{tokenEOF, "", i}) - return tokens, nil +func isExpressionTokenBoundary(expression string, i int) bool { + if i+1 < len(expression) { + next := expression[i : i+2] + if next == "&&" || next == "||" { + return true + } + } + return expression[i] == '!' && (i+1 >= len(expression) || expression[i+1] != '=') } // parseOrExpression parses OR expressions (lowest precedence) @@ -339,25 +339,7 @@ func BreakLongExpression(expression string) []string { // Handle quoted strings - don't break inside quotes // Support single quotes ('), double quotes ("), and backticks (`) if char == '\'' || char == '"' || char == '`' { - quote := char - current.WriteByte(char) - i++ - - // Continue until closing quote - for i < len(expression) { - current.WriteByte(expression[i]) - if expression[i] == quote { - i++ - break - } - if expression[i] == '\\' && i+1 < len(expression) { - i++ // Skip escaped character - if i < len(expression) { - current.WriteByte(expression[i]) - } - } - i++ - } + i = writeQuotedExpression(¤t, expression, i) continue } @@ -391,7 +373,30 @@ func BreakLongExpression(expression string) []string { lines = append(lines, trimmed) } - // If we still have very long lines, try to break at parentheses + return breakLongExpressionLinesAtParentheses(lines) +} + +func writeQuotedExpression(current *strings.Builder, expression string, i int) int { + quote := expression[i] + current.WriteByte(expression[i]) + i++ + for i < len(expression) { + current.WriteByte(expression[i]) + if expression[i] == quote { + return i + 1 + } + if expression[i] == '\\' && i+1 < len(expression) { + i++ + if i < len(expression) { + current.WriteByte(expression[i]) + } + } + i++ + } + return i +} + +func breakLongExpressionLinesAtParentheses(lines []string) []string { var finalLines []string for _, line := range lines { if len(line) > int(constants.MaxExpressionLineLength) { diff --git a/pkg/workflow/expression_safety_validation.go b/pkg/workflow/expression_safety_validation.go index 674b29d5851..70db7b7f74d 100644 --- a/pkg/workflow/expression_safety_validation.go +++ b/pkg/workflow/expression_safety_validation.go @@ -35,105 +35,96 @@ var ( // are in the allowed list and returns an error if any unauthorized expressions are found func validateExpressionSafety(markdownContent string) error { expressionValidationLog.Print("Validating expression safety in markdown content") - matches := ExpressionPatternDotAll.FindAllStringSubmatch(markdownContent, -1) expressionValidationLog.Printf("Found %d expressions to validate", len(matches)) var unauthorizedExpressions []string - for _, match := range matches { if len(match) < 2 { continue } - - // Extract the expression content (everything between ${{ and }}) - expression := strings.TrimSpace(match[1]) - - // Reject expressions that span multiple lines (contain newlines) - if strings.Contains(match[1], "\n") { - unauthorizedExpressions = append(unauthorizedExpressions, expression) - continue + if err := validateExpressionMatch(match[1], &unauthorizedExpressions); err != nil { + return err } + } + if len(unauthorizedExpressions) > 0 { + return expressionSafetyError(unauthorizedExpressions) + } + expressionValidationLog.Print("Expression safety validation passed") + return nil +} - // Try to parse the expression using the parser - parsed, parseErr := ParseExpression(expression) - if parseErr == nil { - // If we can parse it, validate each literal expression in the tree - validationErr := VisitExpressionTree(parsed, func(expr *ExpressionNode) error { - return validateSingleExpression(expr.Expression, ExpressionValidationOptions{ - NeedsStepsRe: NeedsStepsPattern, - InputsRe: InputsPattern, - WorkflowCallInputsRe: WorkflowCallInputsPattern, - AwInputsRe: AWInputsPattern, - AwImportInputsRe: AWImportInputsPattern, - EnvRe: EnvPattern, - UnauthorizedExpressions: &unauthorizedExpressions, - }) - }) - if validationErr != nil { - return validationErr - } - } else { - // If parsing fails, fall back to validating the whole expression as a literal - err := validateSingleExpression(expression, ExpressionValidationOptions{ - NeedsStepsRe: NeedsStepsPattern, - InputsRe: InputsPattern, - WorkflowCallInputsRe: WorkflowCallInputsPattern, - AwInputsRe: AWInputsPattern, - AwImportInputsRe: AWImportInputsPattern, - EnvRe: EnvPattern, - UnauthorizedExpressions: &unauthorizedExpressions, - }) - if err != nil { - return err - } - } +func validateExpressionMatch(rawExpression string, unauthorizedExpressions *[]string) error { + expression := strings.TrimSpace(rawExpression) + if strings.Contains(rawExpression, "\n") { + *unauthorizedExpressions = append(*unauthorizedExpressions, expression) + return nil + } + opts := defaultExpressionValidationOptions(unauthorizedExpressions) + parsed, parseErr := ParseExpression(expression) + if parseErr != nil { + return validateSingleExpression(expression, opts) } + return VisitExpressionTree(parsed, func(expr *ExpressionNode) error { + return validateSingleExpression(expr.Expression, opts) + }) +} - if len(unauthorizedExpressions) > 0 { - expressionValidationLog.Printf("Expression safety validation failed: %d unauthorized expressions found", len(unauthorizedExpressions)) - var unauthorizedList strings.Builder - unauthorizedList.WriteString("\n") - for _, expr := range unauthorizedExpressions { - unauthorizedList.WriteString(" - ") - unauthorizedList.WriteString(expr) +func defaultExpressionValidationOptions(unauthorizedExpressions *[]string) ExpressionValidationOptions { + return ExpressionValidationOptions{ + NeedsStepsRe: NeedsStepsPattern, + InputsRe: InputsPattern, + WorkflowCallInputsRe: WorkflowCallInputsPattern, + AwInputsRe: AWInputsPattern, + AwImportInputsRe: AWImportInputsPattern, + EnvRe: EnvPattern, + UnauthorizedExpressions: unauthorizedExpressions, + } +} - // Find closest matches using fuzzy string matching - closestMatches := parser.FindClosestMatches(expr, constants.AllowedExpressions, maxFuzzyMatchSuggestions) - if len(closestMatches) > 0 { - unauthorizedList.WriteString(" (did you mean: ") - unauthorizedList.WriteString(strings.Join(closestMatches, ", ")) - unauthorizedList.WriteString("?)") - } +func expressionSafetyError(unauthorizedExpressions []string) error { + expressionValidationLog.Printf("Expression safety validation failed: %d unauthorized expressions found", len(unauthorizedExpressions)) + return NewValidationError( + "expressions", + fmt.Sprintf("%d unauthorized expressions found", len(unauthorizedExpressions)), + "expressions are not in the allowed list:"+unauthorizedExpressionList(unauthorizedExpressions), + fmt.Sprintf("Use only allowed expressions:%s\nFor more details, see the expression security documentation.", allowedExpressionList()), + ) +} - unauthorizedList.WriteString("\n") +func unauthorizedExpressionList(unauthorizedExpressions []string) string { + var unauthorizedList strings.Builder + unauthorizedList.WriteString("\n") + for _, expr := range unauthorizedExpressions { + unauthorizedList.WriteString(" - ") + unauthorizedList.WriteString(expr) + closestMatches := parser.FindClosestMatches(expr, constants.AllowedExpressions, maxFuzzyMatchSuggestions) + if len(closestMatches) > 0 { + unauthorizedList.WriteString(" (did you mean: ") + unauthorizedList.WriteString(strings.Join(closestMatches, ", ")) + unauthorizedList.WriteString("?)") } + unauthorizedList.WriteString("\n") + } + return unauthorizedList.String() +} - var allowedList strings.Builder +func allowedExpressionList() string { + var allowedList strings.Builder + allowedList.WriteString("\n") + for _, expr := range constants.AllowedExpressions { + allowedList.WriteString(" - ") + allowedList.WriteString(expr) allowedList.WriteString("\n") - for _, expr := range constants.AllowedExpressions { - allowedList.WriteString(" - ") - allowedList.WriteString(expr) - allowedList.WriteString("\n") - } - allowedList.WriteString(" - needs.*\n") - allowedList.WriteString(" - steps.*\n") - allowedList.WriteString(" - github.event.inputs.*\n") - allowedList.WriteString(" - github.aw.inputs.* (shared workflow inputs)\n") - allowedList.WriteString(" - github.aw.import-inputs.* (import-schema inputs)\n") - allowedList.WriteString(" - inputs.* (workflow_call)\n") - allowedList.WriteString(" - env.*\n") - - return NewValidationError( - "expressions", - fmt.Sprintf("%d unauthorized expressions found", len(unauthorizedExpressions)), - "expressions are not in the allowed list:"+unauthorizedList.String(), - fmt.Sprintf("Use only allowed expressions:%s\nFor more details, see the expression security documentation.", allowedList.String()), - ) } - - expressionValidationLog.Print("Expression safety validation passed") - return nil + allowedList.WriteString(" - needs.*\n") + allowedList.WriteString(" - steps.*\n") + allowedList.WriteString(" - github.event.inputs.*\n") + allowedList.WriteString(" - github.aw.inputs.* (shared workflow inputs)\n") + allowedList.WriteString(" - github.aw.import-inputs.* (import-schema inputs)\n") + allowedList.WriteString(" - inputs.* (workflow_call)\n") + allowedList.WriteString(" - env.*\n") + return allowedList.String() } // ExpressionValidationOptions contains the options for validating a single expression @@ -180,91 +171,81 @@ func validateExpressionForDangerousProps(expression string) error { // validateSingleExpression validates a single literal expression func validateSingleExpression(expression string, opts ExpressionValidationOptions) error { expression = strings.TrimSpace(expression) - - // Allow literal values (string, number, boolean) — safe leaf nodes in compound expressions. - if stringLiteralRegex.MatchString(expression) || - numberLiteralRegex.MatchString(expression) || - expression == "true" || expression == "false" { + if isSafeExpressionLiteral(expression) { return nil } - - // Check for dangerous JavaScript property names (prototype pollution, PR #14826) if err := validateExpressionForDangerousProps(expression); err != nil { return err } - - // Check if this expression is in the allowed list - allowed := false - - if opts.NeedsStepsRe.MatchString(expression) { - allowed = true - } else if opts.InputsRe.MatchString(expression) { - allowed = true - } else if opts.WorkflowCallInputsRe.MatchString(expression) { - allowed = true - } else if opts.AwInputsRe.MatchString(expression) { - allowed = true - } else if opts.AwImportInputsRe != nil && opts.AwImportInputsRe.MatchString(expression) { - allowed = true - } else if opts.EnvRe.MatchString(expression) { - allowed = true - } else if _, ok := constants.AllowedExpressionsSet[expression]; ok { - allowed = true + allowed := isDirectlyAllowedExpression(expression, opts) + if !allowed { + allowed = isAllowedOrExpression(expression, opts) } - - // Check for OR expressions with literals (e.g., "inputs.repository || 'default'") if !allowed { - orMatch := orExpressionPattern.FindStringSubmatch(expression) - if len(orMatch) > 2 { - leftExpr := strings.TrimSpace(orMatch[1]) - rightExpr := strings.TrimSpace(orMatch[2]) - - leftErr := validateSingleExpression(leftExpr, opts) - leftIsSafe := leftErr == nil && !containsExpressionInList(opts.UnauthorizedExpressions, leftExpr) + allowed = isAllowedComparisonExpression(expression, opts) + } + if !allowed { + *opts.UnauthorizedExpressions = append(*opts.UnauthorizedExpressions, expression) + } + return nil +} - if leftIsSafe { - // Check if right side is a literal string (single, double, or backtick quotes) - // Note: Using (?:) for non-capturing group and checking each quote type separately - isStringLiteral := stringLiteralRegex.MatchString(rightExpr) - // Check if right side is a number literal - isNumberLiteral := numberLiteralRegex.MatchString(rightExpr) - // Check if right side is a boolean literal - isBooleanLiteral := rightExpr == "true" || rightExpr == "false" +func isSafeExpressionLiteral(expression string) bool { + return stringLiteralRegex.MatchString(expression) || numberLiteralRegex.MatchString(expression) || + expression == "true" || expression == "false" +} - if isStringLiteral || isNumberLiteral || isBooleanLiteral { - allowed = true - } else { - // If right side is also a safe expression, recursively check it - rightErr := validateSingleExpression(rightExpr, opts) - if rightErr == nil && !containsExpressionInList(opts.UnauthorizedExpressions, rightExpr) { - allowed = true - } - } - } - } +func isDirectlyAllowedExpression(expression string, opts ExpressionValidationOptions) bool { + switch { + case opts.NeedsStepsRe.MatchString(expression): + return true + case opts.InputsRe.MatchString(expression): + return true + case opts.WorkflowCallInputsRe.MatchString(expression): + return true + case opts.AwInputsRe.MatchString(expression): + return true + case opts.AwImportInputsRe != nil && opts.AwImportInputsRe.MatchString(expression): + return true + case opts.EnvRe.MatchString(expression): + return true + default: + _, ok := constants.AllowedExpressionsSet[expression] + return ok } +} - // Validate both sides of comparison expressions recursively. - if !allowed { - comparisonMatch := comparisonExpressionPattern.FindStringSubmatch(expression) - if len(comparisonMatch) > 2 { - leftExpr := strings.TrimSpace(comparisonMatch[1]) - rightExpr := strings.TrimSpace(comparisonMatch[2]) - leftErr := validateSingleExpression(leftExpr, opts) - rightErr := validateSingleExpression(rightExpr, opts) - if leftExpr != "" && rightExpr != "" && - leftErr == nil && !containsExpressionInList(opts.UnauthorizedExpressions, leftExpr) && - rightErr == nil && !containsExpressionInList(opts.UnauthorizedExpressions, rightExpr) { - allowed = true - } - } +func isAllowedOrExpression(expression string, opts ExpressionValidationOptions) bool { + orMatch := orExpressionPattern.FindStringSubmatch(expression) + if len(orMatch) <= 2 { + return false } - - if !allowed { - *opts.UnauthorizedExpressions = append(*opts.UnauthorizedExpressions, expression) + leftExpr := strings.TrimSpace(orMatch[1]) + rightExpr := strings.TrimSpace(orMatch[2]) + leftErr := validateSingleExpression(leftExpr, opts) + leftIsSafe := leftErr == nil && !containsExpressionInList(opts.UnauthorizedExpressions, leftExpr) + if !leftIsSafe { + return false + } + if isSafeExpressionLiteral(rightExpr) { + return true } + rightErr := validateSingleExpression(rightExpr, opts) + return rightErr == nil && !containsExpressionInList(opts.UnauthorizedExpressions, rightExpr) +} - return nil +func isAllowedComparisonExpression(expression string, opts ExpressionValidationOptions) bool { + comparisonMatch := comparisonExpressionPattern.FindStringSubmatch(expression) + if len(comparisonMatch) <= 2 { + return false + } + leftExpr := strings.TrimSpace(comparisonMatch[1]) + rightExpr := strings.TrimSpace(comparisonMatch[2]) + leftErr := validateSingleExpression(leftExpr, opts) + rightErr := validateSingleExpression(rightExpr, opts) + return leftExpr != "" && rightExpr != "" && + leftErr == nil && !containsExpressionInList(opts.UnauthorizedExpressions, leftExpr) && + rightErr == nil && !containsExpressionInList(opts.UnauthorizedExpressions, rightExpr) } // containsExpressionInList checks if an expression is in the list. diff --git a/pkg/workflow/expression_syntax_validation.go b/pkg/workflow/expression_syntax_validation.go index 2a6d1c67261..442231a254a 100644 --- a/pkg/workflow/expression_syntax_validation.go +++ b/pkg/workflow/expression_syntax_validation.go @@ -197,38 +197,42 @@ func validateBalancedQuotes(expr string) error { } } - // Check if we reached end of string with unclosed quote if i == len(expr)-1 { - if inSingleQuote { - return NewValidationError( - "expression", - "unclosed single quote", - "found unclosed single quote in expression: "+expr, - "Add the missing closing single quote (') to your expression.", - ) - } - if inDoubleQuote { - return NewValidationError( - "expression", - "unclosed double quote", - "found unclosed double quote in expression: "+expr, - "Add the missing closing double quote (\") to your expression.", - ) - } - if inBacktick { - return NewValidationError( - "expression", - "unclosed backtick", - "found unclosed backtick in expression: "+expr, - "Add the missing closing backtick (`) to your expression.", - ) - } + return validateNoOpenQuote(expr, inSingleQuote, inDoubleQuote, inBacktick) } } return nil } +func validateNoOpenQuote(expr string, inSingleQuote, inDoubleQuote, inBacktick bool) error { + if inSingleQuote { + return NewValidationError( + "expression", + "unclosed single quote", + "found unclosed single quote in expression: "+expr, + "Add the missing closing single quote (') to your expression.", + ) + } + if inDoubleQuote { + return NewValidationError( + "expression", + "unclosed double quote", + "found unclosed double quote in expression: "+expr, + "Add the missing closing double quote (\") to your expression.", + ) + } + if inBacktick { + return NewValidationError( + "expression", + "unclosed backtick", + "found unclosed backtick in expression: "+expr, + "Add the missing closing backtick (`) to your expression.", + ) + } + return nil +} + // containsLogicalOperators checks if an expression contains logical operators (&&, ||, !). // Note: '!=' also matches '!' — this is acceptable since the expression parser handles it. func containsLogicalOperators(expr string) bool { diff --git a/pkg/workflow/filters.go b/pkg/workflow/filters.go index cda03b2125e..2dd29144c16 100644 --- a/pkg/workflow/filters.go +++ b/pkg/workflow/filters.go @@ -12,36 +12,8 @@ var filtersLog = logger.New("workflow:filters") func (c *Compiler) applyPullRequestDraftFilter(data *WorkflowData, frontmatter map[string]any) { filtersLog.Print("Applying pull request draft filter") - // Use cached On field from ParsedFrontmatter if available, otherwise fall back to map access - var onValue any - var hasOn bool - if data.ParsedFrontmatter != nil && data.ParsedFrontmatter.On != nil { - onValue = data.ParsedFrontmatter.On - hasOn = true - } else { - onValue, hasOn = frontmatter["on"] - } - - // Check if there's an "on" section in the frontmatter - if !hasOn { - return - } - - // Check if "on" is an object (not a string) - onMap, isOnMap := onValue.(map[string]any) - if !isOnMap { - return - } - - // Check if there's a pull_request section - prValue, hasPR := onMap["pull_request"] - if !hasPR { - return - } - - // Check if pull_request is an object with draft settings - prMap, isPRMap := prValue.(map[string]any) - if !isPRMap { + prMap, ok := getPullRequestFilterMap(data, frontmatter) + if !ok { return } @@ -59,45 +31,7 @@ func (c *Compiler) applyPullRequestDraftFilter(data *WorkflowData, frontmatter m } filtersLog.Printf("Found draft filter configuration: draft=%v", draftBool) - - // Generate conditional logic based on draft value using expression nodes - var draftCondition ConditionNode - if draftBool { - // draft: true - include only draft PRs - // The condition should be true for non-pull_request events or for draft pull_requests - notPullRequestEvent := BuildNotEquals( - BuildPropertyAccess("github.event_name"), - BuildStringLiteral("pull_request"), - ) - isDraftPR := BuildEquals( - BuildPropertyAccess("github.event.pull_request.draft"), - BuildBooleanLiteral(true), - ) - draftCondition = &OrNode{ - Left: notPullRequestEvent, - Right: isDraftPR, - } - } else { - // draft: false - exclude draft PRs - // The condition should be true for non-pull_request events or for non-draft pull_requests - notPullRequestEvent := BuildNotEquals( - BuildPropertyAccess("github.event_name"), - BuildStringLiteral("pull_request"), - ) - isNotDraftPR := BuildEquals( - BuildPropertyAccess("github.event.pull_request.draft"), - BuildBooleanLiteral(false), - ) - draftCondition = &OrNode{ - Left: notPullRequestEvent, - Right: isNotDraftPR, - } - } - - // Build condition tree and render - existingCondition := data.If - conditionTree := BuildConditionTree(existingCondition, draftCondition.Render()) - data.If = RenderCondition(conditionTree) + applyConditionNode(data, buildDraftCondition(draftBool)) } // applyPullRequestForkFilter applies fork filter conditions for pull_request triggers @@ -106,90 +40,23 @@ func (c *Compiler) applyPullRequestDraftFilter(data *WorkflowData, frontmatter m func (c *Compiler) applyPullRequestForkFilter(data *WorkflowData, frontmatter map[string]any) { filtersLog.Print("Applying pull request fork filter") - // Use cached On field from ParsedFrontmatter if available, otherwise fall back to map access - var onValue any - var hasOn bool - if data.ParsedFrontmatter != nil && data.ParsedFrontmatter.On != nil { - onValue = data.ParsedFrontmatter.On - hasOn = true - } else { - onValue, hasOn = frontmatter["on"] - } - - // Check if there's an "on" section in the frontmatter - if !hasOn { + prMap, ok := getPullRequestFilterMap(data, frontmatter) + if !ok { return } - // Check if "on" is an object (not a string) - onMap, isOnMap := onValue.(map[string]any) - if !isOnMap { + allowedForks, ok := parseAllowedForks(prMap) + if !ok { return } - // Check if there's a pull_request section - prValue, hasPR := onMap["pull_request"] - if !hasPR { - return - } - - // Check if pull_request is an object with fork settings - prMap, isPRMap := prValue.(map[string]any) - if !isPRMap { - return - } - - // Check for "forks" field (string or array) - forksValue, hasForks := prMap["forks"] - - // Default behavior: If forks field is not specified, only allow same-repo PRs (disallow all forks by default) - var allowedForks []string - if !hasForks { - filtersLog.Print("No forks field specified - applying default fork filter (disallow all forks)") - // Empty allowedForks array means only same-repo PRs are allowed - allowedForks = []string{} - } else { - filtersLog.Print("Found forks filter configuration") - - // Convert forks value to []string, handling both string and array formats - // Handle string format (e.g., forks: "*" or forks: "org/*") - if forksStr, isForksStr := forksValue.(string); isForksStr { - allowedForks = []string{forksStr} - } else if forksArray, isForksArray := forksValue.([]any); isForksArray { - // Handle array format (e.g., forks: ["*", "org/repo"]) - for _, fork := range forksArray { - if forkStr, isForkStr := fork.(string); isForkStr { - allowedForks = append(allowedForks, forkStr) - } - } - } else { - // Invalid forks format, skip - return - } - } - // If "*" wildcard is present, skip fork filtering (allow all forks) if slices.Contains(allowedForks, "*") { filtersLog.Print("Wildcard fork pattern detected, allowing all forks") return // No fork filtering needed } - // Build condition for allowed forks with glob support - notPullRequestEvent := BuildNotEquals( - BuildPropertyAccess("github.event_name"), - BuildStringLiteral("pull_request"), - ) - allowedForksCondition := BuildFromAllowedForks(allowedForks) - - forkCondition := &OrNode{ - Left: notPullRequestEvent, - Right: allowedForksCondition, - } - - // Build condition tree and render - existingCondition := data.If - conditionTree := BuildConditionTree(existingCondition, forkCondition.Render()) - data.If = RenderCondition(conditionTree) + applyConditionNode(data, buildForkCondition(allowedForks)) } // applyLabelFilter applies label name filter conditions for labeled/unlabeled triggers @@ -197,7 +64,21 @@ func (c *Compiler) applyPullRequestForkFilter(data *WorkflowData, frontmatter ma func (c *Compiler) applyLabelFilter(data *WorkflowData, frontmatter map[string]any) { filtersLog.Print("Applying label filter") - // Use cached On field from ParsedFrontmatter if available, otherwise fall back to map access + onMap, ok := getOnFilterMap(data, frontmatter) + if !ok { + return + } + + labelConditions := collectLabelConditions(onMap) + + // If we have label conditions, combine them and apply to the workflow + if len(labelConditions) > 0 { + filtersLog.Printf("Applying label name filters: %d conditions found", len(labelConditions)) + applyConditionNode(data, combineConditionsWithAnd(labelConditions)) + } +} + +func getOnFilterMap(data *WorkflowData, frontmatter map[string]any) (map[string]any, bool) { var onValue any var hasOn bool if data.ParsedFrontmatter != nil && data.ParsedFrontmatter.On != nil { @@ -206,228 +87,199 @@ func (c *Compiler) applyLabelFilter(data *WorkflowData, frontmatter map[string]a } else { onValue, hasOn = frontmatter["on"] } - - // Check if there's an "on" section in the frontmatter if !hasOn { - return + return nil, false } - - // Check if "on" is an object (not a string) onMap, isOnMap := onValue.(map[string]any) - if !isOnMap { - return - } + return onMap, isOnMap +} - // Check both issues, pull_request, and discussion sections for labeled/unlabeled with names - eventSections := []struct { - eventName string - eventValue any - eventNameStr string // For condition checks - }{ - {"issues", onMap["issues"], "issues"}, - {"pull_request", onMap["pull_request"], "pull_request"}, - {"discussion", onMap["discussion"], "discussion"}, +func getPullRequestFilterMap(data *WorkflowData, frontmatter map[string]any) (map[string]any, bool) { + onMap, ok := getOnFilterMap(data, frontmatter) + if !ok { + return nil, false } + prValue, hasPR := onMap["pull_request"] + if !hasPR { + return nil, false + } + prMap, isPRMap := prValue.(map[string]any) + return prMap, isPRMap +} - var labelConditions []ConditionNode - - for _, section := range eventSections { - if section.eventValue == nil { - continue - } - - // Check if the section is an object with types and names - sectionMap, isSectionMap := section.eventValue.(map[string]any) - if !isSectionMap { - continue - } - - // Check for "types" field - typesValue, hasTypes := sectionMap["types"] - if !hasTypes { - continue - } +func buildDraftCondition(draftBool bool) ConditionNode { + notPullRequestEvent := BuildNotEquals( + BuildPropertyAccess("github.event_name"), + BuildStringLiteral("pull_request"), + ) + draftCheck := BuildEquals( + BuildPropertyAccess("github.event.pull_request.draft"), + BuildBooleanLiteral(draftBool), + ) + return &OrNode{ + Left: notPullRequestEvent, + Right: draftCheck, + } +} - // Convert types to []string - var types []string - if typesArray, isTypesArray := typesValue.([]any); isTypesArray { - for _, t := range typesArray { - if tStr, isTStr := t.(string); isTStr { - types = append(types, tStr) - } +func parseAllowedForks(prMap map[string]any) ([]string, bool) { + forksValue, hasForks := prMap["forks"] + if !hasForks { + filtersLog.Print("No forks field specified - applying default fork filter (disallow all forks)") + return []string{}, true + } + filtersLog.Print("Found forks filter configuration") + if forksStr, isForksStr := forksValue.(string); isForksStr { + return []string{forksStr}, true + } + if forksArray, isForksArray := forksValue.([]any); isForksArray { + allowedForks := make([]string, 0, len(forksArray)) + for _, fork := range forksArray { + if forkStr, isForkStr := fork.(string); isForkStr { + allowedForks = append(allowedForks, forkStr) } } + return allowedForks, true + } + return nil, false +} - // Check if types includes "labeled" or "unlabeled" - hasLabeled := false - hasUnlabeled := false - for _, t := range types { - if t == "labeled" { - hasLabeled = true - } - if t == "unlabeled" { - hasUnlabeled = true - } - } +func buildForkCondition(allowedForks []string) ConditionNode { + return &OrNode{ + Left: BuildNotEquals( + BuildPropertyAccess("github.event_name"), + BuildStringLiteral("pull_request"), + ), + Right: BuildFromAllowedForks(allowedForks), + } +} - if !hasLabeled && !hasUnlabeled { - continue - } +func applyConditionNode(data *WorkflowData, condition ConditionNode) { + conditionTree := BuildConditionTree(data.If, condition.Render()) + data.If = RenderCondition(conditionTree) +} - // Check if this section uses native GitHub Actions label filtering - // (indicated by __gh_aw_native_label_filter__ marker) - if nativeFilterValue, hasNativeFilter := sectionMap["__gh_aw_native_label_filter__"]; hasNativeFilter { - if usesNativeFilter, ok := nativeFilterValue.(bool); ok && usesNativeFilter { - // Skip applying job condition filtering for this section - // as it uses native GitHub Actions label filtering - filtersLog.Printf("Skipping label filter for %s: using native GitHub Actions label filtering", section.eventName) - continue - } - } +type labelFilterEventSection struct { + eventName string + eventValue any +} - // Check for "names" field - namesValue, hasNames := sectionMap["names"] - if !hasNames { - continue +func collectLabelConditions(onMap map[string]any) []ConditionNode { + eventSections := []labelFilterEventSection{ + {"issues", onMap["issues"]}, + {"pull_request", onMap["pull_request"]}, + {"discussion", onMap["discussion"]}, + } + var labelConditions []ConditionNode + for _, section := range eventSections { + condition := buildLabelConditionForSection(section) + if condition != nil { + labelConditions = append(labelConditions, condition) } + } + return labelConditions +} - // Convert names to []string, handling both string and array formats - var labelNames []string - if namesStr, isNamesStr := namesValue.(string); isNamesStr { - labelNames = []string{namesStr} - } else if namesArray, isNamesArray := namesValue.([]any); isNamesArray { - for _, name := range namesArray { - if nameStr, isNameStr := name.(string); isNameStr { - labelNames = append(labelNames, nameStr) - } - } - } else { - // Invalid names format, skip - continue - } +func buildLabelConditionForSection(section labelFilterEventSection) ConditionNode { + if section.eventValue == nil { + return nil + } + sectionMap, isSectionMap := section.eventValue.(map[string]any) + if !isSectionMap { + return nil + } + hasLabeled, hasUnlabeled := labelEventTypes(sectionMap["types"]) + if !hasLabeled && !hasUnlabeled { + return nil + } + if usesNativeLabelFilter(sectionMap) { + filtersLog.Printf("Skipping label filter for %s: using native GitHub Actions label filtering", section.eventName) + return nil + } + labelNames, ok := parseStringOrStringArray(sectionMap["names"]) + if !ok || len(labelNames) == 0 { + return nil + } + return buildLabelActionCondition(section.eventName, hasLabeled, hasUnlabeled, buildLabelNameMatch(labelNames)) +} - if len(labelNames) == 0 { - continue +func labelEventTypes(typesValue any) (bool, bool) { + var hasLabeled bool + var hasUnlabeled bool + if typesArray, isTypesArray := typesValue.([]any); isTypesArray { + for _, t := range typesArray { + tStr, isTStr := t.(string) + hasLabeled = hasLabeled || (isTStr && tStr == "labeled") + hasUnlabeled = hasUnlabeled || (isTStr && tStr == "unlabeled") } + } + return hasLabeled, hasUnlabeled +} - // Build condition for this event section - // The condition should be: - // (event_name != 'issues' OR action != 'labeled' OR label.name in names) AND - // (event_name != 'issues' OR action != 'unlabeled' OR label.name in names) - - // For each label name, create a condition - var labelNameConditions []ConditionNode - for _, labelName := range labelNames { - labelNameConditions = append(labelNameConditions, BuildEquals( - BuildPropertyAccess("github.event.label.name"), - BuildStringLiteral(labelName), - )) - } +func usesNativeLabelFilter(sectionMap map[string]any) bool { + nativeFilterValue, hasNativeFilter := sectionMap["__gh_aw_native_label_filter__"] + usesNativeFilter, ok := nativeFilterValue.(bool) + return hasNativeFilter && ok && usesNativeFilter +} - // Combine label name conditions with OR - var labelNameMatch ConditionNode - if len(labelNameConditions) == 1 { - labelNameMatch = labelNameConditions[0] - } else { - labelNameMatch = &DisjunctionNode{Terms: labelNameConditions} +func parseStringOrStringArray(value any) ([]string, bool) { + if str, ok := value.(string); ok { + return []string{str}, true + } + array, ok := value.([]any) + if !ok { + return nil, false + } + values := make([]string, 0, len(array)) + for _, item := range array { + if itemStr, isString := item.(string); isString { + values = append(values, itemStr) } + } + return values, true +} - // Build conditions for labeled and unlabeled - var sectionCondition ConditionNode - - if hasLabeled && hasUnlabeled { - // Both labeled and unlabeled: check for either action - notThisEvent := BuildNotEquals( - BuildPropertyAccess("github.event_name"), - BuildStringLiteral(section.eventNameStr), - ) - - notLabeledAction := BuildNotEquals( - BuildPropertyAccess("github.event.action"), - BuildStringLiteral("labeled"), - ) - - notUnlabeledAction := BuildNotEquals( - BuildPropertyAccess("github.event.action"), - BuildStringLiteral("unlabeled"), - ) - - // (event_name != 'issues') OR (action != 'labeled' AND action != 'unlabeled') OR (label.name matches) - notLabelAction := &AndNode{Left: notLabeledAction, Right: notUnlabeledAction} - sectionCondition = &OrNode{ - Left: notThisEvent, - Right: &OrNode{ - Left: notLabelAction, - Right: labelNameMatch, - }, - } - } else if hasLabeled { - // Only labeled - notThisEvent := BuildNotEquals( - BuildPropertyAccess("github.event_name"), - BuildStringLiteral(section.eventNameStr), - ) - - notLabeledAction := BuildNotEquals( - BuildPropertyAccess("github.event.action"), - BuildStringLiteral("labeled"), - ) - - // (event_name != 'issues') OR (action != 'labeled') OR (label.name matches) - sectionCondition = &OrNode{ - Left: notThisEvent, - Right: &OrNode{ - Left: notLabeledAction, - Right: labelNameMatch, - }, - } - } else if hasUnlabeled { - // Only unlabeled - notThisEvent := BuildNotEquals( - BuildPropertyAccess("github.event_name"), - BuildStringLiteral(section.eventNameStr), - ) - - notUnlabeledAction := BuildNotEquals( - BuildPropertyAccess("github.event.action"), - BuildStringLiteral("unlabeled"), - ) - - // (event_name != 'issues') OR (action != 'unlabeled') OR (label.name matches) - sectionCondition = &OrNode{ - Left: notThisEvent, - Right: &OrNode{ - Left: notUnlabeledAction, - Right: labelNameMatch, - }, - } - } +func buildLabelNameMatch(labelNames []string) ConditionNode { + labelNameConditions := make([]ConditionNode, 0, len(labelNames)) + for _, labelName := range labelNames { + labelNameConditions = append(labelNameConditions, BuildEquals( + BuildPropertyAccess("github.event.label.name"), + BuildStringLiteral(labelName), + )) + } + if len(labelNameConditions) == 1 { + return labelNameConditions[0] + } + return &DisjunctionNode{Terms: labelNameConditions} +} - if sectionCondition != nil { - labelConditions = append(labelConditions, sectionCondition) +func buildLabelActionCondition(eventName string, hasLabeled, hasUnlabeled bool, labelNameMatch ConditionNode) ConditionNode { + notThisEvent := BuildNotEquals(BuildPropertyAccess("github.event_name"), BuildStringLiteral(eventName)) + if hasLabeled && hasUnlabeled { + notLabelAction := &AndNode{ + Left: BuildNotEquals(BuildPropertyAccess("github.event.action"), BuildStringLiteral("labeled")), + Right: BuildNotEquals(BuildPropertyAccess("github.event.action"), BuildStringLiteral("unlabeled")), } + return &OrNode{Left: notThisEvent, Right: &OrNode{Left: notLabelAction, Right: labelNameMatch}} + } + if hasLabeled { + notLabeledAction := BuildNotEquals(BuildPropertyAccess("github.event.action"), BuildStringLiteral("labeled")) + return &OrNode{Left: notThisEvent, Right: &OrNode{Left: notLabeledAction, Right: labelNameMatch}} } + notUnlabeledAction := BuildNotEquals(BuildPropertyAccess("github.event.action"), BuildStringLiteral("unlabeled")) + return &OrNode{Left: notThisEvent, Right: &OrNode{Left: notUnlabeledAction, Right: labelNameMatch}} +} - // If we have label conditions, combine them and apply to the workflow - if len(labelConditions) > 0 { - filtersLog.Printf("Applying label name filters: %d conditions found", len(labelConditions)) - var finalCondition ConditionNode - if len(labelConditions) == 1 { - finalCondition = labelConditions[0] - } else { - // Combine all conditions with AND - finalCondition = labelConditions[0] - for i := 1; i < len(labelConditions); i++ { - finalCondition = &AndNode{ - Left: finalCondition, - Right: labelConditions[i], - } - } +func combineConditionsWithAnd(conditions []ConditionNode) ConditionNode { + if len(conditions) == 1 { + return conditions[0] + } + finalCondition := conditions[0] + for i := 1; i < len(conditions); i++ { + finalCondition = &AndNode{ + Left: finalCondition, + Right: conditions[i], } - - // Build condition tree and render - existingCondition := data.If - conditionTree := BuildConditionTree(existingCondition, finalCondition.Render()) - data.If = RenderCondition(conditionTree) } + return finalCondition } diff --git a/pkg/workflow/frontmatter_error.go b/pkg/workflow/frontmatter_error.go index 037129301f1..af956fa54bb 100644 --- a/pkg/workflow/frontmatter_error.go +++ b/pkg/workflow/frontmatter_error.go @@ -78,83 +78,82 @@ func findFrontmatterFieldLine(frontmatterLines []string, frontmatterStart int, f // Returns error in VSCode-compatible format: filename:line:column: error message func (c *Compiler) createFrontmatterError(filePath, content string, err error, frontmatterLineOffset int) error { frontmatterErrorLog.Printf("Creating frontmatter error for file: %s, offset: %d", filePath, frontmatterLineOffset) - errorStr := err.Error() + if isFormattedFrontmatterParseError(errorStr) { + return formattedFrontmatterError(filePath, content, errorStr, frontmatterLineOffset) + } + frontmatterErrorLog.Printf("Using fallback error message: %v", err) + fallbackFmt := fmt.Sprintf("%s:%d:1: error: %s", filePath, frontmatterLineOffset, err) + return parser.NewFormattedParserError(fallbackFmt) +} - // Check if error already contains formatted yaml.FormatError() output with source context - // yaml.FormatError() produces output like "failed to parse frontmatter:\n[line:col] message\n> line | content..." - if strings.Contains(errorStr, frontmatterParseErrPrefix+"[") && (strings.Contains(errorStr, "\n>") || strings.Contains(errorStr, "|")) { - // Extract line and column from the formatted error for VSCode compatibility - // Pattern: [line:col] message - if matches := lineColPattern.FindStringSubmatch(errorStr); len(matches) >= 4 { - line := matches[1] - col := matches[2] - originalLine := line - originalCol := col - message := matches[3] - // Extract just the first line of the message (before newline) - if idx := strings.Index(message, "\n"); idx != -1 { - message = message[:idx] - } - // Translate raw YAML parser messages to user-friendly plain English. - // Uses the shared translation table from pkg/parser to keep both code paths in sync. - message = parser.TranslateYAMLMessage(message) - line, col, message = improveFrontmatterDiagnostic(content, line, col, message) - - // Format as: filename:line:column: error: message - // This is compatible with VSCode's problem matcher - vscodeFormat := fmt.Sprintf("%s:%s:%s: error: %s", filePath, line, col, message) - - // Extract just the source context lines (skip the [line:col] message line to avoid duplication) - // Find the first line that starts with whitespace + digit + | (source context line) - if loc := sourceContextPattern.FindStringIndex(errorStr); loc != nil { - // Extract from the first source context line to the end - context := errorStr[loc[0]+1:] // +1 to skip the leading newline - if line != originalLine || col != originalCol { - if custom := renderSourceContextForPosition(content, parsePositiveInt(line), parsePositiveInt(col)); custom != "" { - context = custom - } - } - // Return VSCode-compatible format on first line, followed by source context only - frontmatterErrorLog.Print("Formatting error for VSCode compatibility") - return parser.NewFormattedParserError(fmt.Sprintf("%s\n%s", vscodeFormat, context)) - } - - // If we can't extract source context, return just the VSCode format - return parser.NewFormattedParserError(vscodeFormat) - } +func isFormattedFrontmatterParseError(errorStr string) bool { + return strings.Contains(errorStr, frontmatterParseErrPrefix+"[") && + (strings.Contains(errorStr, "\n>") || strings.Contains(errorStr, "|")) +} + +func formattedFrontmatterError(filePath, content, errorStr string, frontmatterLineOffset int) error { + if matches := lineColPattern.FindStringSubmatch(errorStr); len(matches) >= 4 { + return frontmatterLineColError(filePath, content, errorStr, matches) + } + frontmatterErrorLog.Print("Could not extract line/col from formatted error, falling back to frontmatter start") + fallbackFmt := fmt.Sprintf("%s:%d:1: error: %s", filePath, frontmatterLineOffset, frontmatterFallbackMessage(errorStr)) + return parser.NewFormattedParserError(fallbackFmt) +} + +func frontmatterLineColError(filePath, content, errorStr string, matches []string) error { + line := matches[1] + col := matches[2] + originalLine := line + originalCol := col + message := firstLine(matches[3]) + message = parser.TranslateYAMLMessage(message) + line, col, message = improveFrontmatterDiagnostic(content, line, col, message) + vscodeFormat := fmt.Sprintf("%s:%s:%s: error: %s", filePath, line, col, message) + context := frontmatterSourceContext(content, errorStr, line, col, originalLine, originalCol) + if context != "" { + frontmatterErrorLog.Print("Formatting error for VSCode compatibility") + return parser.NewFormattedParserError(fmt.Sprintf("%s\n%s", vscodeFormat, context)) + } + return parser.NewFormattedParserError(vscodeFormat) +} - // Fallback if we can't parse the line/col: emit an IDE-compatible error - // pointing to the frontmatter start so the developer is at least brought to - // the right section rather than the useless line 1, col 1. - frontmatterErrorLog.Print("Could not extract line/col from formatted error, falling back to frontmatter start") - fallbackMsg := "failed to parse YAML frontmatter" - // Try to surface a single-line description from the raw error text. - if _, rest, found := strings.Cut(errorStr, frontmatterParseErrPrefix); found { - firstLine, _, _ := strings.Cut(rest, "\n") - if translated := parser.TranslateYAMLMessage(strings.TrimSpace(firstLine)); translated != "" { - fallbackMsg = "failed to parse YAML frontmatter: " + translated - } +func firstLine(message string) string { + if idx := strings.Index(message, "\n"); idx != -1 { + return message[:idx] + } + return message +} + +func frontmatterSourceContext(content, errorStr, line, col, originalLine, originalCol string) string { + loc := sourceContextPattern.FindStringIndex(errorStr) + if loc == nil { + return "" + } + context := errorStr[loc[0]+1:] + if line != originalLine || col != originalCol { + if custom := renderSourceContextForPosition(content, parsePositiveInt(line), parsePositiveInt(col)); custom != "" { + context = custom } - fallbackFmt := fmt.Sprintf("%s:%d:1: error: %s", filePath, frontmatterLineOffset, fallbackMsg) - return parser.NewFormattedParserError(fallbackFmt) } + return context +} - // Fallback: if not already formatted, create a FormattedParserError pointing to the - // frontmatter start so the IDE navigates to the right file and section rather than - // defaulting to line 1, col 1. - frontmatterErrorLog.Printf("Using fallback error message: %v", err) - fallbackFmt := fmt.Sprintf("%s:%d:1: error: %s", filePath, frontmatterLineOffset, err) - return parser.NewFormattedParserError(fallbackFmt) +func frontmatterFallbackMessage(errorStr string) string { + fallbackMsg := "failed to parse YAML frontmatter" + if _, rest, found := strings.Cut(errorStr, frontmatterParseErrPrefix); found { + firstLine, _, _ := strings.Cut(rest, "\n") + if translated := parser.TranslateYAMLMessage(strings.TrimSpace(firstLine)); translated != "" { + fallbackMsg = "failed to parse YAML frontmatter: " + translated + } + } + return fallbackMsg } // improveFrontmatterDiagnostic adjusts known low-quality parser diagnostics to // point at the true source line and provide user-facing wording. func improveFrontmatterDiagnostic(content, line, col, message string) (string, string, string) { lower := strings.ToLower(strings.TrimSpace(message)) - // Only the translated phrase is checked here because parser.TranslateYAMLMessage - // runs before improveFrontmatterDiagnostic (see createFrontmatterError); the raw - // parser wording is never seen by the time this function is called. isScalarWithNestedKey := strings.Contains(lower, "value cannot have child keys here") if !isScalarWithNestedKey { return line, col, message @@ -199,8 +198,6 @@ func improveFrontmatterDiagnostic(content, line, col, message string) (string, s return line, col, message } - // Find the value's column offset by searching after the colon, not the full line, - // to correctly handle keys and values that share a substring (e.g. "foo: foo"). keyEnd := len(parentMatch[1]) + len(parentMatch[2]) valueCol := colNum - 1 // fallback: original column if colonPos := strings.Index(parentLine[keyEnd:], ":"); colonPos >= 0 { diff --git a/pkg/workflow/frontmatter_extraction_security.go b/pkg/workflow/frontmatter_extraction_security.go index b5aa5b922d6..b97761b24ae 100644 --- a/pkg/workflow/frontmatter_extraction_security.go +++ b/pkg/workflow/frontmatter_extraction_security.go @@ -108,181 +108,139 @@ func (c *Compiler) extractSandboxConfig(frontmatter map[string]any) *SandboxConf frontmatterExtractionSecurityLog.Printf("Sandbox object format with %d fields", len(sandboxObj)) - config := &SandboxConfig{} + return c.extractSandboxObjectConfig(sandboxObj) +} - // Check for new format: { agent: ..., mcp: ... } +func (c *Compiler) extractSandboxObjectConfig(sandboxObj map[string]any) *SandboxConfig { + config := &SandboxConfig{} if agentVal, hasAgent := sandboxObj["agent"]; hasAgent { frontmatterExtractionSecurityLog.Print("Extracting agent sandbox configuration") config.Agent = c.extractAgentSandboxConfig(agentVal) } - if mcpVal, hasMCP := sandboxObj["mcp"]; hasMCP { frontmatterExtractionSecurityLog.Print("Extracting MCP gateway configuration") config.MCP = c.extractMCPGatewayConfig(mcpVal) } - - // If we found agent field, return the new format config if config.Agent != nil { frontmatterExtractionSecurityLog.Print("Sandbox configured with new format (agent)") return config } - - // Handle legacy object format: { type: "...", config: {...} } if typeVal, hasType := sandboxObj["type"]; hasType { if typeStr, ok := typeVal.(string); ok { config.Type = SandboxType(typeStr) } } - - // Extract config if present (custom SRT config) if configVal, hasConfig := sandboxObj["config"]; hasConfig { config.Config = c.extractSRTConfig(configVal) } - return config } // extractAgentSandboxConfig extracts agent sandbox configuration func (c *Compiler) extractAgentSandboxConfig(agentVal any) *AgentSandboxConfig { - // Handle boolean format: agent: false (disables agent sandbox but keeps MCP gateway) + if config, handled := extractAgentSandboxPrimitive(agentVal); handled { + return config + } + + agentObj, ok := agentVal.(map[string]any) + if !ok { + return nil + } + + agentConfig := &AgentSandboxConfig{NetworkIsolation: true} + c.extractAgentSandboxIdentity(agentConfig, agentObj) + c.extractAgentSandboxExecution(agentConfig, agentObj) + c.extractAgentSandboxRuntime(agentConfig, agentObj) + c.extractAgentSandboxModelFallback(agentConfig, agentObj) + return agentConfig +} + +func extractAgentSandboxPrimitive(agentVal any) (*AgentSandboxConfig, bool) { if agentBool, ok := agentVal.(bool); ok { if !agentBool { frontmatterExtractionSecurityLog.Print("Agent sandbox explicitly disabled with agent: false") return &AgentSandboxConfig{ Disabled: true, - } + }, true } - // agent: true is not meaningful, treat as unconfigured frontmatterExtractionSecurityLog.Print("Agent: true specified but has no effect, treating as unconfigured") - return nil + return nil, true } - - // Handle string format: "awf" or false (legacy srt values are auto-migrated) if agentStr, ok := agentVal.(string); ok { agentType := SandboxType(agentStr) if isSupportedSandboxType(agentType) { return &AgentSandboxConfig{ Type: agentType, - } + }, true } - return nil + return nil, true } + return nil, false +} - // Handle object format: { id/type: "...", config: {...}, command: "...", args: [...], env: {...} } - agentObj, ok := agentVal.(map[string]any) - if !ok { - return nil - } - - agentConfig := &AgentSandboxConfig{} - - // Extract ID field (new format) +func (c *Compiler) extractAgentSandboxIdentity(agentConfig *AgentSandboxConfig, agentObj map[string]any) { if idVal, hasID := agentObj["id"]; hasID { if idStr, ok := idVal.(string); ok { agentConfig.ID = idStr } } - - // Extract Type field (legacy format) if typeVal, hasType := agentObj["type"]; hasType { if typeStr, ok := typeVal.(string); ok { agentConfig.Type = SandboxType(typeStr) } } - - // Extract version (AWF version override) if versionVal, hasVersion := agentObj["version"]; hasVersion { if versionStr, ok := versionVal.(string); ok { agentConfig.Version = versionStr } } - - // Extract platform (AWF platform.type override) if platformVal, hasPlatform := agentObj["platform"]; hasPlatform { if platformStr, ok := platformVal.(string); ok { agentConfig.Platform = platformStr } } +} - // Extract sudo (AWF topology egress mode). - // Semantics are inverted from the frontmatter field: - // sudo: false → no sudo = network isolation mode → NetworkIsolation=true - // sudo: true → sudo enabled = normal mode → NetworkIsolation=false (deprecated; error in strict mode, warning otherwise) - // (omitted) → default = network isolation mode → NetworkIsolation=true (same as sudo: false) - agentConfig.NetworkIsolation = true // Default: sudo: false (network isolation enabled) +func (c *Compiler) extractAgentSandboxExecution(agentConfig *AgentSandboxConfig, agentObj map[string]any) { if sudoVal, hasSudo := agentObj["sudo"]; hasSudo { if sudoBool, ok := sudoVal.(bool); ok { agentConfig.NetworkIsolation = !sudoBool if sudoBool { - // sudo: true was explicitly set; record it so validation can warn/error. agentConfig.SudoExplicitlyEnabled = true } } } - - // Extract config for SRT if configVal, hasConfig := agentObj["config"]; hasConfig { agentConfig.Config = c.extractSRTConfig(configVal) } - - // Extract command (custom command to replace AWF binary download) if commandVal, hasCommand := agentObj["command"]; hasCommand { if commandStr, ok := commandVal.(string); ok { agentConfig.Command = commandStr } } - - // Extract args (additional arguments to append) - if argsVal, hasArgs := agentObj["args"]; hasArgs { - if argsSlice, ok := argsVal.([]any); ok { - for _, arg := range argsSlice { - if argStr, ok := arg.(string); ok { - agentConfig.Args = append(agentConfig.Args, argStr) - } - } - } - } - - // Extract env (environment variables to set on the step) - if envVal, hasEnv := agentObj["env"]; hasEnv { - if envObj, ok := envVal.(map[string]any); ok { - agentConfig.Env = make(map[string]string) - for key, value := range envObj { - if valueStr, ok := value.(string); ok { - agentConfig.Env[key] = valueStr - } - } - } - } - - // Extract mounts (container mounts for AWF) - if mountsVal, hasMounts := agentObj["mounts"]; hasMounts { - if mountsSlice, ok := mountsVal.([]any); ok { - for _, mount := range mountsSlice { - if mountStr, ok := mount.(string); ok { - agentConfig.Mounts = append(agentConfig.Mounts, mountStr) - } - } - } + agentConfig.Args = append(agentConfig.Args, stringSliceFromMap(agentObj, "args")...) + if env, ok := stringMapFromMap(agentObj, "env"); ok { + agentConfig.Env = env } + agentConfig.Mounts = append(agentConfig.Mounts, stringSliceFromMap(agentObj, "mounts")...) +} - // Extract runtime (container runtime for the agent container) +func (c *Compiler) extractAgentSandboxRuntime(agentConfig *AgentSandboxConfig, agentObj map[string]any) { if runtimeVal, hasRuntime := agentObj["runtime"]; hasRuntime { if runtimeStr, ok := runtimeVal.(string); ok { agentConfig.Runtime = AgentRuntime(runtimeStr) frontmatterExtractionSecurityLog.Printf("Extracted sandbox.agent.runtime: %s", runtimeStr) } } - - // Extract legacy-security (opt-in to legacy sudo/iptables mode) if legacyVal, hasLegacy := agentObj["legacy-security"]; hasLegacy { if legacyStr, ok := legacyVal.(string); ok && legacyStr == "enable" { agentConfig.LegacySecurity = true frontmatterExtractionSecurityLog.Print("Extracted sandbox.agent.legacy-security: enable") } } +} - // Extract model-fallback (AWF API proxy model fallback enable/disable flag) +func (c *Compiler) extractAgentSandboxModelFallback(agentConfig *AgentSandboxConfig, agentObj map[string]any) { if mfVal, hasMF := agentObj["model-fallback"]; hasMF { switch v := mfVal.(type) { case bool: @@ -300,8 +258,6 @@ func (c *Compiler) extractAgentSandboxConfig(agentVal any) *AgentSandboxConfig { } } } - - return agentConfig } // extractMCPGatewayConfig extracts MCP gateway configuration from frontmatter @@ -324,104 +280,54 @@ func (c *Compiler) extractMCPGatewayConfig(mcpVal any) *MCPGatewayRuntimeConfig } mcpConfig := &MCPGatewayRuntimeConfig{} + c.extractMCPGatewayCoreConfig(mcpConfig, mcpObj) + c.extractMCPGatewayListConfig(mcpConfig, mcpObj) + c.extractMCPGatewayPayloadConfig(mcpConfig, mcpObj) + c.extractMCPGatewayTrustedAndKeepaliveConfig(mcpConfig, mcpObj) + return mcpConfig +} - // Extract container (required for MCP gateway) +func (c *Compiler) extractMCPGatewayCoreConfig(mcpConfig *MCPGatewayRuntimeConfig, mcpObj map[string]any) { if containerVal, hasContainer := mcpObj["container"]; hasContainer { if containerStr, ok := containerVal.(string); ok { mcpConfig.Container = containerStr } } - - // Extract version (for container) if versionVal, hasVersion := mcpObj["version"]; hasVersion { if versionStr, ok := versionVal.(string); ok { mcpConfig.Version = versionStr } } - - // Extract entrypoint (optional container entrypoint override) if entrypointVal, hasEntrypoint := mcpObj["entrypoint"]; hasEntrypoint { if entrypointStr, ok := entrypointVal.(string); ok { mcpConfig.Entrypoint = entrypointStr } } - - // Extract port - if portVal, hasPort := mcpObj["port"]; hasPort { - switch v := portVal.(type) { - case int: - mcpConfig.Port = v - case int64: - mcpConfig.Port = int(v) - case uint: - mcpConfig.Port = int(v) - case uint64: - mcpConfig.Port = int(v) - case float64: - mcpConfig.Port = int(v) - } + if port, ok := intFromMap(mcpObj, "port"); ok { + mcpConfig.Port = port } - - // Extract apiKey if apiKeyVal, hasAPIKey := mcpObj["api-key"]; hasAPIKey { if apiKeyStr, ok := apiKeyVal.(string); ok { mcpConfig.APIKey = apiKeyStr } } - - // Extract domain if domainVal, hasDomain := mcpObj["domain"]; hasDomain { if domainStr, ok := domainVal.(string); ok { mcpConfig.Domain = domainStr } } +} - // Extract args (additional arguments) - if argsVal, hasArgs := mcpObj["args"]; hasArgs { - if argsSlice, ok := argsVal.([]any); ok { - for _, arg := range argsSlice { - if argStr, ok := arg.(string); ok { - mcpConfig.Args = append(mcpConfig.Args, argStr) - } - } - } - } - - // Extract entrypointArgs (for container only) - if entrypointArgsVal, hasEntrypointArgs := mcpObj["entrypointArgs"]; hasEntrypointArgs { - if entrypointArgsSlice, ok := entrypointArgsVal.([]any); ok { - for _, arg := range entrypointArgsSlice { - if argStr, ok := arg.(string); ok { - mcpConfig.EntrypointArgs = append(mcpConfig.EntrypointArgs, argStr) - } - } - } - } - - // Extract env (environment variables) - if envVal, hasEnv := mcpObj["env"]; hasEnv { - if envObj, ok := envVal.(map[string]any); ok { - mcpConfig.Env = make(map[string]string) - for key, value := range envObj { - if valueStr, ok := value.(string); ok { - mcpConfig.Env[key] = valueStr - } - } - } - } - - // Extract mounts (volume mounts for container) - if mountsVal, hasMounts := mcpObj["mounts"]; hasMounts { - if mountsSlice, ok := mountsVal.([]any); ok { - for _, mount := range mountsSlice { - if mountStr, ok := mount.(string); ok { - mcpConfig.Mounts = append(mcpConfig.Mounts, mountStr) - } - } - } +func (c *Compiler) extractMCPGatewayListConfig(mcpConfig *MCPGatewayRuntimeConfig, mcpObj map[string]any) { + mcpConfig.Args = append(mcpConfig.Args, stringSliceFromMap(mcpObj, "args")...) + mcpConfig.EntrypointArgs = append(mcpConfig.EntrypointArgs, stringSliceFromMap(mcpObj, "entrypointArgs")...) + if env, ok := stringMapFromMap(mcpObj, "env"); ok { + mcpConfig.Env = env } + mcpConfig.Mounts = append(mcpConfig.Mounts, stringSliceFromMap(mcpObj, "mounts")...) +} - // Extract payloadDir / payload-dir (directory for storing large payload JSON files) +func (c *Compiler) extractMCPGatewayPayloadConfig(mcpConfig *MCPGatewayRuntimeConfig, mcpObj map[string]any) { for _, key := range []string{"payloadDir", "payload-dir"} { if payloadDirVal, hasPayloadDir := mcpObj[key]; hasPayloadDir { if payloadDirStr, ok := payloadDirVal.(string); ok { @@ -430,8 +336,6 @@ func (c *Compiler) extractMCPGatewayConfig(mcpVal any) *MCPGatewayRuntimeConfig } } } - - // Extract payloadPathPrefix / payload-path-prefix (path prefix to remap payload paths) for _, key := range []string{"payloadPathPrefix", "payload-path-prefix"} { if payloadPathPrefixVal, hasPayloadPathPrefix := mcpObj[key]; hasPayloadPathPrefix { if payloadPathPrefixStr, ok := payloadPathPrefixVal.(string); ok { @@ -440,66 +344,34 @@ func (c *Compiler) extractMCPGatewayConfig(mcpVal any) *MCPGatewayRuntimeConfig } } } - - // Extract payloadSizeThreshold / payload-size-threshold (size threshold in bytes) for _, key := range []string{"payloadSizeThreshold", "payload-size-threshold"} { - if payloadSizeThresholdVal, hasPayloadSizeThreshold := mcpObj[key]; hasPayloadSizeThreshold { - switch v := payloadSizeThresholdVal.(type) { - case int: - mcpConfig.PayloadSizeThreshold = v - case int64: - mcpConfig.PayloadSizeThreshold = int(v) - case uint: - mcpConfig.PayloadSizeThreshold = int(v) - case uint64: - mcpConfig.PayloadSizeThreshold = int(v) - case float64: - mcpConfig.PayloadSizeThreshold = int(v) - } + if payloadSizeThreshold, ok := intFromMap(mcpObj, key); ok { + mcpConfig.PayloadSizeThreshold = payloadSizeThreshold if mcpConfig.PayloadSizeThreshold != 0 { break } } } +} - // Extract trustedBots / trusted-bots (additional bot identities to pass to the gateway) +func (c *Compiler) extractMCPGatewayTrustedAndKeepaliveConfig(mcpConfig *MCPGatewayRuntimeConfig, mcpObj map[string]any) { for _, key := range []string{"trustedBots", "trusted-bots"} { - if trustedBotsVal, hasTrustedBots := mcpObj[key]; hasTrustedBots { - if trustedBotsSlice, ok := trustedBotsVal.([]any); ok { - for _, bot := range trustedBotsSlice { - if botStr, ok := bot.(string); ok { - mcpConfig.TrustedBots = append(mcpConfig.TrustedBots, botStr) - } - } - } + if _, hasTrustedBots := mcpObj[key]; hasTrustedBots { + mcpConfig.TrustedBots = append(mcpConfig.TrustedBots, stringSliceFromMap(mcpObj, key)...) if len(mcpConfig.TrustedBots) > 0 { break } } } - - // Extract keepaliveInterval / keepalive-interval (keepalive ping interval in seconds for HTTP MCP backends) - // 0 = unset (gateway default: 1500s), -1 = disable keepalive, >0 = custom interval in seconds for _, key := range []string{"keepaliveInterval", "keepalive-interval"} { - if keepaliveVal, hasKeepalive := mcpObj[key]; hasKeepalive { - switch v := keepaliveVal.(type) { - case int: - mcpConfig.KeepaliveInterval = v - case int64: - mcpConfig.KeepaliveInterval = int(v) - case uint: - mcpConfig.KeepaliveInterval = int(v) - case uint64: - mcpConfig.KeepaliveInterval = int(v) - case float64: - mcpConfig.KeepaliveInterval = int(v) - } - // Break when the key exists (even if value is 0, to avoid picking up a second key variant) + if keepalive, hasKeepalive := intFromMap(mcpObj, key); hasKeepalive { + mcpConfig.KeepaliveInterval = keepalive + break + } + if _, hasKeepalive := mcpObj[key]; hasKeepalive { break } } - - return mcpConfig } // extractSRTConfig extracts Sandbox Runtime configuration from a map @@ -511,126 +383,24 @@ func (c *Compiler) extractSRTConfig(configVal any) *SandboxRuntimeConfig { srtConfig := &SandboxRuntimeConfig{} - // Extract network config if networkVal, hasNetwork := configObj["network"]; hasNetwork { if networkObj, ok := networkVal.(map[string]any); ok { - netConfig := &SRTNetworkConfig{} - - // Extract allowedDomains - if allowedDomains, hasAllowed := networkObj["allowedDomains"]; hasAllowed { - if domainsSlice, ok := allowedDomains.([]any); ok { - for _, domain := range domainsSlice { - if domainStr, ok := domain.(string); ok { - netConfig.AllowedDomains = append(netConfig.AllowedDomains, domainStr) - } - } - } - } - - // Extract blockedDomains - if blockedDomains, hasBlocked := networkObj["blockedDomains"]; hasBlocked { - if domainsSlice, ok := blockedDomains.([]any); ok { - for _, domain := range domainsSlice { - if domainStr, ok := domain.(string); ok { - netConfig.BlockedDomains = append(netConfig.BlockedDomains, domainStr) - } - } - } - } - - // Extract allowUnixSockets - if unixSockets, hasUnixSockets := networkObj["allowUnixSockets"]; hasUnixSockets { - if socketsSlice, ok := unixSockets.([]any); ok { - for _, socket := range socketsSlice { - if socketStr, ok := socket.(string); ok { - netConfig.AllowUnixSockets = append(netConfig.AllowUnixSockets, socketStr) - } - } - } - } - - // Extract allowLocalBinding - if allowLocalBinding, hasAllowLocalBinding := networkObj["allowLocalBinding"]; hasAllowLocalBinding { - if bindingBool, ok := allowLocalBinding.(bool); ok { - netConfig.AllowLocalBinding = bindingBool - } - } - - // Extract allowAllUnixSockets - if allowAllUnixSockets, hasAllowAllUnixSockets := networkObj["allowAllUnixSockets"]; hasAllowAllUnixSockets { - if unixSocketsBool, ok := allowAllUnixSockets.(bool); ok { - netConfig.AllowAllUnixSockets = unixSocketsBool - } - } - - srtConfig.Network = netConfig + srtConfig.Network = extractSRTNetworkConfig(networkObj) } } - // Extract filesystem config if filesystemVal, hasFilesystem := configObj["filesystem"]; hasFilesystem { if filesystemObj, ok := filesystemVal.(map[string]any); ok { - fsConfig := &SRTFilesystemConfig{} - - // Extract denyRead - if denyRead, hasDenyRead := filesystemObj["denyRead"]; hasDenyRead { - if pathsSlice, ok := denyRead.([]any); ok { - fsConfig.DenyRead = []string{} - for _, path := range pathsSlice { - if pathStr, ok := path.(string); ok { - fsConfig.DenyRead = append(fsConfig.DenyRead, pathStr) - } - } - } - } - - // Extract allowWrite - if allowWrite, hasAllowWrite := filesystemObj["allowWrite"]; hasAllowWrite { - if pathsSlice, ok := allowWrite.([]any); ok { - for _, path := range pathsSlice { - if pathStr, ok := path.(string); ok { - fsConfig.AllowWrite = append(fsConfig.AllowWrite, pathStr) - } - } - } - } - - // Extract denyWrite - if denyWrite, hasDenyWrite := filesystemObj["denyWrite"]; hasDenyWrite { - if pathsSlice, ok := denyWrite.([]any); ok { - fsConfig.DenyWrite = []string{} - for _, path := range pathsSlice { - if pathStr, ok := path.(string); ok { - fsConfig.DenyWrite = append(fsConfig.DenyWrite, pathStr) - } - } - } - } - - srtConfig.Filesystem = fsConfig + srtConfig.Filesystem = extractSRTFilesystemConfig(filesystemObj) } } - // Extract ignoreViolations if ignoreViolations, hasIgnoreViolations := configObj["ignoreViolations"]; hasIgnoreViolations { if violationsObj, ok := ignoreViolations.(map[string]any); ok { - violations := make(map[string][]string) - for key, value := range violationsObj { - if pathsSlice, ok := value.([]any); ok { - var paths []string - for _, path := range pathsSlice { - if pathStr, ok := path.(string); ok { - paths = append(paths, pathStr) - } - } - violations[key] = paths - } - } - srtConfig.IgnoreViolations = violations + srtConfig.IgnoreViolations = stringSliceMapFromAnyMap(violationsObj) } } - // Extract enableWeakerNestedSandbox if enableWeakerNestedSandbox, hasEnableWeaker := configObj["enableWeakerNestedSandbox"]; hasEnableWeaker { if weakerBool, ok := enableWeakerNestedSandbox.(bool); ok { srtConfig.EnableWeakerNestedSandbox = weakerBool @@ -639,3 +409,102 @@ func (c *Compiler) extractSRTConfig(configVal any) *SandboxRuntimeConfig { return srtConfig } + +func extractSRTNetworkConfig(networkObj map[string]any) *SRTNetworkConfig { + netConfig := &SRTNetworkConfig{} + netConfig.AllowedDomains = append(netConfig.AllowedDomains, stringSliceFromMap(networkObj, "allowedDomains")...) + netConfig.BlockedDomains = append(netConfig.BlockedDomains, stringSliceFromMap(networkObj, "blockedDomains")...) + netConfig.AllowUnixSockets = append(netConfig.AllowUnixSockets, stringSliceFromMap(networkObj, "allowUnixSockets")...) + if bindingBool, ok := boolFromMap(networkObj, "allowLocalBinding"); ok { + netConfig.AllowLocalBinding = bindingBool + } + if unixSocketsBool, ok := boolFromMap(networkObj, "allowAllUnixSockets"); ok { + netConfig.AllowAllUnixSockets = unixSocketsBool + } + return netConfig +} + +func extractSRTFilesystemConfig(filesystemObj map[string]any) *SRTFilesystemConfig { + fsConfig := &SRTFilesystemConfig{} + if paths, ok := stringSliceFromAny(filesystemObj["denyRead"]); ok { + fsConfig.DenyRead = paths + } + fsConfig.AllowWrite = append(fsConfig.AllowWrite, stringSliceFromMap(filesystemObj, "allowWrite")...) + if paths, ok := stringSliceFromAny(filesystemObj["denyWrite"]); ok { + fsConfig.DenyWrite = paths + } + return fsConfig +} + +func stringSliceMapFromAnyMap(values map[string]any) map[string][]string { + result := make(map[string][]string) + for key, value := range values { + values, ok := value.([]any) + if !ok { + continue + } + var paths []string + for _, path := range values { + if pathStr, ok := path.(string); ok { + paths = append(paths, pathStr) + } + } + result[key] = paths + } + return result +} + +func stringSliceFromMap(values map[string]any, key string) []string { + slice, _ := stringSliceFromAny(values[key]) + return slice +} + +func stringSliceFromAny(value any) ([]string, bool) { + values, ok := value.([]any) + if !ok { + return nil, false + } + result := make([]string, 0, len(values)) + for _, value := range values { + if valueStr, ok := value.(string); ok { + result = append(result, valueStr) + } + } + return result, true +} + +func stringMapFromMap(values map[string]any, key string) (map[string]string, bool) { + valueMap, ok := values[key].(map[string]any) + if !ok { + return nil, false + } + result := make(map[string]string) + for key, value := range valueMap { + if valueStr, ok := value.(string); ok { + result[key] = valueStr + } + } + return result, true +} + +func intFromMap(values map[string]any, key string) (int, bool) { + switch v := values[key].(type) { + case int: + return v, true + case int64: + return int(v), true + case uint: + return int(v), true + case uint64: + return int(v), true + case float64: + return int(v), true + default: + return 0, false + } +} + +func boolFromMap(values map[string]any, key string) (bool, bool) { + value, ok := values[key].(bool) + return value, ok +} diff --git a/pkg/workflow/frontmatter_extraction_yaml.go b/pkg/workflow/frontmatter_extraction_yaml.go index 630dc2a396a..073345382c8 100644 --- a/pkg/workflow/frontmatter_extraction_yaml.go +++ b/pkg/workflow/frontmatter_extraction_yaml.go @@ -51,75 +51,48 @@ func (c *Compiler) extractTopLevelYAMLSection(frontmatter map[string]any, key st frontmatterLog.Printf("Extracting YAML section: %s", key) - // Convert the value back to YAML format with field ordering - var yamlBytes []byte - var err error + yamlStr, ok := marshalTopLevelYAMLSection(key, value) + if !ok { + return "" + } + return c.postProcessTopLevelYAMLSection(yamlStr, key, frontmatter) +} - // Check if value is a map that we should order alphabetically +func marshalTopLevelYAMLSection(key string, value any) (string, bool) { + var data any if valueMap, ok := value.(map[string]any); ok { - // Use OrderMapFields for alphabetical sorting (empty priority list = all alphabetical) - orderedValue := OrderMapFields(valueMap, []string{}) - // Wrap the ordered value with the key using MapSlice - wrappedData := yaml.MapSlice{{Key: key, Value: orderedValue}} - marshalOptions := DefaultMarshalOptions - if key == "on" || key == "services" { - // Indent sequence items (e.g. schedule cron lists, event `types:` - // arrays, and service `ports:` lists) under their parent key so that - // yamllint's default indentation rule (indent-sequences: true) is - // satisfied. Scoped to the `on:` and `services:` sections so that - // custom `steps:` marshaling elsewhere is unaffected. - marshalOptions = append(append([]yaml.EncodeOption{}, DefaultMarshalOptions...), yaml.IndentSequence(true)) - } - yamlBytes, err = yaml.MarshalWithOptions(wrappedData, marshalOptions...) - if err != nil { - return "" - } + data = yaml.MapSlice{{Key: key, Value: OrderMapFields(valueMap, []string{})}} } else { - // Use standard marshaling for non-map types - marshalOptions := DefaultMarshalOptions - if key == "on" || key == "services" { - marshalOptions = append(append([]yaml.EncodeOption{}, DefaultMarshalOptions...), yaml.IndentSequence(true)) - } - yamlBytes, err = yaml.MarshalWithOptions(map[string]any{key: value}, marshalOptions...) - if err != nil { - return "" - } + data = map[string]any{key: value} + } + yamlBytes, err := yaml.MarshalWithOptions(data, topLevelYAMLMarshalOptions(key)...) + if err != nil { + return "", false } + return strings.TrimSuffix(string(yamlBytes), "\n"), true +} - yamlStr := string(yamlBytes) - // Remove the trailing newline - yamlStr = strings.TrimSuffix(yamlStr, "\n") +func topLevelYAMLMarshalOptions(key string) []yaml.EncodeOption { + if key == "on" || key == "services" { + return append(append([]yaml.EncodeOption{}, DefaultMarshalOptions...), yaml.IndentSequence(true)) + } + return DefaultMarshalOptions +} - // Post-process YAML to ensure cron expressions are quoted - // The YAML library may drop quotes from cron expressions like "0 14 * * 1-5" - // which causes validation errors since they start with numbers but contain spaces +func (c *Compiler) postProcessTopLevelYAMLSection(yamlStr, key string, frontmatter map[string]any) string { yamlStr = parser.QuoteCronExpressions(yamlStr) - - // For top-level env values, quote plain scalars containing ": " because YAML - // treats this token sequence as a mapping separator in plain style. if key == "env" { yamlStr = quoteEnvValuesContainingColonSpace(yamlStr) } - - // Clean up null values - replace `: null` with `:` for cleaner output - // GitHub Actions treats `workflow_dispatch:` and `workflow_dispatch: null` identically yamlStr = CleanYAMLNullValues(yamlStr) - - // Clean up quoted keys - replace "key": with key: at the start of a line - // Don't unquote "on" key as it's a YAML boolean keyword and must remain quoted if key != "on" { yamlStr = UnquoteYAMLKey(yamlStr, key) } - - // Special handling for "on" section - comment out draft and fork fields from pull_request if key == "on" { yamlStr = c.commentOutProcessedFieldsInOnSection(yamlStr, frontmatter) - // Add zizmor ignore comment if workflow_run trigger is present yamlStr = c.addZizmorIgnoreForWorkflowRun(yamlStr) - // Add friendly format comments for schedule cron expressions yamlStr = c.addFriendlyScheduleComments(yamlStr, frontmatter) } - return yamlStr } diff --git a/pkg/workflow/frontmatter_on_section_cleanup.go b/pkg/workflow/frontmatter_on_section_cleanup.go index 965df2a94bf..03b52c66da7 100644 --- a/pkg/workflow/frontmatter_on_section_cleanup.go +++ b/pkg/workflow/frontmatter_on_section_cleanup.go @@ -10,704 +10,537 @@ import ( // commentOutProcessedFieldsInOnSection comments out draft, fork, forks, names, labels, manual-approval, stop-after, skip-if-match, skip-if-no-match, skip-roles, reaction, lock-for-agent, steps, permissions, needs, restore-memory, and stale-check fields in the on section // These fields are processed separately and should be commented for documentation // Exception: names fields in sections with __gh_aw_native_label_filter__ marker in frontmatter are NOT commented out +type onSectionCleanupState struct { + result []string + + inPullRequest bool + inIssues bool + inDiscussion bool + inIssueComment bool + inDeploymentStatus bool + inWorkflowRun bool + inWorkflowRunConclusionArray bool + inForksArray bool + inSkipIfMatch bool + inSkipIfNoMatch bool + inSkipIfCheckFailing bool + inSkipAuthorAssociations bool + inSkipRolesArray bool + inSkipBotsArray bool + inRolesArray bool + inBotsArray bool + inLabelsArray bool + inNeedsArray bool + inGitHubApp bool + inOnSteps bool + inOnPermissions bool + commentBlockIndent string + inCommentBlock bool + currentSection string + currentSectionIndent int + deploymentStatusIndent int + workflowRunIndent int + nativeLabelFilterSections map[string]struct{} +} + func (c *Compiler) commentOutProcessedFieldsInOnSection(yamlStr string, frontmatter map[string]any) string { frontmatterLog.Print("Processing 'on' section to comment out processed fields") - // Check frontmatter for native label filter markers - nativeLabelFilterSections := make(map[string]struct { - }) - if onValue, exists := frontmatter["on"]; exists { - if onMap, ok := onValue.(map[string]any); ok { - for _, sectionKey := range []string{"issues", "pull_request", "discussion", "issue_comment"} { - if sectionValue, hasSec := onMap[sectionKey]; hasSec { - if sectionMap, ok := sectionValue.(map[string]any); ok { - if marker, hasMarker := sectionMap["__gh_aw_native_label_filter__"]; hasMarker { - if useNative, ok := marker.(bool); ok && useNative { - nativeLabelFilterSections[sectionKey] = struct { - }{} - frontmatterLog.Printf("Section %s uses native label filtering", sectionKey) - } - } - } - } - } - } + state := newOnSectionCleanupState(frontmatter) + for _, line := range strings.Split(yamlStr, "\n") { + state.processLine(line) } - lines := strings.Split(yamlStr, "\n") - var result []string - inPullRequest := false - inIssues := false - inDiscussion := false - inIssueComment := false - inDeploymentStatus := false - inWorkflowRun := false - inWorkflowRunConclusionArray := false - inForksArray := false - inSkipIfMatch := false - inSkipIfNoMatch := false - inSkipIfCheckFailing := false - inSkipAuthorAssociations := false - inSkipRolesArray := false - inSkipBotsArray := false - inRolesArray := false - inBotsArray := false - inLabelsArray := false - inNeedsArray := false - inGitHubApp := false - inOnSteps := false - inOnPermissions := false - // Track the leading indentation of the current run of consecutive commented-out - // lines. yamllint's comments-indentation rule flags any comment that is indented - // deeper than the comment above it, so we flatten every line of a commented block - // to the indentation of the block's first line (which sits at the surrounding - // content's level). inCommentBlock is false until the first line of a block is - // emitted and is reset whenever a real (uncommented) line is written. - commentBlockIndent := "" - inCommentBlock := false - currentSection := "" // Track which section we're in ("issues", "pull_request", "discussion", or "issue_comment") - currentSectionIndent := -1 - deploymentStatusIndent := -1 - workflowRunIndent := -1 - // activateEventSection resets all event-section flags and then activates the selected section. - // It also clears every top-level on: extension-array tracker (inBotsArray, inRolesArray, - // inSkipIfCheckFailing, etc.) before entering the new section. This reset is required - // because each activateEventSection call ends with "continue", which bypasses the - // indent-based deactivation logic further down the loop. Without the explicit reset here, - // a stale flag from a preceding bots:/roles:/skip-if-check-failing: block would cause that - // section's list items (e.g. "workflow_run.workflows: - CI") to be incorrectly commented out. - activateEventSection := func(section string, indent int) { - // Clear all top-level on: extension-array state so no sibling section leaks in. - inSkipRolesArray = false - inSkipBotsArray = false - inRolesArray = false - inBotsArray = false - inLabelsArray = false - inNeedsArray = false - // These trackers share the same exit-check-ordering issue: their deactivation - // logic runs after the "continue" that terminates each activateEventSection call, - // so they must also be reset here explicitly. - inSkipIfMatch = false - inSkipIfNoMatch = false - inSkipIfCheckFailing = false - inSkipAuthorAssociations = false - - // Reset the comment-block anchor so the first commented line of the new - // section uses its own indentation rather than a stale indent from the - // previous section (which would otherwise persist because activateEventSection - // ends with "continue" and bypasses the normal else-branch reset). - inCommentBlock = false - commentBlockIndent = "" - - inPullRequest = section == "pull_request" - inIssues = section == "issues" - inDiscussion = section == "discussion" - inIssueComment = section == "issue_comment" - inDeploymentStatus = section == "deployment_status" - inWorkflowRun = section == "workflow_run" - inWorkflowRunConclusionArray = false - inForksArray = false - - switch section { - case "pull_request", "issues", "discussion", "issue_comment": - currentSection = section - currentSectionIndent = indent - default: - currentSection = "" - currentSectionIndent = -1 - } + state.result = dedentTrailingOnCommentBlock(state.result) + return strings.Join(state.result, "\n") +} - if section == "deployment_status" { - deploymentStatusIndent = indent - } else { - deploymentStatusIndent = -1 - } - if section == "workflow_run" { - workflowRunIndent = indent - } else { - workflowRunIndent = -1 - } +func newOnSectionCleanupState(frontmatter map[string]any) *onSectionCleanupState { + return &onSectionCleanupState{ + currentSectionIndent: -1, + deploymentStatusIndent: -1, + workflowRunIndent: -1, + nativeLabelFilterSections: nativeLabelFilterSections(frontmatter), } +} - for _, line := range lines { - trimmedLine := strings.TrimSpace(line) - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - - // Check if we're entering a pull_request, issues, discussion, or issue_comment section. - // Skip these checks when inside on.permissions or on.steps to avoid false matches. - // Example: ` issues: read` inside on.permissions was previously matched as the - // `issues:` event trigger, incorrectly entering the inIssues state and suppressing - // the permission comment-out logic. - if !inOnPermissions && !inOnSteps && !inSkipAuthorAssociations { - if (lineIndent == 2 || lineIndent == 4) && trimmedLine == "pull_request:" { - activateEventSection("pull_request", lineIndent) - result = append(result, line) - continue - } - if (lineIndent == 2 || lineIndent == 4) && trimmedLine == "issues:" { - activateEventSection("issues", lineIndent) - result = append(result, line) - continue - } - if (lineIndent == 2 || lineIndent == 4) && trimmedLine == "discussion:" { - activateEventSection("discussion", lineIndent) - result = append(result, line) - continue - } - if (lineIndent == 2 || lineIndent == 4) && trimmedLine == "issue_comment:" { - activateEventSection("issue_comment", lineIndent) - result = append(result, line) - continue - } - if (lineIndent == 2 || lineIndent == 4) && trimmedLine == "deployment_status:" { - activateEventSection("deployment_status", lineIndent) - result = append(result, line) - continue - } - if (lineIndent == 2 || lineIndent == 4) && trimmedLine == "workflow_run:" { - activateEventSection("workflow_run", lineIndent) - result = append(result, line) - continue - } - } - - // Check if we're leaving the pull_request, issues, discussion, or issue_comment section (new top-level key or end of indent) - if inPullRequest || inIssues || inDiscussion || inIssueComment { - // If line is at or above section indentation, we're out of the section. - if strings.TrimSpace(line) != "" && !strings.HasPrefix(trimmedLine, "#") && - currentSectionIndent >= 0 && lineIndent <= currentSectionIndent { - inPullRequest = false - inIssues = false - inDiscussion = false - inIssueComment = false - inForksArray = false - currentSection = "" - currentSectionIndent = -1 - } - } - - // Check if we're leaving the deployment_status section - if inDeploymentStatus && strings.TrimSpace(line) != "" && !strings.HasPrefix(trimmedLine, "#") && - deploymentStatusIndent >= 0 && lineIndent <= deploymentStatusIndent { - inDeploymentStatus = false - deploymentStatusIndent = -1 - } - - // Check if we're leaving the workflow_run section - if inWorkflowRun && strings.TrimSpace(line) != "" && !strings.HasPrefix(trimmedLine, "#") && - workflowRunIndent >= 0 && lineIndent <= workflowRunIndent { - inWorkflowRun = false - inWorkflowRunConclusionArray = false - workflowRunIndent = -1 - } - - // Skip marker lines in the YAML output - if (inPullRequest || inIssues || inDiscussion || inIssueComment) && strings.Contains(trimmedLine, "__gh_aw_native_label_filter__:") { - // Don't include the marker line in the output +func nativeLabelFilterSections(frontmatter map[string]any) map[string]struct{} { + sections := make(map[string]struct{}) + onValue, exists := frontmatter["on"] + if !exists { + return sections + } + onMap, ok := onValue.(map[string]any) + if !ok { + return sections + } + for _, sectionKey := range []string{"issues", "pull_request", "discussion", "issue_comment"} { + sectionValue, hasSec := onMap[sectionKey] + sectionMap, ok := sectionValue.(map[string]any) + if !hasSec || !ok { continue } - - // Check if we're entering the forks array - if inPullRequest && strings.HasPrefix(trimmedLine, "forks:") { - inForksArray = true - } - - // Check if we're entering skip-roles array - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && strings.HasPrefix(trimmedLine, "skip-roles:") { - // Check if this is an array (next line will be "- ") - // We'll set the flag and handle it on the next iteration - inSkipRolesArray = true - } - - // Check if we're entering skip-bots array - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && strings.HasPrefix(trimmedLine, "skip-bots:") { - // Check if this is an array (next line will be "- ") - // We'll set the flag and handle it on the next iteration - inSkipBotsArray = true + marker, hasMarker := sectionMap["__gh_aw_native_label_filter__"] + useNative, ok := marker.(bool) + if hasMarker && ok && useNative { + sections[sectionKey] = struct{}{} + frontmatterLog.Printf("Section %s uses native label filtering", sectionKey) } + } + return sections +} - // Check if we're entering roles field - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && strings.HasPrefix(trimmedLine, "roles:") { - // Check if this is an array (next line will be "- ") or inline value - inRolesArray = true - } +func (s *onSectionCleanupState) processLine(line string) { + trimmedLine := strings.TrimSpace(line) + lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - // Check if we're entering bots array - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && strings.HasPrefix(trimmedLine, "bots:") { - // Check if this is an array (next line will be "- ") or inline value - inBotsArray = true - } + if s.tryEnterEventSection(line, trimmedLine, lineIndent) { + return + } + s.updateEventSectionExits(line, trimmedLine, lineIndent) + if s.shouldSkipNativeLabelMarker(trimmedLine) { + return + } + s.updateEntryState(trimmedLine, lineIndent) + s.updateExitState(line, trimmedLine) - // Check if we're entering labels array - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && - !inOnSteps && !inOnPermissions && - lineIndent == 2 && trimmedLine == "labels:" { - inLabelsArray = true - } + shouldComment, commentReason := s.commentDecision(line, trimmedLine, lineIndent) + s.appendProcessedLine(line, shouldComment, commentReason) +} - // Check if we're entering needs array - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && - !inOnSteps && !inOnPermissions && - lineIndent == 2 && strings.HasPrefix(trimmedLine, "needs:") { - inNeedsArray = true - } +func (s *onSectionCleanupState) tryEnterEventSection(line, trimmedLine string, lineIndent int) bool { + if s.inOnPermissions || s.inOnSteps || s.inSkipAuthorAssociations { + return false + } + if lineIndent != 2 && lineIndent != 4 { + return false + } - // Check if we're entering on.steps array - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && strings.HasPrefix(trimmedLine, "steps:") { - inOnSteps = true - } + section := "" + switch trimmedLine { + case "pull_request:": + section = "pull_request" + case "issues:": + section = "issues" + case "discussion:": + section = "discussion" + case "issue_comment:": + section = "issue_comment" + case "deployment_status:": + section = "deployment_status" + case "workflow_run:": + section = "workflow_run" + } + if section == "" { + return false + } - // Check if we're entering on.permissions object - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && !inOnPermissions && - strings.HasPrefix(trimmedLine, "permissions:") { - inOnPermissions = true - } + s.activateEventSection(section, lineIndent) + s.result = append(s.result, line) + return true +} - // Check if we're entering skip-if-match object - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && !inSkipIfMatch { - // Check both uncommented and commented forms - if (strings.HasPrefix(trimmedLine, "skip-if-match:") && trimmedLine == "skip-if-match:") || - (strings.HasPrefix(trimmedLine, "# skip-if-match:") && strings.Contains(trimmedLine, "pre-activation job")) { - inSkipIfMatch = true - } - } +func (s *onSectionCleanupState) activateEventSection(section string, indent int) { + s.inSkipRolesArray = false + s.inSkipBotsArray = false + s.inRolesArray = false + s.inBotsArray = false + s.inLabelsArray = false + s.inNeedsArray = false + s.inSkipIfMatch = false + s.inSkipIfNoMatch = false + s.inSkipIfCheckFailing = false + s.inSkipAuthorAssociations = false + s.inCommentBlock = false + s.commentBlockIndent = "" + s.inPullRequest = section == "pull_request" + s.inIssues = section == "issues" + s.inDiscussion = section == "discussion" + s.inIssueComment = section == "issue_comment" + s.inDeploymentStatus = section == "deployment_status" + s.inWorkflowRun = section == "workflow_run" + s.inWorkflowRunConclusionArray = false + s.inForksArray = false + + s.currentSection = "" + s.currentSectionIndent = -1 + if section == "pull_request" || section == "issues" || section == "discussion" || section == "issue_comment" { + s.currentSection = section + s.currentSectionIndent = indent + } + s.deploymentStatusIndent = -1 + if section == "deployment_status" { + s.deploymentStatusIndent = indent + } + s.workflowRunIndent = -1 + if section == "workflow_run" { + s.workflowRunIndent = indent + } +} - // Check if we're entering skip-if-no-match object - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && !inSkipIfNoMatch { - // Check both uncommented and commented forms - if (strings.HasPrefix(trimmedLine, "skip-if-no-match:") && trimmedLine == "skip-if-no-match:") || - (strings.HasPrefix(trimmedLine, "# skip-if-no-match:") && strings.Contains(trimmedLine, "pre-activation job")) { - inSkipIfNoMatch = true - } - } +func (s *onSectionCleanupState) updateEventSectionExits(line, trimmedLine string, lineIndent int) { + if (s.inPullRequest || s.inIssues || s.inDiscussion || s.inIssueComment) && isRealLineAtOrAbove(line, trimmedLine, lineIndent, s.currentSectionIndent) { + s.inPullRequest = false + s.inIssues = false + s.inDiscussion = false + s.inIssueComment = false + s.inForksArray = false + s.currentSection = "" + s.currentSectionIndent = -1 + } + if s.inDeploymentStatus && isRealLineAtOrAbove(line, trimmedLine, lineIndent, s.deploymentStatusIndent) { + s.inDeploymentStatus = false + s.deploymentStatusIndent = -1 + } + if s.inWorkflowRun && isRealLineAtOrAbove(line, trimmedLine, lineIndent, s.workflowRunIndent) { + s.inWorkflowRun = false + s.inWorkflowRunConclusionArray = false + s.workflowRunIndent = -1 + } +} - // Check if we're entering skip-if-check-failing object - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && !inSkipIfCheckFailing { - // Check both uncommented and commented forms - if trimmedLine == "skip-if-check-failing:" || - (strings.HasPrefix(trimmedLine, "# skip-if-check-failing:") && strings.Contains(trimmedLine, "pre-activation job")) { - inSkipIfCheckFailing = true - } - } +func isRealLineAtOrAbove(line, trimmedLine string, lineIndent, sectionIndent int) bool { + return strings.TrimSpace(line) != "" && !strings.HasPrefix(trimmedLine, "#") && sectionIndent >= 0 && lineIndent <= sectionIndent +} - // Check if we're entering skip-author-associations object - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && !inSkipAuthorAssociations { - if strings.HasPrefix(trimmedLine, "skip-author-associations:") && trimmedLine == "skip-author-associations:" { - inSkipAuthorAssociations = true - } - } +func (s *onSectionCleanupState) shouldSkipNativeLabelMarker(trimmedLine string) bool { + return (s.inPullRequest || s.inIssues || s.inDiscussion || s.inIssueComment) && strings.Contains(trimmedLine, "__gh_aw_native_label_filter__:") +} - // Check if we're entering github-app object - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment && !inGitHubApp { - // Check both uncommented and commented forms - if (strings.HasPrefix(trimmedLine, "github-app:") && trimmedLine == "github-app:") || - (strings.HasPrefix(trimmedLine, "# github-app:") && strings.Contains(trimmedLine, "pre-activation job")) { - inGitHubApp = true - } - } +func (s *onSectionCleanupState) updateEntryState(trimmedLine string, lineIndent int) { + s.updateTopLevelArrayEntryState(trimmedLine, lineIndent) + s.updateObjectEntryState(trimmedLine) +} - // Check if we're leaving skip-if-match object (encountering another top-level field) - // Skip this check if we just entered skip-if-match on this line - if inSkipIfMatch && strings.TrimSpace(line) != "" && - !strings.HasPrefix(trimmedLine, "skip-if-match:") && - !strings.HasPrefix(trimmedLine, "# skip-if-match:") { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - // If this is a field at same level as skip-if-match (2 spaces) and not a comment, we're out of skip-if-match - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "#") { - inSkipIfMatch = false - } - } +func (s *onSectionCleanupState) updateTopLevelArrayEntryState(trimmedLine string, lineIndent int) { + inEvent := s.inPullRequest || s.inIssues || s.inDiscussion || s.inIssueComment + if s.inPullRequest && strings.HasPrefix(trimmedLine, "forks:") { + s.inForksArray = true + } + if inEvent { + return + } + if strings.HasPrefix(trimmedLine, "skip-roles:") { + s.inSkipRolesArray = true + } + if strings.HasPrefix(trimmedLine, "skip-bots:") { + s.inSkipBotsArray = true + } + if strings.HasPrefix(trimmedLine, "roles:") { + s.inRolesArray = true + } + if strings.HasPrefix(trimmedLine, "bots:") { + s.inBotsArray = true + } + if !s.inOnSteps && !s.inOnPermissions && lineIndent == 2 && trimmedLine == "labels:" { + s.inLabelsArray = true + } + if !s.inOnSteps && !s.inOnPermissions && lineIndent == 2 && strings.HasPrefix(trimmedLine, "needs:") { + s.inNeedsArray = true + } + if strings.HasPrefix(trimmedLine, "steps:") { + s.inOnSteps = true + } + if !s.inOnPermissions && strings.HasPrefix(trimmedLine, "permissions:") { + s.inOnPermissions = true + } +} - // Check if we're leaving skip-if-no-match object (encountering another top-level field) - // Skip this check if we just entered skip-if-no-match on this line - if inSkipIfNoMatch && strings.TrimSpace(line) != "" && - !strings.HasPrefix(trimmedLine, "skip-if-no-match:") && - !strings.HasPrefix(trimmedLine, "# skip-if-no-match:") { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - // If this is a field at same level as skip-if-no-match (2 spaces) and not a comment, we're out of skip-if-no-match - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "#") { - inSkipIfNoMatch = false - } - } +func (s *onSectionCleanupState) updateObjectEntryState(trimmedLine string) { + if s.inPullRequest || s.inIssues || s.inDiscussion || s.inIssueComment { + return + } + if !s.inSkipIfMatch && ((strings.HasPrefix(trimmedLine, "skip-if-match:") && trimmedLine == "skip-if-match:") || + (strings.HasPrefix(trimmedLine, "# skip-if-match:") && strings.Contains(trimmedLine, "pre-activation job"))) { + s.inSkipIfMatch = true + } + if !s.inSkipIfNoMatch && ((strings.HasPrefix(trimmedLine, "skip-if-no-match:") && trimmedLine == "skip-if-no-match:") || + (strings.HasPrefix(trimmedLine, "# skip-if-no-match:") && strings.Contains(trimmedLine, "pre-activation job"))) { + s.inSkipIfNoMatch = true + } + if !s.inSkipIfCheckFailing && (trimmedLine == "skip-if-check-failing:" || + (strings.HasPrefix(trimmedLine, "# skip-if-check-failing:") && strings.Contains(trimmedLine, "pre-activation job"))) { + s.inSkipIfCheckFailing = true + } + if !s.inSkipAuthorAssociations && strings.HasPrefix(trimmedLine, "skip-author-associations:") && trimmedLine == "skip-author-associations:" { + s.inSkipAuthorAssociations = true + } + if !s.inGitHubApp && ((strings.HasPrefix(trimmedLine, "github-app:") && trimmedLine == "github-app:") || + (strings.HasPrefix(trimmedLine, "# github-app:") && strings.Contains(trimmedLine, "pre-activation job"))) { + s.inGitHubApp = true + } +} - // Check if we're leaving skip-if-check-failing object (encountering another top-level field) - // Skip this check if we just entered skip-if-check-failing on this line - if inSkipIfCheckFailing && strings.TrimSpace(line) != "" && - !strings.HasPrefix(trimmedLine, "skip-if-check-failing:") && - !strings.HasPrefix(trimmedLine, "# skip-if-check-failing:") { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - // If this is a field at same level as skip-if-check-failing (2 spaces) and not a comment, we're out - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "#") { - inSkipIfCheckFailing = false - } - } +func (s *onSectionCleanupState) updateExitState(line, trimmedLine string) { + lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) + s.updateObjectExitState(line, trimmedLine, lineIndent) + s.updateArrayExitState(line, trimmedLine, lineIndent) + s.updateOnStepsAndPermissionsExitState(line, trimmedLine, lineIndent) +} - // Check if we're leaving skip-author-associations object (encountering another top-level field) - if inSkipAuthorAssociations && strings.TrimSpace(line) != "" && - !strings.HasPrefix(trimmedLine, "skip-author-associations:") && - !strings.HasPrefix(trimmedLine, "# skip-author-associations:") { - currentIndent := len(line) - len(strings.TrimLeft(line, " \t")) - if currentIndent == 2 && !strings.HasPrefix(trimmedLine, "#") { - inSkipAuthorAssociations = false - } - } +func (s *onSectionCleanupState) updateObjectExitState(line, trimmedLine string, lineIndent int) { + if s.inSkipIfMatch && shouldExitTopLevelObject(line, trimmedLine, lineIndent, "skip-if-match:") { + s.inSkipIfMatch = false + } + if s.inSkipIfNoMatch && shouldExitTopLevelObject(line, trimmedLine, lineIndent, "skip-if-no-match:") { + s.inSkipIfNoMatch = false + } + if s.inSkipIfCheckFailing && shouldExitTopLevelObject(line, trimmedLine, lineIndent, "skip-if-check-failing:") { + s.inSkipIfCheckFailing = false + } + if s.inSkipAuthorAssociations && shouldExitTopLevelObject(line, trimmedLine, lineIndent, "skip-author-associations:") { + s.inSkipAuthorAssociations = false + } + if s.inGitHubApp && shouldExitTopLevelObject(line, trimmedLine, lineIndent, "github-app:") { + s.inGitHubApp = false + } +} - // Check if we're leaving github-app object (encountering another top-level field) - // Skip this check if we just entered github-app on this line - if inGitHubApp && strings.TrimSpace(line) != "" && - !strings.HasPrefix(trimmedLine, "github-app:") && - !strings.HasPrefix(trimmedLine, "# github-app:") { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - // If this is a field at same level as github-app (2 spaces) and not a comment, we're out of github-app - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "#") { - inGitHubApp = false - } - } +func shouldExitTopLevelObject(line, trimmedLine string, lineIndent int, field string) bool { + if strings.TrimSpace(line) == "" || strings.HasPrefix(trimmedLine, field) || strings.HasPrefix(trimmedLine, "# "+field) { + return false + } + return lineIndent == 2 && !strings.HasPrefix(trimmedLine, "#") +} - // Check if we're leaving the forks array by encountering another top-level field at the same level - if inForksArray && inPullRequest && strings.TrimSpace(line) != "" { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) +func (s *onSectionCleanupState) updateArrayExitState(line, trimmedLine string, lineIndent int) { + if s.inForksArray && s.inPullRequest && strings.TrimSpace(line) != "" && + lineIndent == 4 && !strings.HasPrefix(trimmedLine, "-") && !strings.HasPrefix(trimmedLine, "forks:") { + s.inForksArray = false + } + if s.inSkipRolesArray && shouldExitTopLevelArray(line, trimmedLine, lineIndent, "skip-roles:") { + s.inSkipRolesArray = false + } + if s.inSkipBotsArray && shouldExitTopLevelArray(line, trimmedLine, lineIndent, "skip-bots:") { + s.inSkipBotsArray = false + } + if s.inRolesArray && shouldExitTopLevelArray(line, trimmedLine, lineIndent, "roles:") { + s.inRolesArray = false + } + if s.inBotsArray && shouldExitTopLevelArray(line, trimmedLine, lineIndent, "bots:") { + s.inBotsArray = false + } + if s.inLabelsArray && shouldExitTopLevelArray(line, trimmedLine, lineIndent, "labels:") { + s.inLabelsArray = false + } + if s.inNeedsArray && shouldExitTopLevelArray(line, trimmedLine, lineIndent, "needs:") { + s.inNeedsArray = false + } +} - // If this is a non-dash line at the same level as the forks field (4 spaces), we're out of the array - if lineIndent == 4 && !strings.HasPrefix(trimmedLine, "-") && !strings.HasPrefix(trimmedLine, "forks:") { - inForksArray = false - } - } +func shouldExitTopLevelArray(line, trimmedLine string, lineIndent int, field string) bool { + return strings.TrimSpace(line) != "" && lineIndent == 2 && !strings.HasPrefix(trimmedLine, "-") && + !strings.HasPrefix(trimmedLine, field) && !strings.HasPrefix(trimmedLine, "#") +} - // Check if we're leaving the skip-roles array by encountering another top-level field - if inSkipRolesArray && strings.TrimSpace(line) != "" { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) +func (s *onSectionCleanupState) updateOnStepsAndPermissionsExitState(line, trimmedLine string, lineIndent int) { + if s.inOnSteps && strings.TrimSpace(line) != "" && lineIndent == 2 && !strings.HasPrefix(trimmedLine, "-") && + !strings.HasPrefix(trimmedLine, "steps:") && !strings.HasPrefix(trimmedLine, "#") { + s.inOnSteps = false + } + if s.inOnPermissions && strings.TrimSpace(line) != "" && !strings.HasPrefix(trimmedLine, "permissions:") && + !strings.HasPrefix(trimmedLine, "# permissions:") && lineIndent == 2 && !strings.HasPrefix(trimmedLine, "#") { + s.inOnPermissions = false + } +} - // If this is a non-dash line at the same level as skip-roles (2 spaces), we're out of the array - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "-") && !strings.HasPrefix(trimmedLine, "skip-roles:") && !strings.HasPrefix(trimmedLine, "#") { - inSkipRolesArray = false - } +func (s *onSectionCleanupState) commentDecision(line, trimmedLine string, lineIndent int) (bool, string) { + if !s.inPullRequest && !s.inIssues && !s.inDiscussion && !s.inIssueComment { + if shouldComment, reason := s.topLevelGateCommentDecision(trimmedLine, lineIndent); shouldComment { + return true, reason } - - // Check if we're leaving the skip-bots array by encountering another top-level field - if inSkipBotsArray && strings.TrimSpace(line) != "" { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - - // If this is a non-dash line at the same level as skip-bots (2 spaces), we're out of the array - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "-") && !strings.HasPrefix(trimmedLine, "skip-bots:") && !strings.HasPrefix(trimmedLine, "#") { - inSkipBotsArray = false - } + if shouldComment, reason := s.topLevelMetadataCommentDecision(trimmedLine, lineIndent); shouldComment { + return true, reason } + } + return s.eventCommentDecision(line, trimmedLine) +} - // Check if we're leaving the roles array by encountering another top-level field - if inRolesArray && strings.TrimSpace(line) != "" { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) +func (s *onSectionCleanupState) topLevelGateCommentDecision(trimmedLine string, lineIndent int) (bool, string) { + switch { + case strings.HasPrefix(trimmedLine, "manual-approval:"): + return true, " # Manual approval processed as environment field in activation job" + case strings.HasPrefix(trimmedLine, "stop-after:"): + return true, " # Stop-after processed as stop-time check in pre-activation job" + case strings.HasPrefix(trimmedLine, "skip-if-match:"): + return true, " # Skip-if-match processed as search check in pre-activation job" + case s.inSkipIfMatch && hasAnyOnSectionPrefix(trimmedLine, "query:", "max:", "scope:"): + return true, "" + case strings.HasPrefix(trimmedLine, "skip-if-no-match:"): + return true, " # Skip-if-no-match processed as search check in pre-activation job" + case s.inSkipIfNoMatch && hasAnyOnSectionPrefix(trimmedLine, "query:", "min:", "scope:"): + return true, "" + case strings.HasPrefix(trimmedLine, "skip-if-check-failing:"): + return true, " # Skip-if-check-failing processed as check status gate in pre-activation job" + case s.inSkipIfCheckFailing && hasAnyOnSectionPrefix(trimmedLine, "include:", "exclude:", "branch:", "allow-pending:", "-"): + return true, "" + case strings.HasPrefix(trimmedLine, "skip-author-associations:"): + return true, " # Skip-author-associations compiled into pre-activation job if condition" + case s.inSkipAuthorAssociations && lineIndent > 2: + return true, "" + } + return false, "" +} - // If this is a non-dash line at the same level as roles (2 spaces), we're out of the array - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "-") && !strings.HasPrefix(trimmedLine, "roles:") && !strings.HasPrefix(trimmedLine, "#") { - inRolesArray = false - } - } +func (s *onSectionCleanupState) topLevelMetadataCommentDecision(trimmedLine string, lineIndent int) (bool, string) { + if shouldComment, reason := s.topLevelListCommentDecision(trimmedLine, lineIndent); shouldComment { + return true, reason + } + switch { + case strings.HasPrefix(trimmedLine, "restore-memory:"): + return true, " # Restore-memory enables pre-activation memory restore" + case strings.HasPrefix(trimmedLine, "steps:"): + return true, " # Steps injected into pre-activation job" + case s.inOnSteps: + return true, "" + case strings.HasPrefix(trimmedLine, "permissions:"): + return true, " # Permissions applied to pre-activation job" + case s.inOnPermissions: + return true, "" + case strings.HasPrefix(trimmedLine, "reaction:"): + return true, " # Reaction processed as activation job step" + case strings.HasPrefix(trimmedLine, "github-token:"): + return true, " # GitHub token used for reactions and status comments in activation" + case strings.HasPrefix(trimmedLine, "github-app:"): + return true, " # GitHub App used to mint token for reactions and status comments in activation" + case s.inGitHubApp && isGitHubAppNestedField(trimmedLine): + return true, "" + case strings.HasPrefix(trimmedLine, "stale-check:"): + return true, " # Stale-check processed as frontmatter hash check step in activation job" + } + return false, "" +} - // Check if we're leaving the bots array by encountering another top-level field - if inBotsArray && strings.TrimSpace(line) != "" { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) +func (s *onSectionCleanupState) topLevelListCommentDecision(trimmedLine string, lineIndent int) (bool, string) { + switch { + case strings.HasPrefix(trimmedLine, "skip-roles:"): + return true, " # Skip-roles processed as role check in pre-activation job" + case s.inSkipRolesArray && strings.HasPrefix(trimmedLine, "-"): + return true, " # Skip-roles processed as role check in pre-activation job" + case strings.HasPrefix(trimmedLine, "skip-bots:"): + return true, " # Skip-bots processed as bot check in pre-activation job" + case s.inSkipBotsArray && strings.HasPrefix(trimmedLine, "-"): + return true, " # Skip-bots processed as bot check in pre-activation job" + case strings.HasPrefix(trimmedLine, "roles:"): + return true, " # Roles processed as role check in pre-activation job" + case s.inRolesArray && strings.HasPrefix(trimmedLine, "-"): + return true, " # Roles processed as role check in pre-activation job" + case strings.HasPrefix(trimmedLine, "bots:"): + return true, " # Bots processed as bot check in pre-activation job" + case s.inBotsArray && strings.HasPrefix(trimmedLine, "-"): + return true, " # Bots processed as bot check in pre-activation job" + case !s.inOnSteps && !s.inOnPermissions && lineIndent == 2 && strings.HasPrefix(trimmedLine, "labels:"): + return true, " # Label filtering applied via job conditions" + case s.inLabelsArray && strings.HasPrefix(trimmedLine, "-"): + return true, " # Label filtering applied via job conditions" + case !s.inOnSteps && !s.inOnPermissions && lineIndent == 2 && strings.HasPrefix(trimmedLine, "needs:"): + return true, " # Needs processed as dependency in pre-activation job" + case s.inNeedsArray && strings.HasPrefix(trimmedLine, "-"): + return true, " # Needs processed as dependency in pre-activation job" + } + return false, "" +} - // If this is a non-dash line at the same level as bots (2 spaces), we're out of the array - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "-") && !strings.HasPrefix(trimmedLine, "bots:") && !strings.HasPrefix(trimmedLine, "#") { - inBotsArray = false - } +func hasAnyOnSectionPrefix(value string, prefixes ...string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(value, prefix) { + return true } + } + return false +} - // Check if we're leaving the labels array by encountering another top-level field - if inLabelsArray && strings.TrimSpace(line) != "" { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - - // If this is a non-dash line at the same level as labels (2 spaces), we're out of the array - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "-") && !strings.HasPrefix(trimmedLine, "labels:") && !strings.HasPrefix(trimmedLine, "#") { - inLabelsArray = false - } - } +func (s *onSectionCleanupState) eventCommentDecision(line, trimmedLine string) (bool, string) { + switch { + case s.inPullRequest && strings.Contains(trimmedLine, "draft:"): + return true, " # Draft filtering applied via job conditions" + case s.inPullRequest && strings.HasPrefix(trimmedLine, "forks:"): + return true, " # Fork filtering applied via job conditions" + case s.inForksArray && strings.HasPrefix(trimmedLine, "-"): + return true, " # Fork filtering applied via job conditions" + case s.inDeploymentStatus && strings.HasPrefix(trimmedLine, "state:"): + return true, " # State filtering compiled into if condition" + case s.inDeploymentStatus && strings.HasPrefix(trimmedLine, "-"): + return true, " # State filtering compiled into if condition" + case s.inWorkflowRun && strings.HasPrefix(trimmedLine, "conclusion:"): + s.inWorkflowRunConclusionArray = true + return true, " # Conclusion filtering compiled into if condition" + case s.inWorkflowRunConclusionArray && strings.HasPrefix(trimmedLine, "-"): + return true, " # Conclusion filtering compiled into if condition" + case s.inWorkflowRun && !strings.HasPrefix(trimmedLine, "-") && strings.Contains(trimmedLine, ":"): + s.inWorkflowRunConclusionArray = false + return false, "" + case s.isIssueLikeEvent() && strings.HasPrefix(trimmedLine, "lock-for-agent:"): + return true, " # Lock-for-agent processed as issue locking in activation job" + case s.isIssueLikeEvent() && strings.HasPrefix(trimmedLine, "names:"): + return s.namesFieldCommentDecision() + case s.isIssueLikeEvent() && line != "": + return s.namesArrayItemCommentDecision(trimmedLine) + } + return false, "" +} - // Check if we're leaving the needs array by encountering another top-level field - if inNeedsArray && strings.TrimSpace(line) != "" { - // Get the indentation of the current line - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) +func (s *onSectionCleanupState) isIssueLikeEvent() bool { + return s.inPullRequest || s.inIssues || s.inDiscussion || s.inIssueComment +} - // If this is a non-dash line at the same level as needs (2 spaces), we're out of the array - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "-") && !strings.HasPrefix(trimmedLine, "needs:") && !strings.HasPrefix(trimmedLine, "#") { - inNeedsArray = false - } - } +func (s *onSectionCleanupState) namesFieldCommentDecision() (bool, string) { + if setutil.Contains(s.nativeLabelFilterSections, s.currentSection) { + return false, "" + } + return true, " # Label filtering applied via job conditions" +} - // Check if we're leaving the on.steps array by encountering another top-level field - if inOnSteps && strings.TrimSpace(line) != "" { - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - // If this is a line at the same level as steps (2 spaces) and not a dash or comment, we're out - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "-") && !strings.HasPrefix(trimmedLine, "steps:") && !strings.HasPrefix(trimmedLine, "#") { - inOnSteps = false - } +func (s *onSectionCleanupState) namesArrayItemCommentDecision(trimmedLine string) (bool, string) { + if setutil.Contains(s.nativeLabelFilterSections, s.currentSection) || len(s.result) == 0 { + return false, "" + } + for i := range slices.Backward(s.result) { + prevTrimmed := strings.TrimSpace(s.result[i]) + if prevTrimmed == "" { + continue } - - // Check if we're leaving the on.permissions object by encountering another top-level field - if inOnPermissions && strings.TrimSpace(line) != "" && - !strings.HasPrefix(trimmedLine, "permissions:") && - !strings.HasPrefix(trimmedLine, "# permissions:") { - lineIndent := len(line) - len(strings.TrimLeft(line, " \t")) - if lineIndent == 2 && !strings.HasPrefix(trimmedLine, "#") { - inOnPermissions = false - } + if strings.Contains(prevTrimmed, "names:") && strings.Contains(prevTrimmed, "# Label filtering") { + return strings.HasPrefix(trimmedLine, "-"), " # Label filtering applied via job conditions" } - - // Determine if we should comment out this line - shouldComment := false - var commentReason string - - // Check for top-level fields that should be commented out (not inside pull_request, issues, discussion, or issue_comment) - if !inPullRequest && !inIssues && !inDiscussion && !inIssueComment { - if strings.HasPrefix(trimmedLine, "manual-approval:") { - shouldComment = true - commentReason = " # Manual approval processed as environment field in activation job" - } else if strings.HasPrefix(trimmedLine, "stop-after:") { - shouldComment = true - commentReason = " # Stop-after processed as stop-time check in pre-activation job" - } else if strings.HasPrefix(trimmedLine, "skip-if-match:") { - shouldComment = true - commentReason = " # Skip-if-match processed as search check in pre-activation job" - } else if inSkipIfMatch && (strings.HasPrefix(trimmedLine, "query:") || strings.HasPrefix(trimmedLine, "max:") || strings.HasPrefix(trimmedLine, "scope:")) { - // Comment out nested fields in skip-if-match object - shouldComment = true - commentReason = "" - } else if strings.HasPrefix(trimmedLine, "skip-if-no-match:") { - shouldComment = true - commentReason = " # Skip-if-no-match processed as search check in pre-activation job" - } else if inSkipIfNoMatch && (strings.HasPrefix(trimmedLine, "query:") || strings.HasPrefix(trimmedLine, "min:") || strings.HasPrefix(trimmedLine, "scope:")) { - // Comment out nested fields in skip-if-no-match object - shouldComment = true - commentReason = "" - } else if strings.HasPrefix(trimmedLine, "skip-if-check-failing:") { - shouldComment = true - commentReason = " # Skip-if-check-failing processed as check status gate in pre-activation job" - } else if inSkipIfCheckFailing && (strings.HasPrefix(trimmedLine, "include:") || strings.HasPrefix(trimmedLine, "exclude:") || strings.HasPrefix(trimmedLine, "branch:") || strings.HasPrefix(trimmedLine, "allow-pending:") || strings.HasPrefix(trimmedLine, "-")) { - // Comment out nested fields and list items in skip-if-check-failing object - shouldComment = true - commentReason = "" - } else if strings.HasPrefix(trimmedLine, "skip-author-associations:") { - shouldComment = true - commentReason = " # Skip-author-associations compiled into pre-activation job if condition" - } else if inSkipAuthorAssociations && lineIndent > 2 { - shouldComment = true - commentReason = "" - } else if strings.HasPrefix(trimmedLine, "skip-roles:") { - shouldComment = true - commentReason = " # Skip-roles processed as role check in pre-activation job" - } else if inSkipRolesArray && strings.HasPrefix(trimmedLine, "-") { - // Comment out array items in skip-roles - shouldComment = true - commentReason = " # Skip-roles processed as role check in pre-activation job" - } else if strings.HasPrefix(trimmedLine, "skip-bots:") { - shouldComment = true - commentReason = " # Skip-bots processed as bot check in pre-activation job" - } else if inSkipBotsArray && strings.HasPrefix(trimmedLine, "-") { - // Comment out array items in skip-bots - shouldComment = true - commentReason = " # Skip-bots processed as bot check in pre-activation job" - } else if strings.HasPrefix(trimmedLine, "roles:") { - shouldComment = true - commentReason = " # Roles processed as role check in pre-activation job" - } else if inRolesArray && strings.HasPrefix(trimmedLine, "-") { - // Comment out array items in roles - shouldComment = true - commentReason = " # Roles processed as role check in pre-activation job" - } else if strings.HasPrefix(trimmedLine, "bots:") { - shouldComment = true - commentReason = " # Bots processed as bot check in pre-activation job" - } else if inBotsArray && strings.HasPrefix(trimmedLine, "-") { - // Comment out array items in bots - shouldComment = true - commentReason = " # Bots processed as bot check in pre-activation job" - } else if !inOnSteps && !inOnPermissions && lineIndent == 2 && strings.HasPrefix(trimmedLine, "labels:") { - shouldComment = true - commentReason = " # Label filtering applied via job conditions" - } else if inLabelsArray && strings.HasPrefix(trimmedLine, "-") { - // Comment out array items in labels - shouldComment = true - commentReason = " # Label filtering applied via job conditions" - } else if !inOnSteps && !inOnPermissions && lineIndent == 2 && strings.HasPrefix(trimmedLine, "needs:") { - shouldComment = true - commentReason = " # Needs processed as dependency in pre-activation job" - } else if inNeedsArray && strings.HasPrefix(trimmedLine, "-") { - // Comment out array items in needs - shouldComment = true - commentReason = " # Needs processed as dependency in pre-activation job" - } else if strings.HasPrefix(trimmedLine, "restore-memory:") { - shouldComment = true - commentReason = " # Restore-memory enables pre-activation memory restore" - } else if strings.HasPrefix(trimmedLine, "steps:") { - shouldComment = true - commentReason = " # Steps injected into pre-activation job" - } else if inOnSteps { - // Comment out all content of on.steps (both array items and their nested fields) - shouldComment = true - commentReason = "" - } else if strings.HasPrefix(trimmedLine, "permissions:") { - shouldComment = true - commentReason = " # Permissions applied to pre-activation job" - } else if inOnPermissions { - // Comment out all nested permission scope lines - shouldComment = true - commentReason = "" - } else if strings.HasPrefix(trimmedLine, "reaction:") { - shouldComment = true - commentReason = " # Reaction processed as activation job step" - } else if strings.HasPrefix(trimmedLine, "github-token:") { - shouldComment = true - commentReason = " # GitHub token used for reactions and status comments in activation" - } else if strings.HasPrefix(trimmedLine, "github-app:") { - shouldComment = true - commentReason = " # GitHub App used to mint token for reactions and status comments in activation" - } else if inGitHubApp && isGitHubAppNestedField(trimmedLine) { - // Comment out nested fields and array items in github-app object - shouldComment = true - commentReason = "" - } else if strings.HasPrefix(trimmedLine, "stale-check:") { - shouldComment = true - commentReason = " # Stale-check processed as frontmatter hash check step in activation job" - } + if !strings.HasPrefix(prevTrimmed, "#") || !strings.Contains(prevTrimmed, "Label filtering") { + break } - - if !shouldComment && inPullRequest && strings.Contains(trimmedLine, "draft:") { - shouldComment = true - commentReason = " # Draft filtering applied via job conditions" - } else if inPullRequest && strings.HasPrefix(trimmedLine, "forks:") { - shouldComment = true - commentReason = " # Fork filtering applied via job conditions" - } else if inForksArray && strings.HasPrefix(trimmedLine, "-") { - shouldComment = true - commentReason = " # Fork filtering applied via job conditions" - } else if inDeploymentStatus && strings.HasPrefix(trimmedLine, "state:") { - shouldComment = true - commentReason = " # State filtering compiled into if condition" - } else if inDeploymentStatus && strings.HasPrefix(trimmedLine, "-") { - // Comment out array items inside deployment_status.state - shouldComment = true - commentReason = " # State filtering compiled into if condition" - } else if inWorkflowRun && strings.HasPrefix(trimmedLine, "conclusion:") { - shouldComment = true - commentReason = " # Conclusion filtering compiled into if condition" - inWorkflowRunConclusionArray = true - } else if inWorkflowRunConclusionArray && strings.HasPrefix(trimmedLine, "-") { - // Comment out array items inside workflow_run.conclusion - shouldComment = true - commentReason = " # Conclusion filtering compiled into if condition" - } else if inWorkflowRun && !strings.HasPrefix(trimmedLine, "-") && strings.Contains(trimmedLine, ":") { - // Any new field inside workflow_run resets the conclusion array tracker - inWorkflowRunConclusionArray = false - } else if (inPullRequest || inIssues || inDiscussion || inIssueComment) && strings.HasPrefix(trimmedLine, "lock-for-agent:") { - shouldComment = true - commentReason = " # Lock-for-agent processed as issue locking in activation job" - } else if (inPullRequest || inIssues || inDiscussion || inIssueComment) && strings.HasPrefix(trimmedLine, "names:") { - // Only comment out names if NOT using native label filtering for this section - if !setutil.Contains(nativeLabelFilterSections, currentSection) { - shouldComment = true - commentReason = " # Label filtering applied via job conditions" + if strings.HasPrefix(prevTrimmed, "# -") && strings.Contains(prevTrimmed, "Label filtering") { + if strings.HasPrefix(trimmedLine, "-") { + return true, " # Label filtering applied via job conditions" } - } else if (inPullRequest || inIssues || inDiscussion || inIssueComment) && line != "" { - // Check if we're in a names array (after "names:" line) - // Look back to see if the previous uncommented line was "names:" - // Only do this if NOT using native label filtering for this section - if !setutil.Contains(nativeLabelFilterSections, currentSection) { - if len(result) > 0 { - for i := range slices.Backward(result) { - prevLine := result[i] - prevTrimmed := strings.TrimSpace(prevLine) - - // Skip empty lines - if prevTrimmed == "" { - continue - } - - // If we find "names:", and current line is an array item, comment it - if strings.Contains(prevTrimmed, "names:") && strings.Contains(prevTrimmed, "# Label filtering") { - if strings.HasPrefix(trimmedLine, "-") { - shouldComment = true - commentReason = " # Label filtering applied via job conditions" - } - break - } - - // If we find a different field or commented names array item, break - if !strings.HasPrefix(prevTrimmed, "#") || !strings.Contains(prevTrimmed, "Label filtering") { - break - } - - // If it's a commented names array item, continue - if strings.HasPrefix(prevTrimmed, "# -") && strings.Contains(prevTrimmed, "Label filtering") { - if strings.HasPrefix(trimmedLine, "-") { - shouldComment = true - commentReason = " # Label filtering applied via job conditions" - } - continue - } - - break - } - } - } // Close native filter check + continue } + break + } + return false, "" +} - if shouldComment { - trimmed := strings.TrimLeft(line, " \t") - - // Forks array items must preserve their own indentation so that commented - // array entries appear more deeply indented than the parent "forks:" key, - // matching the original YAML structure. Resetting the comment-block anchor - // here lets each "- …" item re-anchor to its own indentation level. - if inForksArray && strings.HasPrefix(trimmed, "-") { - inCommentBlock = false - commentBlockIndent = "" - } - - // Flatten the indentation of the commented block. The first non-blank - // line of a block adopts its natural indentation (which matches the - // surrounding content level) as the block anchor; every subsequent line - // — including blank lines inside a multi-line `steps:` script — reuses - // that same indentation. This keeps the whole block in a single comment - // group at a constant indent, so nested fields never appear more deeply - // indented than the comment above them (which is what yamllint's - // comments-indentation rule flags). Blank source lines are not allowed to - // anchor the block, so it always aligns to a real field. - if !inCommentBlock && trimmed != "" { - commentBlockIndent = "" - if len(line) > len(trimmed) { - commentBlockIndent = line[:len(line)-len(trimmed)] - } - inCommentBlock = true - } +func (s *onSectionCleanupState) appendProcessedLine(line string, shouldComment bool, commentReason string) { + if !shouldComment { + s.inCommentBlock = false + s.commentBlockIndent = "" + s.result = append(s.result, line) + return + } - commentedLine := commentBlockIndent + "# " + trimmed + commentReason - // Strip any trailing whitespace carried from the source content (e.g. - // commented-out multi-line `steps:` scripts whose lines end in spaces, or - // blank lines that would otherwise become "# " with trailing whitespace). - // Trailing whitespace on a comment is never meaningful and yamllint flags - // it as trailing-spaces. - commentedLine = strings.TrimRight(commentedLine, " \t") - result = append(result, commentedLine) - } else { - inCommentBlock = false - commentBlockIndent = "" - result = append(result, line) + trimmed := strings.TrimLeft(line, " \t") + if s.inForksArray && strings.HasPrefix(trimmed, "-") { + s.inCommentBlock = false + s.commentBlockIndent = "" + } + if !s.inCommentBlock && trimmed != "" { + s.commentBlockIndent = "" + if len(line) > len(trimmed) { + s.commentBlockIndent = line[:len(line)-len(trimmed)] } + s.inCommentBlock = true } - result = dedentTrailingOnCommentBlock(result) - - return strings.Join(result, "\n") + commentedLine := s.commentBlockIndent + "# " + trimmed + commentReason + commentedLine = strings.TrimRight(commentedLine, " \t") + s.result = append(s.result, commentedLine) } // dedentTrailingOnCommentBlock re-indents the final run of commented-out lines at the diff --git a/pkg/workflow/frontmatter_parsing.go b/pkg/workflow/frontmatter_parsing.go index 253d32e8d91..ee47b821962 100644 --- a/pkg/workflow/frontmatter_parsing.go +++ b/pkg/workflow/frontmatter_parsing.go @@ -13,10 +13,6 @@ func ParseFrontmatterConfig(frontmatter map[string]any) (*FrontmatterConfig, err frontmatterTypesLog.Printf("Parsing frontmatter config with %d fields", len(frontmatter)) var config FrontmatterConfig - // Use JSON marshaling for the entire frontmatter conversion. - // TemplatableInt32.UnmarshalJSON transparently handles both integer literals - // (e.g. timeout-minutes: 30) and GitHub Actions expressions - // (e.g. timeout-minutes: ${{ inputs.timeout }}) during unmarshaling. jsonBytes, err := json.Marshal(frontmatter) if err != nil { frontmatterTypesLog.Printf("Failed to marshal frontmatter: %v", err) @@ -28,75 +24,93 @@ func ParseFrontmatterConfig(frontmatter map[string]any) (*FrontmatterConfig, err return nil, fmt.Errorf("failed to unmarshal frontmatter into config: %w", err) } - if err := validateRunsOnValue(config.RunsOn); err != nil { + if err := validateFrontmatterRunsOnFields(frontmatter, &config); err != nil { return nil, err } + populateTypedFrontmatterConfig(frontmatter, &config) + + frontmatterTypesLog.Printf("Successfully parsed frontmatter config: name=%s, engine=%v", config.Name, config.Engine) + return &config, nil +} + +func validateFrontmatterRunsOnFields(frontmatter map[string]any, config *FrontmatterConfig) error { + if err := validateRunsOnValue(config.RunsOn); err != nil { + return err + } if err := validateRunsOnValue(config.RunsOnSlim); err != nil { - return nil, err + return err } if safeOutputsRaw, ok := frontmatter["safe-outputs"].(map[string]any); ok { if err := validateRunsOnValue(safeOutputsRaw["runs-on"]); err != nil { - return nil, err + return err } if threatRaw, ok := safeOutputsRaw["threat-detection"].(map[string]any); ok { - if err := validateRunsOnValue(threatRaw["runs-on"]); err != nil { - return nil, err - } + return validateRunsOnValue(threatRaw["runs-on"]) } } + return nil +} - // Parse typed Runtimes field if runtimes exist - if len(config.Runtimes) > 0 { - runtimesTyped, err := parseRuntimesConfig(config.Runtimes) - if err == nil { - config.RuntimesTyped = runtimesTyped - frontmatterTypesLog.Printf("Parsed typed runtimes config with %d runtimes", countRuntimes(runtimesTyped)) - } - } +func populateTypedFrontmatterConfig(frontmatter map[string]any, config *FrontmatterConfig) { + populateTypedRuntimes(config) + populateTypedPermissions(config) + populateCheckoutConfigs(config) + populateOnNeedsConfig(config) - // Parse typed Permissions field if permissions exist - if len(config.Permissions) > 0 { - permissionsTyped, err := parsePermissionsConfig(config.Permissions) - if err == nil { - config.PermissionsTyped = permissionsTyped - frontmatterTypesLog.Print("Parsed typed permissions config") - } + config.ExperimentConfigs = extractExperimentConfigsFromFrontmatter(frontmatter) + config.ModelPolicyAllowed, config.ModelPolicyBlocked = extractModelPolicyFromFrontmatter(frontmatter) + if rawSkills, ok := frontmatter["skills"].([]any); ok { + config.SkillReferences = parseRawSkillReferences(rawSkills) } +} - // Parse checkout field - supports single object, array of objects, or false to disable - if config.Checkout != nil { - if checkoutValue, ok := config.Checkout.(bool); ok && !checkoutValue { - config.CheckoutDisabled = true - frontmatterTypesLog.Print("Checkout disabled via checkout: false") - } else { - checkoutConfigs, err := ParseCheckoutConfigs(config.Checkout) - if err == nil { - config.CheckoutConfigs = checkoutConfigs - frontmatterTypesLog.Printf("Parsed checkout config: %d entries", len(checkoutConfigs)) - } - } +func populateTypedRuntimes(config *FrontmatterConfig) { + if len(config.Runtimes) == 0 { + return } + runtimesTyped, err := parseRuntimesConfig(config.Runtimes) + if err == nil { + config.RuntimesTyped = runtimesTyped + frontmatterTypesLog.Printf("Parsed typed runtimes config with %d runtimes", countRuntimes(runtimesTyped)) + } +} - // Parse typed on.needs field if on exists - if len(config.On) > 0 { - onNeeds, err := parseOnNeedsConfig(config.On) - if err == nil { - config.OnNeeds = onNeeds - frontmatterTypesLog.Printf("Parsed typed on.needs config with %d entries", len(onNeeds)) - } +func populateTypedPermissions(config *FrontmatterConfig) { + if len(config.Permissions) == 0 { + return + } + permissionsTyped, err := parsePermissionsConfig(config.Permissions) + if err == nil { + config.PermissionsTyped = permissionsTyped + frontmatterTypesLog.Print("Parsed typed permissions config") } +} - // Populate typed ExperimentConfigs from the raw frontmatter map so that both the - // legacy bare-array form and the new object form are available as ExperimentConfig - // structs without callers needing to type-assert config.Experiments entries. - config.ExperimentConfigs = extractExperimentConfigsFromFrontmatter(frontmatter) - config.ModelPolicyAllowed, config.ModelPolicyBlocked = extractModelPolicyFromFrontmatter(frontmatter) - if rawSkills, ok := frontmatter["skills"].([]any); ok { - config.SkillReferences = parseRawSkillReferences(rawSkills) +func populateCheckoutConfigs(config *FrontmatterConfig) { + if config.Checkout == nil { + return + } + if checkoutValue, ok := config.Checkout.(bool); ok && !checkoutValue { + config.CheckoutDisabled = true + frontmatterTypesLog.Print("Checkout disabled via checkout: false") + return + } + checkoutConfigs, err := ParseCheckoutConfigs(config.Checkout) + if err == nil { + config.CheckoutConfigs = checkoutConfigs + frontmatterTypesLog.Printf("Parsed checkout config: %d entries", len(checkoutConfigs)) } +} - frontmatterTypesLog.Printf("Successfully parsed frontmatter config: name=%s, engine=%v", config.Name, config.Engine) - return &config, nil +func populateOnNeedsConfig(config *FrontmatterConfig) { + if len(config.On) == 0 { + return + } + onNeeds, err := parseOnNeedsConfig(config.On) + if err == nil { + config.OnNeeds = onNeeds + frontmatterTypesLog.Printf("Parsed typed on.needs config with %d entries", len(onNeeds)) + } } func extractModelPolicyFromFrontmatter(frontmatter map[string]any) ([]string, []string) { @@ -149,212 +163,175 @@ func parseRuntimesConfig(runtimes map[string]any) (*RuntimesConfig, error) { continue } - // Extract version (optional) - var version string - if versionAny, hasVersion := configMap["version"]; hasVersion { - // Convert version to string - switch v := versionAny.(type) { - case string: - version = v - case int: - version = strconv.Itoa(v) - case float64: - if v == float64(int(v)) { - version = strconv.Itoa(int(v)) - } else { - version = fmt.Sprintf("%g", v) - } - default: - continue - } - } - - // Extract if condition (optional) - var ifCondition string - if ifAny, hasIf := configMap["if"]; hasIf { - if ifStr, ok := ifAny.(string); ok { - ifCondition = ifStr - } + runtimeConfig, ok := parseRuntimeConfigMap(configMap) + if !ok { + continue } + assignRuntimeConfig(config, runtimeID, runtimeConfig) + } - // Extract action-repo and action-version overrides (optional) - actionRepo, _ := configMap["action-repo"].(string) - actionVersion, _ := configMap["action-version"].(string) + return config, nil +} - // Extract run-install-scripts flag (optional) - var runInstallScripts *bool - if rsAny, hasRS := configMap["run-install-scripts"]; hasRS { - if rsBool, ok := rsAny.(bool); ok { - runInstallScripts = &rsBool - } +func parseRuntimeConfigMap(configMap map[string]any) (*RuntimeConfig, bool) { + var version string + if versionAny, hasVersion := configMap["version"]; hasVersion { + var ok bool + version, ok = parseRuntimeVersion(versionAny) + if !ok { + return nil, false } + } - // Extract cooldown flag (optional, default true when omitted) - var cooldown *bool - if cooldownAny, hasCooldown := configMap["cooldown"]; hasCooldown { - if cooldownBool, ok := cooldownAny.(bool); ok { - cooldown = &cooldownBool - } - } + ifCondition, _ := configMap["if"].(string) + actionRepo, _ := configMap["action-repo"].(string) + actionVersion, _ := configMap["action-version"].(string) + runInstallScripts := parseOptionalBool(configMap["run-install-scripts"]) + cooldown := parseOptionalBool(configMap["cooldown"]) - // Create runtime config with all fields - runtimeConfig := &RuntimeConfig{ - Version: version, - If: ifCondition, - ActionRepo: actionRepo, - ActionVersion: actionVersion, - Cooldown: cooldown, - RunInstallScripts: runInstallScripts, - } + return &RuntimeConfig{ + Version: version, + If: ifCondition, + ActionRepo: actionRepo, + ActionVersion: actionVersion, + Cooldown: cooldown, + RunInstallScripts: runInstallScripts, + }, true +} - // Map to specific runtime field - switch runtimeID { - case "node": - config.Node = runtimeConfig - case "python": - config.Python = runtimeConfig - case "go": - config.Go = runtimeConfig - case "uv": - config.UV = runtimeConfig - case "bun": - config.Bun = runtimeConfig - case "deno": - config.Deno = runtimeConfig - case "dotnet": - config.Dotnet = runtimeConfig - case "elixir": - config.Elixir = runtimeConfig - case "gh-aw": - config.GhAw = runtimeConfig - case "haskell": - config.Haskell = runtimeConfig - case "java": - config.Java = runtimeConfig - case "ruby": - config.Ruby = runtimeConfig +func parseRuntimeVersion(versionAny any) (string, bool) { + switch v := versionAny.(type) { + case string: + return v, true + case int: + return strconv.Itoa(v), true + case float64: + if v == float64(int(v)) { + return strconv.Itoa(int(v)), true } + return fmt.Sprintf("%g", v), true + default: + return "", false } +} - return config, nil +func parseOptionalBool(value any) *bool { + if boolValue, ok := value.(bool); ok { + return &boolValue + } + return nil +} + +func assignRuntimeConfig(config *RuntimesConfig, runtimeID string, runtimeConfig *RuntimeConfig) { + switch runtimeID { + case "node": + config.Node = runtimeConfig + case "python": + config.Python = runtimeConfig + case "go": + config.Go = runtimeConfig + case "uv": + config.UV = runtimeConfig + case "bun": + config.Bun = runtimeConfig + case "deno": + config.Deno = runtimeConfig + case "dotnet": + config.Dotnet = runtimeConfig + case "elixir": + config.Elixir = runtimeConfig + case "gh-aw": + config.GhAw = runtimeConfig + case "haskell": + config.Haskell = runtimeConfig + case "java": + config.Java = runtimeConfig + case "ruby": + config.Ruby = runtimeConfig + } } // parsePermissionsConfig converts a map[string]any to PermissionsConfig func parsePermissionsConfig(permissions map[string]any) (*PermissionsConfig, error) { config := &PermissionsConfig{} - // Check if it's a shorthand permission (single string value) - if len(permissions) == 1 { - for key, value := range permissions { - if strValue, ok := value.(string); ok { - shorthandPerms := []string{"read-all", "write-all", "read", "write", "none"} - for _, shorthand := range shorthandPerms { - if key == shorthand || strValue == shorthand { - config.Shorthand = shorthand - return config, nil - } - } - } - } + if shorthand, ok := parsePermissionShorthand(permissions); ok { + config.Shorthand = shorthand + return config, nil } - // Parse detailed permissions for scope, level := range permissions { if levelStr, ok := level.(string); ok { - switch scope { - // GitHub Actions permission scopes - case "actions": - config.Actions = levelStr - case "checks": - config.Checks = levelStr - case "contents": - config.Contents = levelStr - case "deployments": - config.Deployments = levelStr - case "id-token": - config.IDToken = levelStr - case "issues": - config.Issues = levelStr - case "discussions": - config.Discussions = levelStr - case "packages": - config.Packages = levelStr - case "pages": - config.Pages = levelStr - case "pull-requests": - config.PullRequests = levelStr - case "repository-projects": - config.RepositoryProjects = levelStr - case "security-events": - config.SecurityEvents = levelStr - case "statuses": - config.Statuses = levelStr - case "vulnerability-alerts": - config.VulnerabilityAlerts = levelStr - case "organization-projects": - config.OrganizationProjects = levelStr - // GitHub App-only permission scopes - case "administration": - config.Administration = levelStr - case "environments": - config.Environments = levelStr - case "git-signing": - config.GitSigning = levelStr - case "workflows": - config.Workflows = levelStr - case "repository-hooks": - config.RepositoryHooks = levelStr - case "single-file": - config.SingleFile = levelStr - case "codespaces": - config.Codespaces = levelStr - case "repository-custom-properties": - config.RepositoryCustomProperties = levelStr - case "members": - config.Members = levelStr - case "organization-administration": - config.OrganizationAdministration = levelStr - case "team-discussions": - config.TeamDiscussions = levelStr - case "organization-hooks": - config.OrganizationHooks = levelStr - case "organization-members": - config.OrganizationMembers = levelStr - case "organization-packages": - config.OrganizationPackages = levelStr - case "organization-self-hosted-runners": - config.OrganizationSelfHostedRunners = levelStr - case "organization-custom-org-roles": - config.OrganizationCustomOrgRoles = levelStr - case "organization-custom-properties": - config.OrganizationCustomProperties = levelStr - case "organization-custom-repository-roles": - config.OrganizationCustomRepositoryRoles = levelStr - case "organization-announcement-banners": - config.OrganizationAnnouncementBanners = levelStr - case "organization-events": - config.OrganizationEvents = levelStr - case "organization-plan": - config.OrganizationPlan = levelStr - case "organization-user-blocking": - config.OrganizationUserBlocking = levelStr - case "organization-personal-access-token-requests": - config.OrganizationPersonalAccessTokenReqs = levelStr - case "organization-personal-access-tokens": - config.OrganizationPersonalAccessTokens = levelStr - case "organization-copilot": - config.OrganizationCopilot = levelStr - case "organization-codespaces": - config.OrganizationCodespaces = levelStr - case "email-addresses": - config.EmailAddresses = levelStr - case "codespaces-lifecycle-admin": - config.CodespacesLifecycleAdmin = levelStr - case "codespaces-metadata": - config.CodespacesMetadata = levelStr + if setter, exists := permissionScopeSetters[scope]; exists { + setter(config, levelStr) } } } return config, nil } + +func parsePermissionShorthand(permissions map[string]any) (string, bool) { + if len(permissions) != 1 { + return "", false + } + for key, value := range permissions { + strValue, ok := value.(string) + if !ok { + return "", false + } + for _, shorthand := range []string{"read-all", "write-all", "read", "write", "none"} { + if key == shorthand || strValue == shorthand { + return shorthand, true + } + } + } + return "", false +} + +var permissionScopeSetters = map[string]func(*PermissionsConfig, string){ + "actions": func(c *PermissionsConfig, v string) { c.Actions = v }, + "checks": func(c *PermissionsConfig, v string) { c.Checks = v }, + "contents": func(c *PermissionsConfig, v string) { c.Contents = v }, + "deployments": func(c *PermissionsConfig, v string) { c.Deployments = v }, + "id-token": func(c *PermissionsConfig, v string) { c.IDToken = v }, + "issues": func(c *PermissionsConfig, v string) { c.Issues = v }, + "discussions": func(c *PermissionsConfig, v string) { c.Discussions = v }, + "packages": func(c *PermissionsConfig, v string) { c.Packages = v }, + "pages": func(c *PermissionsConfig, v string) { c.Pages = v }, + "pull-requests": func(c *PermissionsConfig, v string) { c.PullRequests = v }, + "repository-projects": func(c *PermissionsConfig, v string) { c.RepositoryProjects = v }, + "security-events": func(c *PermissionsConfig, v string) { c.SecurityEvents = v }, + "statuses": func(c *PermissionsConfig, v string) { c.Statuses = v }, + "vulnerability-alerts": func(c *PermissionsConfig, v string) { c.VulnerabilityAlerts = v }, + "organization-projects": func(c *PermissionsConfig, v string) { c.OrganizationProjects = v }, + "administration": func(c *PermissionsConfig, v string) { c.Administration = v }, + "environments": func(c *PermissionsConfig, v string) { c.Environments = v }, + "git-signing": func(c *PermissionsConfig, v string) { c.GitSigning = v }, + "workflows": func(c *PermissionsConfig, v string) { c.Workflows = v }, + "repository-hooks": func(c *PermissionsConfig, v string) { c.RepositoryHooks = v }, + "single-file": func(c *PermissionsConfig, v string) { c.SingleFile = v }, + "codespaces": func(c *PermissionsConfig, v string) { c.Codespaces = v }, + "repository-custom-properties": func(c *PermissionsConfig, v string) { c.RepositoryCustomProperties = v }, + "members": func(c *PermissionsConfig, v string) { c.Members = v }, + "organization-administration": func(c *PermissionsConfig, v string) { c.OrganizationAdministration = v }, + "team-discussions": func(c *PermissionsConfig, v string) { c.TeamDiscussions = v }, + "organization-hooks": func(c *PermissionsConfig, v string) { c.OrganizationHooks = v }, + "organization-members": func(c *PermissionsConfig, v string) { c.OrganizationMembers = v }, + "organization-packages": func(c *PermissionsConfig, v string) { c.OrganizationPackages = v }, + "organization-self-hosted-runners": func(c *PermissionsConfig, v string) { c.OrganizationSelfHostedRunners = v }, + "organization-custom-org-roles": func(c *PermissionsConfig, v string) { c.OrganizationCustomOrgRoles = v }, + "organization-custom-properties": func(c *PermissionsConfig, v string) { c.OrganizationCustomProperties = v }, + "organization-custom-repository-roles": func(c *PermissionsConfig, v string) { c.OrganizationCustomRepositoryRoles = v }, + "organization-announcement-banners": func(c *PermissionsConfig, v string) { c.OrganizationAnnouncementBanners = v }, + "organization-events": func(c *PermissionsConfig, v string) { c.OrganizationEvents = v }, + "organization-plan": func(c *PermissionsConfig, v string) { c.OrganizationPlan = v }, + "organization-user-blocking": func(c *PermissionsConfig, v string) { c.OrganizationUserBlocking = v }, + "organization-personal-access-token-requests": func(c *PermissionsConfig, v string) { c.OrganizationPersonalAccessTokenReqs = v }, + "organization-personal-access-tokens": func(c *PermissionsConfig, v string) { c.OrganizationPersonalAccessTokens = v }, + "organization-copilot": func(c *PermissionsConfig, v string) { c.OrganizationCopilot = v }, + "organization-codespaces": func(c *PermissionsConfig, v string) { c.OrganizationCodespaces = v }, + "email-addresses": func(c *PermissionsConfig, v string) { c.EmailAddresses = v }, + "codespaces-lifecycle-admin": func(c *PermissionsConfig, v string) { c.CodespacesLifecycleAdmin = v }, + "codespaces-metadata": func(c *PermissionsConfig, v string) { c.CodespacesMetadata = v }, +} diff --git a/pkg/workflow/frontmatter_serialization.go b/pkg/workflow/frontmatter_serialization.go index 7996eb3a7c3..c0e68cd572e 100644 --- a/pkg/workflow/frontmatter_serialization.go +++ b/pkg/workflow/frontmatter_serialization.go @@ -64,7 +64,17 @@ func (fc *FrontmatterConfig) ToMap() map[string]any { frontmatterTypesLog.Printf("Converting FrontmatterConfig to map: name=%s", fc.Name) result := make(map[string]any) - // Core fields + fc.addCoreFieldsToMap(result) + fc.addConfigurationSectionsToMap(result) + fc.addEventAndTriggerFieldsToMap(result) + fc.addNetworkAndSandboxFieldsToMap(result) + fc.addFeatureAndEnvironmentFieldsToMap(result) + fc.addExecutionSettingsToMap(result) + fc.addImportAndMetadataFieldsToMap(result) + return result +} + +func (fc *FrontmatterConfig) addCoreFieldsToMap(result map[string]any) { if fc.Name != "" { result["name"] = fc.Name } @@ -95,15 +105,15 @@ func (fc *FrontmatterConfig) ToMap() map[string]any { if len(fc.Labels) > 0 { result["labels"] = fc.Labels } +} - // Configuration sections +func (fc *FrontmatterConfig) addConfigurationSectionsToMap(result map[string]any) { if fc.Tools != nil { result["tools"] = fc.Tools.ToMap() } if fc.MCPServers != nil { result["mcp-servers"] = fc.MCPServers } - // Prefer RuntimesTyped over Runtimes for conversion if fc.RuntimesTyped != nil { result["runtimes"] = runtimesConfigToMap(fc.RuntimesTyped) } else if fc.Runtimes != nil { @@ -120,12 +130,12 @@ func (fc *FrontmatterConfig) ToMap() map[string]any { // Convert MCPScriptsConfig to map - would need a ToMap method result["mcp-scripts"] = fc.MCPScripts } +} - // Event and trigger configuration +func (fc *FrontmatterConfig) addEventAndTriggerFieldsToMap(result map[string]any) { if fc.On != nil { result["on"] = fc.On } - // Prefer PermissionsTyped over Permissions for conversion if fc.PermissionsTyped != nil { result["permissions"] = permissionsConfigToMap(fc.PermissionsTyped) } else if fc.Permissions != nil { @@ -137,37 +147,41 @@ func (fc *FrontmatterConfig) ToMap() map[string]any { if fc.If != "" { result["if"] = fc.If } +} - // Network and sandbox +func (fc *FrontmatterConfig) addNetworkAndSandboxFieldsToMap(result map[string]any) { if fc.Network != nil { - // Convert NetworkPermissions to map format - // If allowed list is just ["defaults"], convert to string format "defaults" - if len(fc.Network.Allowed) == 1 && fc.Network.Allowed[0] == "defaults" && !fc.Network.AllowedInput && fc.Network.Firewall == nil && len(fc.Network.Blocked) == 0 { - result["network"] = "defaults" - } else { - networkMap := make(map[string]any) - if len(fc.Network.Allowed) > 0 { - networkMap["allowed"] = fc.Network.Allowed - } - if fc.Network.AllowedInput { - networkMap["allowed-input"] = true - } - if len(fc.Network.Blocked) > 0 { - networkMap["blocked"] = fc.Network.Blocked - } - if fc.Network.Firewall != nil { - networkMap["firewall"] = fc.Network.Firewall - } - if len(networkMap) > 0 { - result["network"] = networkMap - } + if networkValue, ok := networkPermissionsToMapValue(fc.Network); ok { + result["network"] = networkValue } } if fc.Sandbox != nil { result["sandbox"] = fc.Sandbox } +} + +func networkPermissionsToMapValue(network *NetworkPermissions) (any, bool) { + if len(network.Allowed) == 1 && network.Allowed[0] == "defaults" && + !network.AllowedInput && network.Firewall == nil && len(network.Blocked) == 0 { + return "defaults", true + } + networkMap := make(map[string]any) + if len(network.Allowed) > 0 { + networkMap["allowed"] = network.Allowed + } + if network.AllowedInput { + networkMap["allowed-input"] = true + } + if len(network.Blocked) > 0 { + networkMap["blocked"] = network.Blocked + } + if network.Firewall != nil { + networkMap["firewall"] = network.Firewall + } + return networkMap, len(networkMap) > 0 +} - // Features and environment +func (fc *FrontmatterConfig) addFeatureAndEnvironmentFieldsToMap(result map[string]any) { if fc.Features != nil { result["features"] = fc.Features } @@ -177,8 +191,9 @@ func (fc *FrontmatterConfig) ToMap() map[string]any { if fc.Secrets != nil { result["secrets"] = fc.Secrets } +} - // Execution settings +func (fc *FrontmatterConfig) addExecutionSettingsToMap(result map[string]any) { if !isNilValue(fc.RunsOn) { result["runs-on"] = fc.RunsOn } @@ -212,8 +227,9 @@ func (fc *FrontmatterConfig) ToMap() map[string]any { if fc.Cache != nil { result["cache"] = fc.Cache } +} - // Import and inclusion +func (fc *FrontmatterConfig) addImportAndMetadataFieldsToMap(result map[string]any) { if fc.Imports != nil { result["imports"] = fc.Imports } @@ -228,8 +244,6 @@ func (fc *FrontmatterConfig) ToMap() map[string]any { if fc.SecretMasking != nil { result["secret-masking"] = fc.SecretMasking } - - return result } func isNilValue(v any) bool { @@ -326,149 +340,77 @@ func permissionsConfigToMap(config *PermissionsConfig) map[string]any { frontmatterTypesLog.Print("Converting detailed PermissionsConfig to map") result := make(map[string]any) - // GitHub Actions permission scopes - if config.Actions != "" { - result["actions"] = config.Actions - } - if config.Checks != "" { - result["checks"] = config.Checks - } - if config.Contents != "" { - result["contents"] = config.Contents - } - if config.Deployments != "" { - result["deployments"] = config.Deployments - } - if config.IDToken != "" { - result["id-token"] = config.IDToken - } - if config.Issues != "" { - result["issues"] = config.Issues - } - if config.Discussions != "" { - result["discussions"] = config.Discussions - } - if config.Packages != "" { - result["packages"] = config.Packages - } - if config.Pages != "" { - result["pages"] = config.Pages - } - if config.PullRequests != "" { - result["pull-requests"] = config.PullRequests - } - if config.RepositoryProjects != "" { - result["repository-projects"] = config.RepositoryProjects - } - if config.SecurityEvents != "" { - result["security-events"] = config.SecurityEvents - } - if config.Statuses != "" { - result["statuses"] = config.Statuses - } - if config.VulnerabilityAlerts != "" { - result["vulnerability-alerts"] = config.VulnerabilityAlerts - } - if config.OrganizationProjects != "" { - result["organization-projects"] = config.OrganizationProjects - } + addPermissionScopeValues(result, permissionConfigScopeValues(config)) - // GitHub App-only permission scopes - repository-level - if config.Administration != "" { - result["administration"] = config.Administration - } - if config.Environments != "" { - result["environments"] = config.Environments - } - if config.GitSigning != "" { - result["git-signing"] = config.GitSigning - } - if config.Workflows != "" { - result["workflows"] = config.Workflows - } - if config.RepositoryHooks != "" { - result["repository-hooks"] = config.RepositoryHooks - } - if config.SingleFile != "" { - result["single-file"] = config.SingleFile - } - if config.Codespaces != "" { - result["codespaces"] = config.Codespaces - } - if config.RepositoryCustomProperties != "" { - result["repository-custom-properties"] = config.RepositoryCustomProperties + if len(result) == 0 { + return nil } - // GitHub App-only permission scopes - organization-level - if config.Members != "" { - result["members"] = config.Members - } - if config.OrganizationAdministration != "" { - result["organization-administration"] = config.OrganizationAdministration - } - if config.TeamDiscussions != "" { - result["team-discussions"] = config.TeamDiscussions - } - if config.OrganizationHooks != "" { - result["organization-hooks"] = config.OrganizationHooks - } - if config.OrganizationMembers != "" { - result["organization-members"] = config.OrganizationMembers - } - if config.OrganizationPackages != "" { - result["organization-packages"] = config.OrganizationPackages - } - if config.OrganizationSelfHostedRunners != "" { - result["organization-self-hosted-runners"] = config.OrganizationSelfHostedRunners - } - if config.OrganizationCustomOrgRoles != "" { - result["organization-custom-org-roles"] = config.OrganizationCustomOrgRoles - } - if config.OrganizationCustomProperties != "" { - result["organization-custom-properties"] = config.OrganizationCustomProperties - } - if config.OrganizationCustomRepositoryRoles != "" { - result["organization-custom-repository-roles"] = config.OrganizationCustomRepositoryRoles - } - if config.OrganizationAnnouncementBanners != "" { - result["organization-announcement-banners"] = config.OrganizationAnnouncementBanners - } - if config.OrganizationEvents != "" { - result["organization-events"] = config.OrganizationEvents - } - if config.OrganizationPlan != "" { - result["organization-plan"] = config.OrganizationPlan - } - if config.OrganizationUserBlocking != "" { - result["organization-user-blocking"] = config.OrganizationUserBlocking - } - if config.OrganizationPersonalAccessTokenReqs != "" { - result["organization-personal-access-token-requests"] = config.OrganizationPersonalAccessTokenReqs - } - if config.OrganizationPersonalAccessTokens != "" { - result["organization-personal-access-tokens"] = config.OrganizationPersonalAccessTokens - } - if config.OrganizationCopilot != "" { - result["organization-copilot"] = config.OrganizationCopilot - } - if config.OrganizationCodespaces != "" { - result["organization-codespaces"] = config.OrganizationCodespaces - } + return result +} - // GitHub App-only permission scopes - user-level - if config.EmailAddresses != "" { - result["email-addresses"] = config.EmailAddresses - } - if config.CodespacesLifecycleAdmin != "" { - result["codespaces-lifecycle-admin"] = config.CodespacesLifecycleAdmin - } - if config.CodespacesMetadata != "" { - result["codespaces-metadata"] = config.CodespacesMetadata +func addPermissionScopeValues(result map[string]any, values []struct { + key string + value string +}) { + for _, entry := range values { + if entry.value != "" { + result[entry.key] = entry.value + } } +} - if len(result) == 0 { - return nil +func permissionConfigScopeValues(config *PermissionsConfig) []struct { + key string + value string +} { + return []struct { + key string + value string + }{ + {"actions", config.Actions}, + {"checks", config.Checks}, + {"contents", config.Contents}, + {"deployments", config.Deployments}, + {"id-token", config.IDToken}, + {"issues", config.Issues}, + {"discussions", config.Discussions}, + {"packages", config.Packages}, + {"pages", config.Pages}, + {"pull-requests", config.PullRequests}, + {"repository-projects", config.RepositoryProjects}, + {"security-events", config.SecurityEvents}, + {"statuses", config.Statuses}, + {"vulnerability-alerts", config.VulnerabilityAlerts}, + {"organization-projects", config.OrganizationProjects}, + {"administration", config.Administration}, + {"environments", config.Environments}, + {"git-signing", config.GitSigning}, + {"workflows", config.Workflows}, + {"repository-hooks", config.RepositoryHooks}, + {"single-file", config.SingleFile}, + {"codespaces", config.Codespaces}, + {"repository-custom-properties", config.RepositoryCustomProperties}, + {"members", config.Members}, + {"organization-administration", config.OrganizationAdministration}, + {"team-discussions", config.TeamDiscussions}, + {"organization-hooks", config.OrganizationHooks}, + {"organization-members", config.OrganizationMembers}, + {"organization-packages", config.OrganizationPackages}, + {"organization-self-hosted-runners", config.OrganizationSelfHostedRunners}, + {"organization-custom-org-roles", config.OrganizationCustomOrgRoles}, + {"organization-custom-properties", config.OrganizationCustomProperties}, + {"organization-custom-repository-roles", config.OrganizationCustomRepositoryRoles}, + {"organization-announcement-banners", config.OrganizationAnnouncementBanners}, + {"organization-events", config.OrganizationEvents}, + {"organization-plan", config.OrganizationPlan}, + {"organization-user-blocking", config.OrganizationUserBlocking}, + {"organization-personal-access-token-requests", config.OrganizationPersonalAccessTokenReqs}, + {"organization-personal-access-tokens", config.OrganizationPersonalAccessTokens}, + {"organization-copilot", config.OrganizationCopilot}, + {"organization-codespaces", config.OrganizationCodespaces}, + {"email-addresses", config.EmailAddresses}, + {"codespaces-lifecycle-admin", config.CodespacesLifecycleAdmin}, + {"codespaces-metadata", config.CodespacesMetadata}, } - - return result } diff --git a/pkg/workflow/gemini_engine.go b/pkg/workflow/gemini_engine.go index ff18878b950..25feafe30f5 100644 --- a/pkg/workflow/gemini_engine.go +++ b/pkg/workflow/gemini_engine.go @@ -166,123 +166,113 @@ func (e *GeminiEngine) GetExecutionSteps(workflowData *WorkflowData, logFile str steps = append(steps, settingsStep) // Build gemini CLI arguments based on configuration - var geminiArgs []string - - // Model is passed via the native GEMINI_MODEL environment variable only when explicitly - // configured. When not configured, the Gemini CLI uses its built-in default model. - // This avoids embedding the value directly in the shell command (which fails template injection - // validation for GitHub Actions expressions like ${{ inputs.model }}). - modelConfigured := workflowData.EngineConfig != nil && workflowData.EngineConfig.Model != "" - - // Gemini CLI reads MCP config from .gemini/settings.json (project-level) - // The conversion script (convert_gateway_config_gemini.sh) writes settings.json - // during the MCP setup step, so no --mcp-config flag is needed here. - - // Auto-approve all tool executions (equivalent to Codex's --dangerously-bypass-approvals-and-sandbox) - // Without this, Gemini CLI's default approval mode rejects tool calls with "Tool execution denied by policy" - geminiArgs = append(geminiArgs, "--yolo") - - // Skip the workspace trust check so --yolo is not overridden to "default" approval mode. - // Gemini CLI v1.x checks whether the working directory is trusted and overrides --yolo - // with "default" approval mode (exit code 55) when the folder is untrusted. - // GEMINI_CLI_TRUST_WORKSPACE=true (also set in the step env) handles the same case via - // environment variable, but --skip-trust is more reliable when AWF's sandbox does not - // forward all host environment variables into the container. - geminiArgs = append(geminiArgs, "--skip-trust") - - // Add streaming JSON output (JSONL format, compatible with the log parser) - geminiArgs = append(geminiArgs, "--output-format", "stream-json") - - // Note: the --prompt argument is appended raw after shellJoinArgs below because it contains - // a shell command substitution ("$(cat ...)") that must NOT go through shellEscapeArg — - // single-quoting it would prevent shell expansion at runtime. - - // Build the command + modelConfigured := geminiModelConfigured(workflowData) + geminiCommand := buildGeminiCommand(workflowData) + + // Build the full command with AWF wrapping if enabled + firewallEnabled := isFirewallEnabled(workflowData) + command := buildGeminiExecutionCommand(workflowData, geminiCommand, logFile, firewallEnabled) + + // Build environment variables + env := e.buildGeminiEnv(workflowData, firewallEnabled, modelConfigured) + + // Generate the execution step + stepLines := []string{ + " - name: Execute Gemini CLI", + " id: agentic_execution", + } + + // Filter environment variables for security + allowedSecrets := e.GetRequiredSecretNames(workflowData) + filteredEnv := FilterEnvForSecrets(env, allowedSecrets) + + // Inject GH_TOKEN for CLI proxy (added after filtering since it uses a special + // fallback expression that is always allowed when cli-proxy is enabled) + addCliProxyGHTokenToEnv(filteredEnv, workflowData) + + // Format step with command and env + stepLines = FormatStepWithCommandAndEnv(stepLines, command, filteredEnv) + + steps = append(steps, GitHubActionStep(stepLines)) + return steps +} + +func geminiModelConfigured(workflowData *WorkflowData) bool { + return workflowData.EngineConfig != nil && workflowData.EngineConfig.Model != "" +} + +func buildGeminiCommand(workflowData *WorkflowData) string { + geminiArgs := []string{"--yolo", "--skip-trust", "--output-format", "stream-json"} commandName := "gemini" if workflowData.EngineConfig != nil && workflowData.EngineConfig.Command != "" { commandName = workflowData.EngineConfig.Command } - - // Append the prompt arg raw (not through shellJoinArgs) to preserve shell expansion geminiCommand := fmt.Sprintf(`%s %s --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"`, commandName, shellJoinArgs(geminiArgs)) - geminiCommand = getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + geminiCommand + return getWorkspaceCommandPrefixFor(workflowData.EngineConfig) + geminiCommand +} - // Build the full command with AWF wrapping if enabled - var command string - firewallEnabled := isFirewallEnabled(workflowData) +func buildGeminiExecutionCommand(workflowData *WorkflowData, geminiCommand string, logFile string, firewallEnabled bool) string { if firewallEnabled { - // Get allowed domains: prefer the pre-warmed cache on WorkflowData to avoid - // re-running the expensive map+sort operation. - var allowedDomains string - if workflowData.CachedAllowedDomainsComputed { - allowedDomains = workflowData.CachedAllowedDomainsStr - } else { - allowedDomains = GetAllowedDomainsForEngine(constants.GeminiEngine, - workflowData.NetworkPermissions, - workflowData.Tools, - workflowData.Runtimes, - ) - } - // Add GHES/custom API target domains to the firewall allow-list when engine.api-target is set - if workflowData.EngineConfig != nil && workflowData.EngineConfig.APITarget != "" { - allowedDomains = mergeAPITargetDomains(allowedDomains, workflowData.EngineConfig.APITarget) - } - - npmPathSetup := GetNpmBinPathSetup() - geminiCommandWithPath := fmt.Sprintf("%s && %s", npmPathSetup, geminiCommand) - // Add MCP CLI bin directory to PATH when cli-proxy is enabled - if mcpCLIPath := GetMCPCLIPathSetup(workflowData); mcpCLIPath != "" { - geminiCommandWithPath = fmt.Sprintf("%s && %s", mcpCLIPath, geminiCommandWithPath) - } - - command = BuildAWFCommand(AWFCommandConfig{ - EngineName: "gemini", - EngineCommand: geminiCommandWithPath, - LogFile: logFile, - WorkflowData: workflowData, - UsesTTY: false, - AllowedDomains: allowedDomains, - // Create the agent step summary file before AWF starts so it is accessible - // inside the sandbox. The agent writes its step summary content here, and the - // file is appended to $GITHUB_STEP_SUMMARY after secret redaction. - PathSetup: "touch " + AgentStepSummaryPath, - // Exclude every env var whose step-env value is a secret so the agent - // cannot read raw token values via bash tools (env / printenv). + return BuildAWFCommand(AWFCommandConfig{ + EngineName: "gemini", EngineCommand: geminiCommandWithPath(workflowData, geminiCommand), + LogFile: logFile, WorkflowData: workflowData, UsesTTY: false, + AllowedDomains: geminiAllowedDomains(workflowData), + PathSetup: "touch " + AgentStepSummaryPath, ExcludeEnvVarNames: ComputeAWFExcludeEnvVarNames(workflowData, []string{"GEMINI_API_KEY"}), }) - } else { - command = fmt.Sprintf(`set -o pipefail + } + return fmt.Sprintf(`set -o pipefail printf '%%s' "$(date +%%s%%3N)" > %s touch %s (umask 177 && touch %s) %s 2>&1 | tee -a %s`, AgentCLIStartMsPath, AgentStepSummaryPath, logFile, geminiCommand, logFile) +} + +func geminiCommandWithPath(workflowData *WorkflowData, geminiCommand string) string { + geminiCommandWithPath := fmt.Sprintf("%s && %s", GetNpmBinPathSetup(), geminiCommand) + if mcpCLIPath := GetMCPCLIPathSetup(workflowData); mcpCLIPath != "" { + geminiCommandWithPath = fmt.Sprintf("%s && %s", mcpCLIPath, geminiCommandWithPath) } + return geminiCommandWithPath +} - // Build environment variables - env := map[string]string{ - "GEMINI_API_KEY": "${{ secrets.GEMINI_API_KEY }}", - "GH_AW_PROMPT": constants.AwPromptsFile, - // Tag the step as a GitHub AW agentic execution for discoverability by agents - "GITHUB_AW": "true", - "GITHUB_WORKSPACE": "${{ github.workspace }}", - "RUNNER_TEMP": "${{ runner.temp }}", - // Override GITHUB_STEP_SUMMARY with a path that exists inside the sandbox. - // The runner's original path is unreachable within the AWF isolated filesystem; - // we create this file before the agent starts and append it to the real - // $GITHUB_STEP_SUMMARY after secret redaction. - "GITHUB_STEP_SUMMARY": AgentStepSummaryPath, - // Enable verbose debug logging from Gemini CLI for better diagnostics. - // Gemini CLI uses the npm 'debug' package, and 'gemini-cli:*' enables all - // internal Gemini CLI debug channels (see: https://gemini-cli-docs.pages.dev/cli/configuration). - // Non-JSON debug lines are gracefully skipped by ParseLogMetrics. - "DEBUG": "gemini-cli:*", - // Trust the workspace to prevent Gemini CLI v1.x from overriding --yolo to default - // approval mode when the workspace is untrusted, which causes exit code 55. - "GEMINI_CLI_TRUST_WORKSPACE": "true", +func geminiAllowedDomains(workflowData *WorkflowData) string { + var allowedDomains string + if workflowData.CachedAllowedDomainsComputed { + allowedDomains = workflowData.CachedAllowedDomainsStr + } else { + allowedDomains = GetAllowedDomainsForEngine(constants.GeminiEngine, workflowData.NetworkPermissions, workflowData.Tools, workflowData.Runtimes) } + if workflowData.EngineConfig != nil && workflowData.EngineConfig.APITarget != "" { + allowedDomains = mergeAPITargetDomains(allowedDomains, workflowData.EngineConfig.APITarget) + } + return allowedDomains +} + +func (e *GeminiEngine) buildGeminiEnv(workflowData *WorkflowData, firewallEnabled bool, modelConfigured bool) map[string]string { + env := baseGeminiEnv(workflowData) injectWorkflowCallNetworkAllowedEnv(env, workflowData) - // Indicate the phase: "agent" for the main run, "detection" for threat detection - // Include the compiler version so agents can identify which gh-aw version generated the workflow + applyGeminiOptionalEnv(env, workflowData, firewallEnabled, modelConfigured) + applySafeOutputEnvToMap(env, workflowData) + applyTraceContextEnvToMap(env) + applyEngineCwdEnv(env, workflowData) + if workflowData.EngineConfig != nil && len(workflowData.EngineConfig.Env) > 0 { + maps.Copy(env, workflowData.EngineConfig.Env) + } + agentConfig := getAgentConfig(workflowData) + if agentConfig != nil && len(agentConfig.Env) > 0 { + maps.Copy(env, agentConfig.Env) + geminiLog.Printf("Added %d custom env vars from agent config", len(agentConfig.Env)) + } + return env +} + +func baseGeminiEnv(workflowData *WorkflowData) map[string]string { + env := map[string]string{ + "GEMINI_API_KEY": "${{ secrets.GEMINI_API_KEY }}", "GH_AW_PROMPT": constants.AwPromptsFile, + "GITHUB_AW": "true", "GITHUB_WORKSPACE": "${{ github.workspace }}", "RUNNER_TEMP": "${{ runner.temp }}", + "GITHUB_STEP_SUMMARY": AgentStepSummaryPath, "DEBUG": "gemini-cli:*", "GEMINI_CLI_TRUST_WORKSPACE": "true", + } if workflowData.IsDetectionRun { env["GH_AW_PHASE"] = "detection" } else { @@ -293,77 +283,24 @@ touch %s } else { env["GH_AW_VERSION"] = "dev" } + return env +} - // Add MCP config env var if needed (points to .gemini/settings.json for Gemini) +func applyGeminiOptionalEnv(env map[string]string, workflowData *WorkflowData, firewallEnabled bool, modelConfigured bool) { if HasMCPServers(workflowData) { env["GH_AW_MCP_CONFIG"] = "${{ github.workspace }}/.gemini/settings.json" } - - // When the firewall (AWF) is enabled with --enable-api-proxy, point Gemini CLI at the - // LLM gateway sidecar instead of the real googleapis.com endpoint. if firewallEnabled { env["GEMINI_API_BASE_URL"] = fmt.Sprintf("http://host.docker.internal:%d", constants.GeminiLLMGatewayPort) - - // Set git identity environment variables so the first git commit succeeds inside the - // container. AWF's --env-all forwards these to the container, ensuring git does not - // rely on the host-side ~/.gitconfig which is not visible in the sandbox. maps.Copy(env, getGitIdentityEnvVars()) } - - // Add safe outputs env - applySafeOutputEnvToMap(env, workflowData) - - // Propagate W3C trace context so engine spans nest under the gh-aw.agent.setup span. - applyTraceContextEnvToMap(env) - if workflowData.EngineConfig != nil && workflowData.EngineConfig.MaxTurns != "" { env["GH_AW_MAX_TURNS"] = workflowData.EngineConfig.MaxTurns } else { env["GH_AW_MAX_TURNS"] = compilerenv.BuildDefaultMaxTurnsExpression() } - - // Set the model environment variable only when explicitly configured. - // When model is configured, use the native GEMINI_MODEL env var - the Gemini CLI reads it - // directly, avoiding the need to embed the value in the shell command (which would fail - // template injection validation for GitHub Actions expressions like ${{ inputs.model }}). - // When model is not configured, let the Gemini CLI use its built-in default model. if modelConfigured { geminiLog.Printf("Setting %s env var for model: %s", constants.GeminiCLIModelEnvVar, workflowData.EngineConfig.Model) env[constants.GeminiCLIModelEnvVar] = workflowData.EngineConfig.Model } - - // Add custom environment variables from engine config. - // This allows users to override the default engine token expression (e.g. - // GEMINI_API_KEY: ${{ secrets.MY_ORG_GEMINI_KEY }}) via engine.env. - applyEngineCwdEnv(env, workflowData) - if workflowData.EngineConfig != nil && len(workflowData.EngineConfig.Env) > 0 { - maps.Copy(env, workflowData.EngineConfig.Env) - } - - // Add custom environment variables from agent config - agentConfig := getAgentConfig(workflowData) - if agentConfig != nil && len(agentConfig.Env) > 0 { - maps.Copy(env, agentConfig.Env) - geminiLog.Printf("Added %d custom env vars from agent config", len(agentConfig.Env)) - } - - // Generate the execution step - stepLines := []string{ - " - name: Execute Gemini CLI", - " id: agentic_execution", - } - - // Filter environment variables for security - allowedSecrets := e.GetRequiredSecretNames(workflowData) - filteredEnv := FilterEnvForSecrets(env, allowedSecrets) - - // Inject GH_TOKEN for CLI proxy (added after filtering since it uses a special - // fallback expression that is always allowed when cli-proxy is enabled) - addCliProxyGHTokenToEnv(filteredEnv, workflowData) - - // Format step with command and env - stepLines = FormatStepWithCommandAndEnv(stepLines, command, filteredEnv) - - steps = append(steps, GitHubActionStep(stepLines)) - return steps } diff --git a/pkg/workflow/gemini_tools.go b/pkg/workflow/gemini_tools.go index fb2f6ea2deb..64965352a89 100644 --- a/pkg/workflow/gemini_tools.go +++ b/pkg/workflow/gemini_tools.go @@ -30,85 +30,69 @@ var geminiToolsLog = logger.New("workflow:gemini_tools") // computeGeminiToolsCore maps neutral tool names to Gemini CLI built-in tool names // for use in the tools.core allowlist in .gemini/settings.json. -// -// Neutral tool → Gemini CLI tool mapping: -// - bash: [cmd, ...] → run_shell_command(cmd), ... (one entry per command) -// - bash: * or bash: nil → run_shell_command (allow all shell commands) -// - edit: {} → replace, write_file (file write tools) -// -// Read-only file system tools are always included as they are essential for -// agentic workflows: glob, grep_search, list_directory, read_file, read_many_files. -// -// See: https://github.com/google-gemini/gemini-cli/blob/main/docs/tools/file-system.md -// See: https://github.com/google-gemini/gemini-cli/blob/main/docs/tools/shell.md func computeGeminiToolsCore(tools map[string]any) []string { - // Always include essential read-only file system tools - toolsCore := []string{ - "glob", - "grep_search", - "list_directory", - "read_file", - "read_many_files", - } - + toolsCore := defaultGeminiToolsCore() if tools == nil { return toolsCore } - - // Map bash neutral tool to run_shell_command - if bashConfig, hasBash := tools["bash"]; hasBash { - bashCommands, ok := bashConfig.([]any) - if !ok || len(bashCommands) == 0 { - // bash with no specific commands - allow all shell commands - geminiToolsLog.Print("bash (no specific commands) → run_shell_command") - toolsCore = append(toolsCore, "run_shell_command") - } else { - // Check for wildcard (* or :*) - hasWildcard := false - for _, cmd := range bashCommands { - if cmdStr, ok := cmd.(string); ok && (cmdStr == "*" || cmdStr == ":*") { - hasWildcard = true - break - } - } - if hasWildcard { - geminiToolsLog.Print("bash wildcard → run_shell_command") - toolsCore = append(toolsCore, "run_shell_command") - } else { - // Add an entry for each specific command: run_shell_command(cmd) - for _, cmd := range bashCommands { - if cmdStr, ok := cmd.(string); ok { - // Normalize trailing " *" wildcard (e.g. "jq *" → "jq") so that - // all engines emit the canonical prefix form (run_shell_command(jq)) - // regardless of whether the command was written with or without the wildcard. - normalized, _ := normalizeBashCommand(cmdStr) - entry := fmt.Sprintf("run_shell_command(%s)", normalized) - geminiToolsLog.Printf("bash %q → %s", cmdStr, entry) - toolsCore = append(toolsCore, entry) - } - } - } - } - } - - // Map edit neutral tool to write_file and replace (Gemini's file write tools) + toolsCore = append(toolsCore, geminiBashToolsCore(tools)...) if _, hasEdit := tools["edit"]; hasEdit { geminiToolsLog.Print("edit → replace, write_file") - toolsCore = append(toolsCore, "replace") - toolsCore = append(toolsCore, "write_file") + toolsCore = append(toolsCore, "replace", "write_file") } - - // Map web-fetch neutral tool to web_fetch (Gemini's native HTTP fetch tool) - // See: https://geminicli.com/docs/tools/web-fetch/ if _, hasWebFetch := tools["web-fetch"]; hasWebFetch { geminiToolsLog.Print("web-fetch → web_fetch") toolsCore = append(toolsCore, "web_fetch") } - sort.Strings(toolsCore) return toolsCore } +func defaultGeminiToolsCore() []string { + return []string{"glob", "grep_search", "list_directory", "read_file", "read_many_files"} +} + +func geminiBashToolsCore(tools map[string]any) []string { + bashConfig, hasBash := tools["bash"] + if !hasBash { + return nil + } + bashCommands, ok := bashConfig.([]any) + if !ok || len(bashCommands) == 0 { + geminiToolsLog.Print("bash (no specific commands) → run_shell_command") + return []string{"run_shell_command"} + } + if geminiBashHasWildcard(bashCommands) { + geminiToolsLog.Print("bash wildcard → run_shell_command") + return []string{"run_shell_command"} + } + return geminiSpecificBashTools(bashCommands) +} + +func geminiBashHasWildcard(bashCommands []any) bool { + for _, cmd := range bashCommands { + if cmdStr, ok := cmd.(string); ok && (cmdStr == "*" || cmdStr == ":*") { + return true + } + } + return false +} + +func geminiSpecificBashTools(bashCommands []any) []string { + var toolsCore []string + for _, cmd := range bashCommands { + cmdStr, ok := cmd.(string) + if !ok { + continue + } + normalized, _ := normalizeBashCommand(cmdStr) + entry := fmt.Sprintf("run_shell_command(%s)", normalized) + geminiToolsLog.Printf("bash %q → %s", cmdStr, entry) + toolsCore = append(toolsCore, entry) + } + return toolsCore +} + // generateGeminiSettingsStep creates a GitHub Actions step that writes the // Gemini CLI project settings file (.gemini/settings.json) before execution. // @@ -133,11 +117,9 @@ func (e *GeminiEngine) generateGeminiSettingsStep(workflowData *WorkflowData) Gi workflowDataWithEffectiveTools.Tools = tools tools = withMountedCLIShellCommandsInRestrictedBash(&workflowDataWithEffectiveTools) - // Compute tools.core from neutral tool configuration toolsCore := computeGeminiToolsCore(tools) geminiToolsLog.Printf("tools.core entries: %d", len(toolsCore)) - // Build the settings JSON object config := map[string]any{ "context": map[string]any{ "includeDirectories": []string{"/tmp/"}, diff --git a/pkg/workflow/gh_cli_permissions.go b/pkg/workflow/gh_cli_permissions.go index 306fcdbe32f..d4c8d1f5d94 100644 --- a/pkg/workflow/gh_cli_permissions.go +++ b/pkg/workflow/gh_cli_permissions.go @@ -75,31 +75,16 @@ type compiledAPIPathPattern struct { appPermissions []PermissionScope } -var getCompiledGHCLIPermissions = sync.OnceValues(func() (compiledGHCLIPermissions, error) { +var getCompiledGHCLIPermissions = sync.OnceValues(loadCompiledGHCLIPermissions) + +func loadCompiledGHCLIPermissions() (compiledGHCLIPermissions, error) { var data ghCLIPermissionsData if err := json.Unmarshal(ghCLIPermissionsJSON, &data); err != nil { return compiledGHCLIPermissions{}, fmt.Errorf("failed to load gh CLI permissions from JSON: %w", err) } - cp := compiledGHCLIPermissions{ - readCommands: make(map[string][]PermissionScope), - writeCommands: make(map[string][]PermissionScope), - groupReadPermissions: make(map[string][]PermissionScope), - appReadCommands: make(map[string][]PermissionScope), - appWriteCommands: make(map[string][]PermissionScope), - groupAppReadPermissions: make(map[string][]PermissionScope), - } - - // Build the subcommand regex dynamically from the JSON group keys so that adding a new - // group to gh_cli_permissions.json automatically extends the pattern without a code change. - groups := make([]string, 0, len(data.SubcommandGroups)) - for group := range data.SubcommandGroups { - groups = append(groups, regexp.QuoteMeta(group)) - } - sort.Strings(groups) // deterministic alternation order - subcommandPattern := `(?m)(?:^|[\s|;])gh\s+(` + strings.Join(groups, "|") + `)\s+([\w][\w-]*)\b` - // Defensive check: the pattern is built from embedded JSON keys quoted with - // regexp.QuoteMeta, so a compile error would indicate unexpected data corruption. + cp := newCompiledGHCLIPermissions() + subcommandPattern := buildGHCLISubcommandPattern(data.SubcommandGroups) subcommandRE, err := regexp.Compile(subcommandPattern) if err != nil { return compiledGHCLIPermissions{}, fmt.Errorf("invalid gh subcommand pattern %q: %w", subcommandPattern, err) @@ -107,62 +92,79 @@ var getCompiledGHCLIPermissions = sync.OnceValues(func() (compiledGHCLIPermissio cp.subcommandRE = subcommandRE for group, sg := range data.SubcommandGroups { - readPerms := make([]PermissionScope, len(sg.ReadPermissions)) - for i, p := range sg.ReadPermissions { - readPerms[i] = PermissionScope(p) - } - writePerms := make([]PermissionScope, len(sg.WritePermissions)) - for i, p := range sg.WritePermissions { - writePerms[i] = PermissionScope(p) - } - appReadPerms := make([]PermissionScope, len(sg.AppReadPermissions)) - for i, p := range sg.AppReadPermissions { - appReadPerms[i] = PermissionScope(p) - } - appWritePerms := make([]PermissionScope, len(sg.AppWritePermissions)) - for i, p := range sg.AppWritePermissions { - appWritePerms[i] = PermissionScope(p) - } - - // Store group-level fallback (used when specific action is unknown). - cp.groupReadPermissions[group] = readPerms - cp.groupAppReadPermissions[group] = appReadPerms - - for _, action := range sg.ReadSubcommands { - key := group + " " + action - cp.readCommands[key] = readPerms - cp.appReadCommands[key] = appReadPerms - } - for _, action := range sg.WriteSubcommands { - key := group + " " + action - cp.writeCommands[key] = writePerms - cp.appWriteCommands[key] = appWritePerms - } + addGHCLISubcommandGroup(&cp, group, sg) } for _, ap := range data.APIPathPatterns { - re, err := regexp.Compile(ap.Pattern) - if err != nil { - return compiledGHCLIPermissions{}, fmt.Errorf("invalid gh API path pattern %q in gh_cli_permissions.json: %w", ap.Pattern, err) - } - perms := make([]PermissionScope, len(ap.Permissions)) - for i, p := range ap.Permissions { - perms[i] = PermissionScope(p) + if err := addGHCLIAPIPathPattern(&cp, ap); err != nil { + return compiledGHCLIPermissions{}, err } - appPerms := make([]PermissionScope, len(ap.AppPermissions)) - for i, p := range ap.AppPermissions { - appPerms[i] = PermissionScope(p) - } - cp.apiPathPatterns = append(cp.apiPathPatterns, compiledAPIPathPattern{ - re: re, - permissions: perms, - appPermissions: appPerms, - }) } ghCLIPermissionsLog.Printf("Loaded gh CLI permissions: version=%s, subcommand_groups=%d, api_path_patterns=%d", data.Version, len(data.SubcommandGroups), len(data.APIPathPatterns)) return cp, nil -}) +} + +func newCompiledGHCLIPermissions() compiledGHCLIPermissions { + return compiledGHCLIPermissions{ + readCommands: make(map[string][]PermissionScope), + writeCommands: make(map[string][]PermissionScope), + groupReadPermissions: make(map[string][]PermissionScope), + appReadCommands: make(map[string][]PermissionScope), + appWriteCommands: make(map[string][]PermissionScope), + groupAppReadPermissions: make(map[string][]PermissionScope), + } +} + +func buildGHCLISubcommandPattern(groupsMap map[string]ghCLISubcommandGroup) string { + groups := make([]string, 0, len(groupsMap)) + for group := range groupsMap { + groups = append(groups, regexp.QuoteMeta(group)) + } + sort.Strings(groups) + return `(?m)(?:^|[\s|;])gh\s+(` + strings.Join(groups, "|") + `)\s+([\w][\w-]*)\b` +} + +func addGHCLISubcommandGroup(cp *compiledGHCLIPermissions, group string, sg ghCLISubcommandGroup) { + readPerms := permissionScopesFromStrings(sg.ReadPermissions) + writePerms := permissionScopesFromStrings(sg.WritePermissions) + appReadPerms := permissionScopesFromStrings(sg.AppReadPermissions) + appWritePerms := permissionScopesFromStrings(sg.AppWritePermissions) + cp.groupReadPermissions[group] = readPerms + cp.groupAppReadPermissions[group] = appReadPerms + + for _, action := range sg.ReadSubcommands { + key := group + " " + action + cp.readCommands[key] = readPerms + cp.appReadCommands[key] = appReadPerms + } + for _, action := range sg.WriteSubcommands { + key := group + " " + action + cp.writeCommands[key] = writePerms + cp.appWriteCommands[key] = appWritePerms + } +} + +func addGHCLIAPIPathPattern(cp *compiledGHCLIPermissions, ap ghCLIAPIPathPattern) error { + re, err := regexp.Compile(ap.Pattern) + if err != nil { + return fmt.Errorf("invalid gh API path pattern %q in gh_cli_permissions.json: %w", ap.Pattern, err) + } + cp.apiPathPatterns = append(cp.apiPathPatterns, compiledAPIPathPattern{ + re: re, + permissions: permissionScopesFromStrings(ap.Permissions), + appPermissions: permissionScopesFromStrings(ap.AppPermissions), + }) + return nil +} + +func permissionScopesFromStrings(values []string) []PermissionScope { + scopes := make([]PermissionScope, len(values)) + for i, p := range values { + scopes[i] = PermissionScope(p) + } + return scopes +} // ghAPICmdRE matches `gh api` at a command boundary, capturing the rest of the line. var ghAPICmdRE = regexp.MustCompile(`(?m)(?:^|[\s|;])gh\s+api\s+(.+)`) @@ -289,59 +291,13 @@ func inferPermissionsFromShellScripts(scripts []string) (map[PermissionScope]Per } addScopes := func(scopes []PermissionScope) { - for _, scope := range scopes { - if _, exists := perms[scope]; !exists { - perms[scope] = PermissionRead - } - } + addReadScopes(perms, scopes) } for _, script := range scripts { - // Match gh patterns. - for _, m := range ghCLIPermissions.subcommandRE.FindAllStringSubmatch(script, -1) { - group := strings.ToLower(m[1]) - action := strings.ToLower(m[2]) - key := group + " " + action - - matched := false - // Check explicit read mapping first. - if readPerms, ok := ghCLIPermissions.readCommands[key]; ok { - addScopes(readPerms) - matched = true - } - if appReadPerms, ok := ghCLIPermissions.appReadCommands[key]; ok { - addScopes(appReadPerms) - matched = true - } - if matched { - continue - } - // Write commands only need read-level permissions in the activation job context. - // (Full write escalation is rejected by detectWriteCommandsInShellScripts instead.) - if readPerms, ok := ghCLIPermissions.writeCommands[key]; ok { - addScopes(readPerms) - matched = true - } - if appWritePerms, ok := ghCLIPermissions.appWriteCommands[key]; ok { - addScopes(appWritePerms) - matched = true - } - if matched { - continue - } - // Fall back to group-level read permissions for unrecognised actions. - addScopes(ghCLIPermissions.groupReadPermissions[group]) - addScopes(ghCLIPermissions.groupAppReadPermissions[group]) - } - - // Match gh api patterns. + inferSubcommandPermissions(script, ghCLIPermissions, addScopes) for _, path := range extractGHAPIEndpoints(script) { - for _, ap := range ghCLIPermissions.apiPathPatterns { - if ap.re.MatchString(path) { - addScopes(ap.permissions) - addScopes(ap.appPermissions) - } - } + inferAPIPathPermissions(path, ghCLIPermissions.apiPathPatterns, addScopes) } } @@ -349,6 +305,60 @@ func inferPermissionsFromShellScripts(scripts []string) (map[PermissionScope]Per return perms, nil } +func addReadScopes(perms map[PermissionScope]PermissionLevel, scopes []PermissionScope) { + for _, scope := range scopes { + if _, exists := perms[scope]; !exists { + perms[scope] = PermissionRead + } + } +} + +func inferSubcommandPermissions(script string, ghCLIPermissions compiledGHCLIPermissions, addScopes func([]PermissionScope)) { + for _, m := range ghCLIPermissions.subcommandRE.FindAllStringSubmatch(script, -1) { + group := strings.ToLower(m[1]) + action := strings.ToLower(m[2]) + key := group + " " + action + if addMappedSubcommandScopes(key, ghCLIPermissions, addScopes) { + continue + } + addScopes(ghCLIPermissions.groupReadPermissions[group]) + addScopes(ghCLIPermissions.groupAppReadPermissions[group]) + } +} + +func addMappedSubcommandScopes(key string, ghCLIPermissions compiledGHCLIPermissions, addScopes func([]PermissionScope)) bool { + matched := false + if readPerms, ok := ghCLIPermissions.readCommands[key]; ok { + addScopes(readPerms) + matched = true + } + if appReadPerms, ok := ghCLIPermissions.appReadCommands[key]; ok { + addScopes(appReadPerms) + matched = true + } + if matched { + return true + } + if readPerms, ok := ghCLIPermissions.writeCommands[key]; ok { + addScopes(readPerms) + matched = true + } + if appWritePerms, ok := ghCLIPermissions.appWriteCommands[key]; ok { + addScopes(appWritePerms) + matched = true + } + return matched +} + +func inferAPIPathPermissions(path string, patterns []compiledAPIPathPattern, addScopes func([]PermissionScope)) { + for _, ap := range patterns { + if ap.re.MatchString(path) { + addScopes(ap.permissions) + addScopes(ap.appPermissions) + } + } +} + // detectWriteCommandsInShellScripts returns all write gh CLI commands found in the // given scripts, formatted as "gh " (e.g. "gh pr create"). // The slice contains no duplicates and is sorted deterministically in discovery order. diff --git a/pkg/workflow/github_toolsets.go b/pkg/workflow/github_toolsets.go index 5345012b847..22897afd280 100644 --- a/pkg/workflow/github_toolsets.go +++ b/pkg/workflow/github_toolsets.go @@ -27,75 +27,64 @@ var GitHubToolsetsExcludedFromAll = []string{"dependabot"} // into their constituent toolsets. It handles comma-separated lists and deduplicates. func ParseGitHubToolsets(toolsetsStr string) []string { toolsetsLog.Printf("Parsing GitHub toolsets: %q", toolsetsStr) - if toolsetsStr == "" { toolsetsLog.Printf("Empty toolsets string, using defaults: %v", DefaultGitHubToolsets) return DefaultGitHubToolsets } + expander := githubToolsetExpander{seen: make(map[string]struct{})} + for _, toolset := range strings.Split(toolsetsStr, ",") { + expander.expand(strings.TrimSpace(toolset)) + } + toolsetsLog.Printf("Parsed toolsets result: %d unique toolsets expanded from input", len(expander.expanded)) + return expander.expanded +} - toolsets := strings.Split(toolsetsStr, ",") - var expanded []string - seenToolsets := make(map[string]struct { - }) +type githubToolsetExpander struct { + expanded []string + seen map[string]struct{} +} - for _, toolset := range toolsets { - toolset = strings.TrimSpace(toolset) - if toolset == "" { - continue - } +func (e *githubToolsetExpander) expand(toolset string) { + if toolset == "" { + return + } + switch toolset { + case "default": + toolsetsLog.Printf("Expanding 'default' to %d toolsets", len(DefaultGitHubToolsets)) + e.addMany(DefaultGitHubToolsets) + case "action-friendly": + toolsetsLog.Printf("Expanding 'action-friendly' to %d toolsets", len(ActionFriendlyGitHubToolsets)) + e.addMany(ActionFriendlyGitHubToolsets) + case "all": + e.expandAll() + default: + e.add(toolset) + } +} - switch toolset { - case "default": - // Add default toolsets - toolsetsLog.Printf("Expanding 'default' to %d toolsets", len(DefaultGitHubToolsets)) - for _, dt := range DefaultGitHubToolsets { - if !setutil.Contains(seenToolsets, dt) { - expanded = append(expanded, dt) - seenToolsets[dt] = struct { - }{} - } - } - case "action-friendly": - // Add action-friendly toolsets (excludes "users" which GitHub Actions tokens don't support) - toolsetsLog.Printf("Expanding 'action-friendly' to %d toolsets", len(ActionFriendlyGitHubToolsets)) - for _, dt := range ActionFriendlyGitHubToolsets { - if !setutil.Contains(seenToolsets, dt) { - expanded = append(expanded, dt) - seenToolsets[dt] = struct { - }{} - } - } - case "all": - // Add all toolsets from the toolset permissions map, excluding those that - // require GitHub App-only permissions (see GitHubToolsetsExcludedFromAll). - toolsetsLog.Printf("Expanding 'all' to toolsets from permissions map (excluding %v)", GitHubToolsetsExcludedFromAll) - excludedMap := make(map[string]struct { - }, len(GitHubToolsetsExcludedFromAll)) - for _, ex := range GitHubToolsetsExcludedFromAll { - excludedMap[ex] = struct { - }{} - } - toolsetPermissionsMap := getToolsetPermissionsMap() - for t := range toolsetPermissionsMap { - if setutil.Contains(excludedMap, t) { - continue - } - if !setutil.Contains(seenToolsets, t) { - expanded = append(expanded, t) - seenToolsets[t] = struct { - }{} - } - } - default: - // Add individual toolset - if !setutil.Contains(seenToolsets, toolset) { - expanded = append(expanded, toolset) - seenToolsets[toolset] = struct { - }{} - } +func (e *githubToolsetExpander) expandAll() { + toolsetsLog.Printf("Expanding 'all' to toolsets from permissions map (excluding %v)", GitHubToolsetsExcludedFromAll) + excludedMap := make(map[string]struct{}, len(GitHubToolsetsExcludedFromAll)) + for _, ex := range GitHubToolsetsExcludedFromAll { + excludedMap[ex] = struct{}{} + } + for t := range getToolsetPermissionsMap() { + if !setutil.Contains(excludedMap, t) { + e.add(t) } } +} + +func (e *githubToolsetExpander) addMany(toolsets []string) { + for _, toolset := range toolsets { + e.add(toolset) + } +} - toolsetsLog.Printf("Parsed toolsets result: %d unique toolsets expanded from input", len(expanded)) - return expanded +func (e *githubToolsetExpander) add(toolset string) { + if setutil.Contains(e.seen, toolset) { + return + } + e.expanded = append(e.expanded, toolset) + e.seen[toolset] = struct{}{} } diff --git a/pkg/workflow/glob_validation.go b/pkg/workflow/glob_validation.go index ec7c514430a..42f86705f7b 100644 --- a/pkg/workflow/glob_validation.go +++ b/pkg/workflow/glob_validation.go @@ -90,74 +90,20 @@ func (v *globValidator) validateNext() bool { switch c { case '\\': - switch v.scan.Peek() { - case '[', '?', '*': - c = v.scan.Next() - if v.isRef { - v.invalidRefChar(c, "ref name cannot contain spaces, ~, ^, :, [, ?, *") - } - case '+', '\\', '!': - c = v.scan.Next() - default: - if v.isRef { - v.invalidRefChar('\\', "only special characters [, ?, +, *, \\, ! can be escaped with \\") - c = v.scan.Next() - } - } + c = v.validateEscapedGlobChar() case '?': - if !v.prec { - v.unexpected('?', "special character ? (zero or one)", "the preceding character must not be special character") - } + v.validateQuantifier('?', "special character ? (zero or one)") prec = false case '+': - if !v.prec { - v.unexpected('+', "special character + (one or more)", "the preceding character must not be special character") - } + v.validateQuantifier('+', "special character + (one or more)") prec = false case '*': prec = false case '[': - if v.scan.Peek() == ']' { - c = v.scan.Next() - v.unexpected(']', "content of character match []", "character match must not be empty") - break - } - chars := 0 - Loop: - for { - c = v.scan.Next() - switch c { - case ']': - break Loop - case scanner.EOF: - v.unexpected(c, "end of character match []", "missing ]") - return false - default: - if v.scan.Peek() != '-' { - chars++ - continue Loop - } - chars += 2 - s := c - _ = v.scan.Next() // eat '-'; return value not needed - switch v.scan.Peek() { - case ']': - c = v.scan.Next() - v.unexpected(c, "character range in []", "end of range is missing") - break Loop - case scanner.EOF: - // do nothing - default: - c = v.scan.Next() - if s > c { - why := fmt.Sprintf("start of range %q (%d) is larger than end of range %q (%d)", s, s, c, c) - v.unexpected(c, "character range in []", why) - } - } - } - } - if chars == 1 { - v.unexpected(c, "character match []", "character match with single character is useless. simply use x instead of [x]") + var ok bool + c, ok = v.validateCharacterClass() + if !ok { + return false } case '\r': if v.scan.Peek() == '\n' { @@ -174,6 +120,94 @@ func (v *globValidator) validateNext() bool { } v.prec = prec + return v.validatePatternTail(c) +} + +func (v *globValidator) validateEscapedGlobChar() rune { + switch v.scan.Peek() { + case '[', '?', '*': + c := v.scan.Next() + if v.isRef { + v.invalidRefChar(c, "ref name cannot contain spaces, ~, ^, :, [, ?, *") + } + return c + case '+', '\\', '!': + return v.scan.Next() + default: + if v.isRef { + v.invalidRefChar('\\', "only special characters [, ?, +, *, \\, ! can be escaped with \\") + return v.scan.Next() + } + return '\\' + } +} + +func (v *globValidator) validateQuantifier(c rune, what string) { + if !v.prec { + v.unexpected(c, what, "the preceding character must not be special character") + } +} + +func (v *globValidator) validateCharacterClass() (rune, bool) { + if v.scan.Peek() == ']' { + c := v.scan.Next() + v.unexpected(']', "content of character match []", "character match must not be empty") + return c, true + } + c, chars, ok := v.scanCharacterClassBody() + if !ok { + return c, false + } + if chars == 1 { + v.unexpected(c, "character match []", "character match with single character is useless. simply use x instead of [x]") + } + return c, true +} + +func (v *globValidator) scanCharacterClassBody() (rune, int, bool) { + chars := 0 + for { + c := v.scan.Next() + switch c { + case ']': + return c, chars, true + case scanner.EOF: + v.unexpected(c, "end of character match []", "missing ]") + return c, chars, false + default: + added, endChar, closed := v.validateCharacterClassRange(c) + chars += added + if closed { + return endChar, chars, true + } + } + } +} + +func (v *globValidator) validateCharacterClassRange(c rune) (int, rune, bool) { + if v.scan.Peek() != '-' { + return 1, c, false + } + s := c + _ = v.scan.Next() // eat '-'; return value not needed + switch v.scan.Peek() { + case ']': + c = v.scan.Next() + v.unexpected(c, "character range in []", "end of range is missing") + return 2, c, true + case scanner.EOF: + return 2, c, false + default: + c = v.scan.Next() + if s > c { + why := fmt.Sprintf("start of range %q (%d) is larger than end of range %q (%d)", s, s, c, c) + v.unexpected(c, "character range in []", why) + } + return 2, c, false + } +} + +func (v *globValidator) validatePatternTail(c rune) bool { if v.scan.Peek() == scanner.EOF { if v.isRef && (c == '/' || c == '.') { v.invalidRefChar(c, "ref name must not end with / and .") diff --git a/pkg/workflow/header.go b/pkg/workflow/header.go index 4d4b6ed95bc..9a1403c0f77 100644 --- a/pkg/workflow/header.go +++ b/pkg/workflow/header.go @@ -30,37 +30,54 @@ func GenerateWorkflowHeader(sourceFile string, generatedBy string, customInstruc // Add auto-generated disclaimer as the very first line. // This ensures agents see the notice immediately and can load the debugging skill. // Include version for released builds only (not "dev") + writeWorkflowHeaderDisclaimer(&header, generatedBy) + header.WriteString("#\n") + + // Add ASCII logo + writeWorkflowHeaderLogo(&header) + header.WriteString("#\n") + header.WriteString("#\n") + + // Add regeneration instructions + writeWorkflowHeaderRegeneration(&header, sourceFile) + + // Add custom instructions if provided + writeWorkflowHeaderCustomInstructions(&header, customInstructions) + + header.WriteString("#\n") + + headerLog.Printf("Generated header with %d bytes", header.Len()) + return header.String() +} + +func writeWorkflowHeaderDisclaimer(header *strings.Builder, generatedBy string) { const skillPointer = " To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md" - if generatedBy != "" { - if IsReleasedVersion(GetVersion()) { - fmt.Fprintf(&header, "# This file was automatically generated by %s (%s). DO NOT EDIT.%s\n", generatedBy, GetVersion(), skillPointer) - } else { - fmt.Fprintf(&header, "# This file was automatically generated by %s. DO NOT EDIT.%s\n", generatedBy, skillPointer) - } + if generatedBy == "" { + fmt.Fprintf(header, "# This file was automatically generated. DO NOT EDIT.%s\n", skillPointer) + return + } + if IsReleasedVersion(GetVersion()) { + fmt.Fprintf(header, "# This file was automatically generated by %s (%s). DO NOT EDIT.%s\n", generatedBy, GetVersion(), skillPointer) } else { - fmt.Fprintf(&header, "# This file was automatically generated. DO NOT EDIT.%s\n", skillPointer) + fmt.Fprintf(header, "# This file was automatically generated by %s. DO NOT EDIT.%s\n", generatedBy, skillPointer) } - header.WriteString("#\n") +} - // Add ASCII logo - // TrimRight removes only trailing newlines, preserving per-line leading spaces +func writeWorkflowHeaderLogo(header *strings.Builder) { logoLines := strings.Split(strings.TrimRight(headerAsciiLogo, "\n"), "\n") headerLog.Printf("Adding ASCII logo with %d lines", len(logoLines)) for _, line := range logoLines { - // Strip trailing whitespace from each logo line so the generated comment - // lines do not carry trailing spaces (yamllint trailing-spaces). if trimmed := strings.TrimRight(line, " \t"); trimmed == "" { header.WriteString("#\n") } else { - fmt.Fprintf(&header, "# %s\n", trimmed) + fmt.Fprintf(header, "# %s\n", trimmed) } } - header.WriteString("#\n") - header.WriteString("#\n") +} - // Add regeneration instructions +func writeWorkflowHeaderRegeneration(header *strings.Builder, sourceFile string) { if sourceFile != "" { - fmt.Fprintf(&header, "# To update this file, edit %s and run:\n", sourceFile) + fmt.Fprintf(header, "# To update this file, edit %s and run:\n", sourceFile) } else { header.WriteString("# To regenerate this workflow, run:\n") } @@ -68,24 +85,20 @@ func GenerateWorkflowHeader(sourceFile string, generatedBy string, customInstruc header.WriteString("# Not all edits will cause changes to this file.\n") header.WriteString("#\n") header.WriteString("# For more information: https://github.github.com/gh-aw/introduction/overview/\n") +} - // Add custom instructions if provided - if customInstructions != "" { - header.WriteString("#\n") - // Split custom instructions into lines and prefix each with "# " - instructionLines := strings.Split(strings.TrimSpace(customInstructions), "\n") - headerLog.Printf("Adding %d lines of custom instructions", len(instructionLines)) - for _, line := range instructionLines { - if trimmedLine := strings.TrimSpace(line); trimmedLine == "" { - header.WriteString("#\n") - } else { - fmt.Fprintf(&header, "# %s\n", trimmedLine) - } - } +func writeWorkflowHeaderCustomInstructions(header *strings.Builder, customInstructions string) { + if customInstructions == "" { + return } - header.WriteString("#\n") - - headerLog.Printf("Generated header with %d bytes", header.Len()) - return header.String() + instructionLines := strings.Split(strings.TrimSpace(customInstructions), "\n") + headerLog.Printf("Adding %d lines of custom instructions", len(instructionLines)) + for _, line := range instructionLines { + if trimmedLine := strings.TrimSpace(line); trimmedLine == "" { + header.WriteString("#\n") + } else { + fmt.Fprintf(header, "# %s\n", trimmedLine) + } + } } diff --git a/pkg/workflow/imports.go b/pkg/workflow/imports.go index 750bec1e96a..990dc534624 100644 --- a/pkg/workflow/imports.go +++ b/pkg/workflow/imports.go @@ -199,97 +199,122 @@ func (c *Compiler) MergeSafeOutputs(topSafeOutputs *SafeOutputsConfig, importedS return nil, fmt.Errorf("failed to get safe output type keys: %w", err) } - // Collect all safe output types defined in the top-level config. - // When topRawSafeOutputs is provided (from raw frontmatter), use only keys that are - // explicitly present in the raw map to avoid counting auto-defaults as user-defined types. - // When nil, fall back to inspecting the processed config struct (legacy/test behaviour). - topDefinedTypes := make(map[string]struct { - }) - if topSafeOutputs != nil { - for _, key := range typeKeys { - if topRawSafeOutputs != nil { - if _, exists := topRawSafeOutputs[key]; exists { - topDefinedTypes[key] = struct { - }{} - } - } else if hasSafeOutputType(topSafeOutputs, key) { - topDefinedTypes[key] = struct { - }{} + topDefinedTypes := collectTopDefinedSafeOutputTypes(topSafeOutputs, topRawSafeOutputs, typeKeys) + importsLog.Printf("Top-level safe-outputs defines %d types", len(topDefinedTypes)) + + importedConfigs, importedDefinedTypes, accumulatedExclude, err := collectImportedSafeOutputConfigs( + importedSafeOutputsJSON, + typeKeys, + topDefinedTypes, + ) + if err != nil { + return nil, err + } + + importsLog.Printf("Found %d imported safe-outputs configs with %d types", len(importedConfigs), len(importedDefinedTypes)) + + // If no imported configs found (neither safe output types nor meta fields), return the original + if len(importedConfigs) == 0 { + return topSafeOutputs, nil + } + + // Initialize result with top-level config or create new one + result := topSafeOutputs + if result == nil { + result = &SafeOutputsConfig{} + } + + if result, err = mergeImportedSafeOutputConfigs(result, importedConfigs, c); err != nil { + return nil, err + } + + applyAccumulatedProtectedFilesExclude(result, accumulatedExclude) + + importsLog.Printf("Successfully merged safe-outputs from imports") + return result, nil +} + +func collectTopDefinedSafeOutputTypes(topSafeOutputs *SafeOutputsConfig, topRawSafeOutputs map[string]any, typeKeys []string) map[string]struct{} { + topDefinedTypes := make(map[string]struct{}) + if topSafeOutputs == nil { + return topDefinedTypes + } + for _, key := range typeKeys { + if topRawSafeOutputs != nil { + if _, exists := topRawSafeOutputs[key]; exists { + topDefinedTypes[key] = struct{}{} } + } else if hasSafeOutputType(topSafeOutputs, key) { + topDefinedTypes[key] = struct{}{} } } - importsLog.Printf("Top-level safe-outputs defines %d types", len(topDefinedTypes)) - - // Track types defined in imported configs for conflict detection - importedDefinedTypes := make(map[string]struct { - }) + return topDefinedTypes +} - // Collect all imported configs. This includes configs with only meta fields (like allowed-domains, - // staged, env, github-token, max-patch-size, runs-on) as well as those defining safe output types. - // Meta fields can be imported even when no safe output types are defined. - var importedConfigs []map[string]any - // Collect protected-files exclude lists from type-conflicting imports so they can be - // merged as a set even when the importing config already defines the same handler type. - // These are keyed by handler type name (e.g. "create-pull-request"). +func collectImportedSafeOutputConfigs(importedJSON []string, typeKeys []string, topDefinedTypes map[string]struct{}) ([]map[string]any, map[string]struct{}, map[string][]string, error) { + importedDefinedTypes := make(map[string]struct{}) accumulatedExclude := map[string][]string{} - - for _, configJSON := range importedSafeOutputsJSON { + var importedConfigs []map[string]any + for _, configJSON := range importedJSON { if configJSON == "" || configJSON == "{}" { continue } - - var config map[string]any - if err := json.Unmarshal([]byte(configJSON), &config); err != nil { - importsLog.Printf("Skipping malformed safe-outputs config: %v", err) + config, err := decodeImportedSafeOutputConfig(configJSON) + if err != nil { continue } - - // Check for conflicts and remove types already defined in top-level config - // Main workflow definitions take precedence over imports (override behavior). - // Exception: protected-files.exclude is always extracted before deletion so that - // exclude lists from imported configs are merged as a set into the result. - for _, key := range typeKeys { - if _, exists := config[key]; exists { - if setutil.Contains(topDefinedTypes, key) { - // Main workflow overrides imported definition — extract protected-files - // exclude lists before removing the type entry. - if handlerCfg, ok := config[key].(map[string]any); ok { - if pf, ok := handlerCfg["protected-files"].(map[string]any); ok { - if excludeFiles := parseStringSliceAny(pf["exclude"], importsLog); len(excludeFiles) > 0 { - accumulatedExclude[key] = sliceutil.MergeUnique(accumulatedExclude[key], excludeFiles...) - importsLog.Printf("Saved protected-files exclude from overridden import %s: %v", key, excludeFiles) - } - } - } - importsLog.Printf("Main workflow overrides imported safe-output: %s", key) - delete(config, key) - continue - } - if setutil.Contains(importedDefinedTypes, key) { - return nil, fmt.Errorf("safe-outputs conflict: '%s' is defined in multiple imported workflows. Each safe-output type can only be defined once", key) - } - importedDefinedTypes[key] = struct { - }{} - } + if err := reconcileImportedSafeOutputTypes(config, typeKeys, topDefinedTypes, importedDefinedTypes, accumulatedExclude); err != nil { + return nil, nil, nil, err } - importedConfigs = append(importedConfigs, config) } + return importedConfigs, importedDefinedTypes, accumulatedExclude, nil +} - importsLog.Printf("Found %d imported safe-outputs configs with %d types", len(importedConfigs), len(importedDefinedTypes)) +func decodeImportedSafeOutputConfig(configJSON string) (map[string]any, error) { + var config map[string]any + if err := json.Unmarshal([]byte(configJSON), &config); err != nil { + importsLog.Printf("Skipping malformed safe-outputs config: %v", err) + return nil, err + } + return config, nil +} - // If no imported configs found (neither safe output types nor meta fields), return the original - if len(importedConfigs) == 0 { - return topSafeOutputs, nil +func reconcileImportedSafeOutputTypes(config map[string]any, typeKeys []string, topDefinedTypes, importedDefinedTypes map[string]struct{}, accumulatedExclude map[string][]string) error { + for _, key := range typeKeys { + if _, exists := config[key]; !exists { + continue + } + if setutil.Contains(topDefinedTypes, key) { + saveProtectedFilesExcludeFromOverride(config, key, accumulatedExclude) + importsLog.Printf("Main workflow overrides imported safe-output: %s", key) + delete(config, key) + continue + } + if setutil.Contains(importedDefinedTypes, key) { + return fmt.Errorf("safe-outputs conflict: '%s' is defined in multiple imported workflows. Each safe-output type can only be defined once", key) + } + importedDefinedTypes[key] = struct{}{} } + return nil +} - // Initialize result with top-level config or create new one - result := topSafeOutputs - if result == nil { - result = &SafeOutputsConfig{} +func saveProtectedFilesExcludeFromOverride(config map[string]any, key string, accumulatedExclude map[string][]string) { + handlerCfg, ok := config[key].(map[string]any) + if !ok { + return + } + pf, ok := handlerCfg["protected-files"].(map[string]any) + if !ok { + return + } + if excludeFiles := parseStringSliceAny(pf["exclude"], importsLog); len(excludeFiles) > 0 { + accumulatedExclude[key] = sliceutil.MergeUnique(accumulatedExclude[key], excludeFiles...) + importsLog.Printf("Saved protected-files exclude from overridden import %s: %v", key, excludeFiles) } +} - // Merge each imported config +func mergeImportedSafeOutputConfigs(result *SafeOutputsConfig, importedConfigs []map[string]any, c *Compiler) (*SafeOutputsConfig, error) { for _, config := range importedConfigs { var err error result, err = mergeSafeOutputConfig(result, config, c) @@ -297,33 +322,26 @@ func (c *Compiler) MergeSafeOutputs(topSafeOutputs *SafeOutputsConfig, importedS return nil, err } } + return result, nil +} - // Apply protected-files exclude lists accumulated from type-conflicting imports. - // These are merged as a set so that importing a base workflow can add to exclusions - // without completely replacing the main workflow's handler configuration. - if len(accumulatedExclude) > 0 { - if result.CreatePullRequests != nil { - if excludeFiles, ok := accumulatedExclude["create-pull-request"]; ok && len(excludeFiles) > 0 { - result.CreatePullRequests.ProtectedFilesExclude = sliceutil.MergeUnique( - result.CreatePullRequests.ProtectedFilesExclude, - excludeFiles..., - ) - importsLog.Printf("Merged %d accumulated protected-files exclude(s) into create-pull-request", len(excludeFiles)) - } - } - if result.PushToPullRequestBranch != nil { - if excludeFiles, ok := accumulatedExclude["push-to-pull-request-branch"]; ok && len(excludeFiles) > 0 { - result.PushToPullRequestBranch.ProtectedFilesExclude = sliceutil.MergeUnique( - result.PushToPullRequestBranch.ProtectedFilesExclude, - excludeFiles..., - ) - importsLog.Printf("Merged %d accumulated protected-files exclude(s) into push-to-pull-request-branch", len(excludeFiles)) - } - } +func applyAccumulatedProtectedFilesExclude(result *SafeOutputsConfig, accumulatedExclude map[string][]string) { + if len(accumulatedExclude) == 0 { + return } + if result.CreatePullRequests != nil { + mergeAccumulatedExclude("create-pull-request", accumulatedExclude, &result.CreatePullRequests.ProtectedFilesExclude) + } + if result.PushToPullRequestBranch != nil { + mergeAccumulatedExclude("push-to-pull-request-branch", accumulatedExclude, &result.PushToPullRequestBranch.ProtectedFilesExclude) + } +} - importsLog.Printf("Successfully merged safe-outputs from imports") - return result, nil +func mergeAccumulatedExclude(key string, accumulatedExclude map[string][]string, target *[]string) { + if excludeFiles, ok := accumulatedExclude[key]; ok && len(excludeFiles) > 0 { + *target = sliceutil.MergeUnique(*target, excludeFiles...) + importsLog.Printf("Merged %d accumulated protected-files exclude(s) into %s", len(excludeFiles), key) + } } // hasSafeOutputType checks if a SafeOutputsConfig has a specific safe output type defined @@ -357,8 +375,29 @@ func mergeSafeOutputConfig(result *SafeOutputsConfig, config map[string]any, c * // Merge each safe output type (only set if nil in result). // Types with custom merge semantics are handled below. - specialMergeFields := map[string]struct { - }{ + mergeRegularSafeOutputFields(result, importedConfig) + mergePullRequestSafeOutputFields(result, importedConfig) + mergeDefaultedSafeOutputFields(result, importedConfig, config) + mergeSafeOutputMetaFields(result, importedConfig) + mergeSafeOutputMessages(result, importedConfig) + mergeAdditionalSafeOutputMetaFields(result, importedConfig) + + // Merge steps: concatenate imported steps after main workflow's steps + if len(importedConfig.Steps) > 0 { + result.Steps = append(result.Steps, importedConfig.Steps...) + } + + // NOTE: Jobs are NOT merged here. They are handled separately in compiler_orchestrator.go + // via mergeSafeJobsFromIncludedConfigs and extractSafeJobsFromFrontmatter. + // The Jobs field is managed independently from other safe-output types to support + // complex merge scenarios and conflict detection across multiple imports. + + importsLog.Print("Safe-output config merge completed") + return result, nil +} + +func mergeRegularSafeOutputFields(result, importedConfig *SafeOutputsConfig) { + specialMergeFields := map[string]struct{}{ "CreatePullRequests": {}, "PushToPullRequestBranch": {}, "MissingTool": {}, @@ -373,12 +412,12 @@ func mergeSafeOutputConfig(result *SafeOutputsConfig, config map[string]any, c * } mergeSafeOutputFieldIfNil(result, importedConfig, handler.StructField) } +} +func mergePullRequestSafeOutputFields(result, importedConfig *SafeOutputsConfig) { if result.CreatePullRequests == nil && importedConfig.CreatePullRequests != nil { result.CreatePullRequests = importedConfig.CreatePullRequests } else if result.CreatePullRequests != nil && importedConfig.CreatePullRequests != nil { - // Merge protected-files exclude lists as a set so that imports can extend exclusions - // without replacing the top-level configuration entirely. result.CreatePullRequests.ProtectedFilesExclude = sliceutil.MergeUnique( result.CreatePullRequests.ProtectedFilesExclude, importedConfig.CreatePullRequests.ProtectedFilesExclude..., @@ -394,6 +433,9 @@ func mergeSafeOutputConfig(result *SafeOutputsConfig, config map[string]any, c * importedConfig.PushToPullRequestBranch.ProtectedFilesExclude..., ) } +} + +func mergeDefaultedSafeOutputFields(result, importedConfig *SafeOutputsConfig, config map[string]any) { // missing-tool, missing-data, noop, and report-incomplete are auto-defaulted by // extractSafeOutputsConfig whenever any safe-outputs are present, even when the user // has not explicitly configured those types. This means result.X can be non-nil (the @@ -425,8 +467,9 @@ func mergeSafeOutputConfig(result *SafeOutputsConfig, config map[string]any, c * if _, hasTD := config["threat-detection"]; hasTD && importedConfig.ThreatDetection != nil { result.ThreatDetection = importedConfig.ThreatDetection } +} - // Merge meta-configuration fields (only set if empty/zero in result) +func mergeSafeOutputMetaFields(result, importedConfig *SafeOutputsConfig) { if len(result.AllowedDomains) == 0 && len(importedConfig.AllowedDomains) > 0 { result.AllowedDomains = importedConfig.AllowedDomains } @@ -454,8 +497,9 @@ func mergeSafeOutputConfig(result *SafeOutputsConfig, config map[string]any, c * if len(importedConfig.Needs) > 0 { result.Needs = sliceutil.MergeUnique(result.Needs, importedConfig.Needs...) } +} - // Merge Messages configuration at field level (main workflow entries override imported entries) +func mergeSafeOutputMessages(result, importedConfig *SafeOutputsConfig) { if importedConfig.Messages != nil { if result.Messages == nil { // If main has no messages, use imported messages entirely @@ -465,8 +509,9 @@ func mergeSafeOutputConfig(result *SafeOutputsConfig, config map[string]any, c * result.Messages = mergeMessagesConfig(result.Messages, importedConfig.Messages) } } +} - // Merge additional meta-configuration fields +func mergeAdditionalSafeOutputMetaFields(result, importedConfig *SafeOutputsConfig) { if result.Footer == nil && importedConfig.Footer != nil { result.Footer = importedConfig.Footer } @@ -490,19 +535,6 @@ func mergeSafeOutputConfig(result *SafeOutputsConfig, config map[string]any, c * if result.Mentions == nil && importedConfig.Mentions != nil { result.Mentions = importedConfig.Mentions } - - // Merge steps: concatenate imported steps after main workflow's steps - if len(importedConfig.Steps) > 0 { - result.Steps = append(result.Steps, importedConfig.Steps...) - } - - // NOTE: Jobs are NOT merged here. They are handled separately in compiler_orchestrator.go - // via mergeSafeJobsFromIncludedConfigs and extractSafeJobsFromFrontmatter. - // The Jobs field is managed independently from other safe-output types to support - // complex merge scenarios and conflict detection across multiple imports. - - importsLog.Print("Safe-output config merge completed") - return result, nil } // mergeMessagesConfig merges two SafeOutputMessagesConfig structs at the field level. diff --git a/pkg/workflow/jobs.go b/pkg/workflow/jobs.go index efc5a31e669..d4bd9296a3b 100644 --- a/pkg/workflow/jobs.go +++ b/pkg/workflow/jobs.go @@ -208,15 +208,24 @@ func (jm *JobManager) WriteJobsYAML(b *strings.Builder) { // renderJobTo writes a single job to b directly, with no intermediate string allocation. func (jm *JobManager) renderJobTo(b *strings.Builder, job *Job) { jobLog.Printf("Rendering job: %s (steps=%d, needs=%d, reusable=%t)", job.Name, len(job.Steps), len(job.Needs), job.Uses != "") - fmt.Fprintf(b, " %s:\n", job.Name) - - // Add display name if present if job.DisplayName != "" { fmt.Fprintf(b, " name: %s\n", job.DisplayName) } + renderJobNeedsTo(b, job) + renderJobIfTo(b, job) + renderJobExecutionConfigTo(b, job) + renderJobSecurityAndLimitsTo(b, job) + renderJobEnvAndOutputsTo(b, job) + if job.Uses != "" { + renderReusableJobTo(b, job) + } else { + renderJobStepsTo(b, job) + } + b.WriteString("\n") +} - // Add needs clause if there are dependencies +func renderJobNeedsTo(b *strings.Builder, job *Job) { if len(job.Needs) > 0 { if len(job.Needs) == 1 { fmt.Fprintf(b, " needs: %s\n", job.Needs[0]) @@ -231,72 +240,57 @@ func (jm *JobManager) renderJobTo(b *strings.Builder, job *Job) { } } } +} - // Add if condition if present +func renderJobIfTo(b *strings.Builder, job *Job) { if job.If != "" { - // Add zizmor ignore comment if this job has workflow_run safety checks if job.HasWorkflowRunSafetyChecks { b.WriteString(" # zizmor: ignore[dangerous-triggers] - workflow_run trigger is secured with role and fork validation\n") } - - // Check if expression is multiline or longer than MaxExpressionLineLength characters if hasNewlineInStringLiteral(job.If) { - // The condition contains a literal newline inside a GitHub Actions expression string literal - // (e.g. startsWith(body, '/command\n') for matching bot comments with attribution metadata). - // Use a YAML double-quoted scalar so the \n escape is preserved as a real newline after - // YAML parsing, which GitHub Actions then evaluates correctly. fmt.Fprintf(b, " if: \"%s\"\n", escapeForYAMLDoubleQuoted(job.If)) } else if strings.Contains(job.If, "\n") || len(job.If) > int(constants.MaxExpressionLineLength) { - // Use YAML folded style for multiline expressions or long expressions - b.WriteString(" if: >\n") - - if strings.Contains(job.If, "\n") { - // Already has newlines, use existing logic - lines := strings.SplitSeq(job.If, "\n") - for line := range lines { - if strings.TrimSpace(line) != "" { - fmt.Fprintf(b, " %s\n", strings.TrimSpace(line)) - } - } - } else { - // Long single-line expression, break it into logical lines - lines := BreakLongExpression(job.If) - for _, line := range lines { - fmt.Fprintf(b, " %s\n", strings.TrimSpace(line)) - } - } + renderFoldedJobIfTo(b, job.If) } else { - // Single line expression that's not too long fmt.Fprintf(b, " if: %s\n", job.If) } } +} - // Add runs-on +func renderFoldedJobIfTo(b *strings.Builder, condition string) { + b.WriteString(" if: >\n") + if strings.Contains(condition, "\n") { + for line := range strings.SplitSeq(condition, "\n") { + if strings.TrimSpace(line) != "" { + fmt.Fprintf(b, " %s\n", strings.TrimSpace(line)) + } + } + return + } + for _, line := range BreakLongExpression(condition) { + fmt.Fprintf(b, " %s\n", strings.TrimSpace(line)) + } +} + +func renderJobExecutionConfigTo(b *strings.Builder, job *Job) { if job.RunsOn != "" { fmt.Fprintf(b, " %s\n", job.RunsOn) } - - // Add strategy section if job.Strategy != "" { fmt.Fprintf(b, " %s\n", strings.TrimRight(job.Strategy, "\n")) } - - // Add environment section if job.Environment != "" { fmt.Fprintf(b, " %s\n", job.Environment) } - - // Add container section if job.Container != "" { fmt.Fprintf(b, " %s\n", job.Container) } - - // Add services section if job.Services != "" { fmt.Fprintf(b, " %s\n", job.Services) } +} - // Add permissions section +func renderJobSecurityAndLimitsTo(b *strings.Builder, job *Job) { if job.PermissionsComment != "" { for line := range strings.SplitSeq(strings.TrimRight(job.PermissionsComment, "\n"), "\n") { fmt.Fprintf(b, " %s\n", line) @@ -305,101 +299,82 @@ func (jm *JobManager) renderJobTo(b *strings.Builder, job *Job) { if job.Permissions != "" { fmt.Fprintf(b, " %s\n", job.Permissions) } - - // Add concurrency section if job.Concurrency != "" { fmt.Fprintf(b, " %s\n", job.Concurrency) } - - // Add timeout-minutes if specified if job.TimeoutMinutesExpression != "" { - // TimeoutMinutesExpression is validated when parsed from frontmatter in compiler_jobs.go. fmt.Fprintf(b, " timeout-minutes: %s\n", job.TimeoutMinutesExpression) } else if job.TimeoutMinutes > 0 { fmt.Fprintf(b, " timeout-minutes: %d\n", job.TimeoutMinutes) } - - // Add continue-on-error only when explicitly set if job.ContinueOnError != nil { fmt.Fprintf(b, " continue-on-error: %t\n", *job.ContinueOnError) } +} - // Add environment variables section +func renderJobEnvAndOutputsTo(b *strings.Builder, job *Job) { env := buildRenderedJobEnv(job) if len(env) > 0 { b.WriteString(" env:\n") - // Sort environment variable keys for consistent output envKeys := sliceutil.SortedKeys(env) - for _, key := range envKeys { fmt.Fprintf(b, " %s: %s\n", key, env[key]) } } - - // Add outputs section if len(job.Outputs) > 0 { b.WriteString(" outputs:\n") - // Sort output keys for consistent output outputKeys := sliceutil.SortedKeys(job.Outputs) - for _, key := range outputKeys { fmt.Fprintf(b, " %s: %s\n", key, job.Outputs[key]) } } +} - // Check if this is a reusable workflow call - if job.Uses != "" { - jobLog.Printf("Rendering reusable workflow call: %s uses=%s with=%d secrets=%d", job.Name, job.Uses, len(job.With), len(job.Secrets)) - // Add uses directive for reusable workflow - fmt.Fprintf(b, " uses: %s\n", job.Uses) - - // Add with parameters if present - if len(job.With) > 0 { - b.WriteString(" with:\n") - // Sort keys for consistent output - withKeys := sliceutil.SortedKeys(job.With) - - for _, key := range withKeys { - value := job.With[key] - // Format the value based on its type - switch v := value.(type) { - case string: - fmt.Fprintf(b, " %s: %s\n", key, v) - case int, int64, float64: - fmt.Fprintf(b, " %s: %v\n", key, v) - case bool: - fmt.Fprintf(b, " %s: %t\n", key, v) - default: - fmt.Fprintf(b, " %s: %v\n", key, v) - } - } - } - - // Add secrets if present - if job.SecretsInherit { - b.WriteString(" secrets: inherit\n") - } else if len(job.Secrets) > 0 { - b.WriteString(" secrets:\n") - // Sort secret keys for consistent output - secretKeys := sliceutil.SortedKeys(job.Secrets) +func renderReusableJobTo(b *strings.Builder, job *Job) { + jobLog.Printf("Rendering reusable workflow call: %s uses=%s with=%d secrets=%d", job.Name, job.Uses, len(job.With), len(job.Secrets)) + fmt.Fprintf(b, " uses: %s\n", job.Uses) + renderReusableJobWithTo(b, job) + renderReusableJobSecretsTo(b, job) +} - for _, key := range secretKeys { - fmt.Fprintf(b, " %s: %s\n", key, job.Secrets[key]) - } +func renderReusableJobWithTo(b *strings.Builder, job *Job) { + if len(job.With) == 0 { + return + } + b.WriteString(" with:\n") + for _, key := range sliceutil.SortedKeys(job.With) { + switch v := job.With[key].(type) { + case string: + fmt.Fprintf(b, " %s: %s\n", key, v) + case int, int64, float64: + fmt.Fprintf(b, " %s: %v\n", key, v) + case bool: + fmt.Fprintf(b, " %s: %t\n", key, v) + default: + fmt.Fprintf(b, " %s: %v\n", key, v) } - } else { - // Add steps section (only for non-reusable workflow jobs) - if len(job.Steps) > 0 { - b.WriteString(" steps:\n") - for _, step := range job.Steps { - // Each step is already formatted with proper indentation - b.WriteString(step) - } + } +} + +func renderReusableJobSecretsTo(b *strings.Builder, job *Job) { + if job.SecretsInherit { + b.WriteString(" secrets: inherit\n") + } else if len(job.Secrets) > 0 { + b.WriteString(" secrets:\n") + for _, key := range sliceutil.SortedKeys(job.Secrets) { + fmt.Fprintf(b, " %s: %s\n", key, job.Secrets[key]) } } +} - // Add newline after each job for proper formatting - b.WriteString("\n") +func renderJobStepsTo(b *strings.Builder, job *Job) { + if len(job.Steps) == 0 { + return + } + b.WriteString(" steps:\n") + for _, step := range job.Steps { + b.WriteString(step) + } } func buildRenderedJobEnv(job *Job) map[string]string { diff --git a/pkg/workflow/js.go b/pkg/workflow/js.go index 8314d6dc86c..ac4961c2b7d 100644 --- a/pkg/workflow/js.go +++ b/pkg/workflow/js.go @@ -105,31 +105,20 @@ func removeJavaScriptCommentsFromLine(line string, inBlockComment *bool) string for i < len(runes) { if *inBlockComment { // Look for end of block comment - if i < len(runes)-1 && runes[i] == '*' && runes[i+1] == '/' { - *inBlockComment = false - i += 2 // Skip '*/' - } else { - i++ - } + i = consumeJavaScriptBlockComment(runes, i, inBlockComment) continue } // Check for start of comments - if i < len(runes)-1 { - // Block comment start - if runes[i] == '/' && runes[i+1] == '*' { - *inBlockComment = true - i += 2 // Skip '/*' - continue - } - // Line comment start - if runes[i] == '/' && runes[i+1] == '/' { - // Check if we're inside a string literal or regex literal - beforeSlash := string(runes[:i]) - if !isInsideStringLiteral(beforeSlash) && !isInsideRegexLiteral(beforeSlash) { - // Rest of line is a comment, stop processing - break - } + if startedBlock, next := startsJavaScriptBlockComment(runes, i); startedBlock { + *inBlockComment = true + i = next + continue + } + if startsJavaScriptLineComment(runes, i) { + beforeSlash := string(runes[:i]) + if !isInsideStringLiteral(beforeSlash) && !isInsideRegexLiteral(beforeSlash) { + break } } @@ -137,64 +126,14 @@ func removeJavaScriptCommentsFromLine(line string, inBlockComment *bool) string if runes[i] == '/' { beforeSlash := string(runes[:i]) if !isInsideStringLiteral(beforeSlash) && !isInsideRegexLiteral(beforeSlash) && canStartRegexLiteral(beforeSlash) { - // This is likely a regex literal - result.WriteRune(runes[i]) // Write the opening / - i++ - - // Process inside regex literal - for i < len(runes) { - if runes[i] == '/' { - // Check if it's escaped - escapeCount := 0 - j := i - 1 - for j >= 0 && runes[j] == '\\' { - escapeCount++ - j-- - } - if escapeCount%2 == 0 { - // Not escaped, end of regex - result.WriteRune(runes[i]) // Write the closing / - i++ - // Skip regex flags (g, i, m, etc.) - for i < len(runes) && (runes[i] >= 'a' && runes[i] <= 'z' || runes[i] >= 'A' && runes[i] <= 'Z') { - result.WriteRune(runes[i]) - i++ - } - break - } - } - result.WriteRune(runes[i]) - i++ - } + i = writeJavaScriptRegexLiteral(&result, runes, i) continue } } // Check for string literals if runes[i] == '"' || runes[i] == '\'' || runes[i] == '`' { - quote := runes[i] - result.WriteRune(runes[i]) - i++ - - // Process inside string literal - for i < len(runes) { - result.WriteRune(runes[i]) - if runes[i] == quote { - // Check if it's escaped - escapeCount := 0 - j := i - 1 - for j >= 0 && runes[j] == '\\' { - escapeCount++ - j-- - } - if escapeCount%2 == 0 { - // Not escaped, end of string - i++ - break - } - } - i++ - } + i = writeJavaScriptStringLiteral(&result, runes, i) continue } @@ -205,6 +144,72 @@ func removeJavaScriptCommentsFromLine(line string, inBlockComment *bool) string return result.String() } +func consumeJavaScriptBlockComment(runes []rune, i int, inBlockComment *bool) int { + if i < len(runes)-1 && runes[i] == '*' && runes[i+1] == '/' { + *inBlockComment = false + return i + 2 + } + return i + 1 +} + +func startsJavaScriptBlockComment(runes []rune, i int) (bool, int) { + if i < len(runes)-1 && runes[i] == '/' && runes[i+1] == '*' { + return true, i + 2 + } + return false, i +} + +func startsJavaScriptLineComment(runes []rune, i int) bool { + return i < len(runes)-1 && runes[i] == '/' && runes[i+1] == '/' +} + +func writeJavaScriptRegexLiteral(result *strings.Builder, runes []rune, i int) int { + result.WriteRune(runes[i]) + i++ + for i < len(runes) { + if runes[i] == '/' && isUnescapedRune(runes, i) { + result.WriteRune(runes[i]) + i++ + return writeJavaScriptRegexFlags(result, runes, i) + } + result.WriteRune(runes[i]) + i++ + } + return i +} + +func writeJavaScriptRegexFlags(result *strings.Builder, runes []rune, i int) int { + for i < len(runes) && (runes[i] >= 'a' && runes[i] <= 'z' || runes[i] >= 'A' && runes[i] <= 'Z') { + result.WriteRune(runes[i]) + i++ + } + return i +} + +func writeJavaScriptStringLiteral(result *strings.Builder, runes []rune, i int) int { + quote := runes[i] + result.WriteRune(runes[i]) + i++ + for i < len(runes) { + result.WriteRune(runes[i]) + if runes[i] == quote && isUnescapedRune(runes, i) { + return i + 1 + } + i++ + } + return i +} + +func isUnescapedRune(runes []rune, i int) bool { + escapeCount := 0 + j := i - 1 + for j >= 0 && runes[j] == '\\' { + escapeCount++ + j-- + } + return escapeCount%2 == 0 +} + // isInsideStringLiteral checks if we're currently inside a string literal // by counting unescaped quotes before the current position func isInsideStringLiteral(text string) bool { @@ -218,26 +223,14 @@ func isInsideStringLiteral(text string) bool { case '\'': if !inDoubleQuote && !inBacktick { // Check if escaped - escapeCount := 0 - j := i - 1 - for j >= 0 && runes[j] == '\\' { - escapeCount++ - j-- - } - if escapeCount%2 == 0 { + if isUnescapedRune(runes, i) { inSingleQuote = !inSingleQuote } } case '"': if !inSingleQuote && !inBacktick { // Check if escaped - escapeCount := 0 - j := i - 1 - for j >= 0 && runes[j] == '\\' { - escapeCount++ - j-- - } - if escapeCount%2 == 0 { + if isUnescapedRune(runes, i) { inDoubleQuote = !inDoubleQuote } } @@ -265,26 +258,14 @@ func isInsideRegexLiteral(text string) bool { case '\'': if !inDoubleQuote && !inBacktick && !inRegex { // Check if escaped - escapeCount := 0 - j := i - 1 - for j >= 0 && runes[j] == '\\' { - escapeCount++ - j-- - } - if escapeCount%2 == 0 { + if isUnescapedRune(runes, i) { inSingleQuote = !inSingleQuote } } case '"': if !inSingleQuote && !inBacktick && !inRegex { // Check if escaped - escapeCount := 0 - j := i - 1 - for j >= 0 && runes[j] == '\\' { - escapeCount++ - j-- - } - if escapeCount%2 == 0 { + if isUnescapedRune(runes, i) { inDoubleQuote = !inDoubleQuote } } @@ -295,13 +276,7 @@ func isInsideRegexLiteral(text string) bool { case '/': if !inSingleQuote && !inDoubleQuote && !inBacktick { // Check if escaped - escapeCount := 0 - j := i - 1 - for j >= 0 && runes[j] == '\\' { - escapeCount++ - j-- - } - if escapeCount%2 == 0 { + if isUnescapedRune(runes, i) { if inRegex { // End of regex inRegex = false diff --git a/pkg/workflow/known_needs_expressions.go b/pkg/workflow/known_needs_expressions.go index a3a2b13db14..1865fb88450 100644 --- a/pkg/workflow/known_needs_expressions.go +++ b/pkg/workflow/known_needs_expressions.go @@ -14,103 +14,71 @@ import ( var knownNeedsLog = logger.New("workflow:known_needs") // generateKnownNeedsExpressions generates expression mappings for all known needs.* variables -// that could be referenced in the prompt. This ensures that these variables are available -// for interpolation even if the markdown changes without recompilation. -// -// IMPORTANT: The prompt is generated in the ACTIVATION job, so it can only access outputs -// from jobs that the activation job depends on (i.e., jobs that run BEFORE activation). -// This typically includes: -// - needs.pre_activation.outputs.* (activated, matched_command) - only when pre_activation job exists -// - needs..outputs.* for custom jobs that run before activation -// -// The function does NOT generate mappings for jobs that run AFTER activation: -// - needs.activation.outputs.* (activation is the current job) -// - needs.agent.outputs.* (agent runs AFTER activation, includes detection_success/detection_conclusion) -// - needs..outputs.* (these run AFTER agent) -// -// preActivationJobCreated indicates whether the pre_activation job was created for this workflow. -// When false, no pre_activation output mappings are generated to avoid actionlint errors. -// -// Returns a slice of ExpressionMapping that should be merged with other expression mappings. +// that could be referenced in the prompt. func generateKnownNeedsExpressions(data *WorkflowData, preActivationJobCreated bool) []*ExpressionMapping { knownNeedsLog.Print("Generating known needs.* expressions for activation job") - var mappings []*ExpressionMapping - - // Pre-activation job outputs (activation depends on pre_activation only when it exists) - // Only generate these mappings when the pre_activation job was actually created; - // otherwise referencing needs.pre_activation.outputs.* causes actionlint errors. if preActivationJobCreated { - // Always include the "activated" output - activatedExpr := fmt.Sprintf("needs.%s.outputs.%s", constants.PreActivationJobName, constants.ActivatedOutput) - activatedEnvVar := fmt.Sprintf("GH_AW_NEEDS_%s_OUTPUTS_%s", - normalizeJobNameForEnvVar(string(constants.PreActivationJobName)), - normalizeJobNameForEnvVar(constants.ActivatedOutput)) - mappings = append(mappings, &ExpressionMapping{ - Original: fmt.Sprintf("${{ %s }}", activatedExpr), - EnvVar: activatedEnvVar, - Content: activatedExpr, - }) - - // Only include "matched_command" when the workflow has a command trigger, - // since matched_command is only declared in the pre_activation job outputs for command workflows. - if len(data.Command) > 0 { - matchedCmdExpr := fmt.Sprintf("needs.%s.outputs.%s", constants.PreActivationJobName, constants.MatchedCommandOutput) - matchedCmdEnvVar := fmt.Sprintf("GH_AW_NEEDS_%s_OUTPUTS_%s", - normalizeJobNameForEnvVar(string(constants.PreActivationJobName)), - normalizeJobNameForEnvVar(constants.MatchedCommandOutput)) - mappings = append(mappings, &ExpressionMapping{ - Original: fmt.Sprintf("${{ %s }}", matchedCmdExpr), - EnvVar: matchedCmdEnvVar, - Content: matchedCmdExpr, - }) - } + mappings = append(mappings, preActivationNeedsMappings(data)...) } - - // Custom job outputs from frontmatter jobs - // Only include custom jobs that would run before activation - // (i.e., jobs that don't depend on activation, pre_activation, agent, or detection) if data.Jobs != nil { - customJobNames := getCustomJobsBeforeActivation(data) - for _, jobName := range customJobNames { - // If the job has explicit outputs declared in the frontmatter, skip the generic "output" - // env var unless "output" is explicitly among those declared outputs. - // This prevents actionlint errors when the job declares specific outputs but not "output". - if jobConfig, ok := data.Jobs[jobName].(map[string]any); ok { - if outputsField, hasOutputs := jobConfig["outputs"]; hasOutputs && outputsField != nil { - if outputsMap, ok := outputsField.(map[string]any); ok { - if _, hasOutputKey := outputsMap["output"]; !hasOutputKey { - // Job has explicit outputs but "output" is not among them - skip - knownNeedsLog.Printf("Skipping generic 'output' env var for job '%s': has explicit outputs without 'output'", jobName) - continue - } - } - } - } + mappings = append(mappings, customJobNeedsMappings(data)...) + } + knownNeedsLog.Printf("Generated %d known needs.* expression mappings", len(mappings)) + return mappings +} - // For custom jobs without explicit outputs (or with "output" declared), - // add the most commonly used output name: "output" - commonCustomOutputs := []string{ - "output", - } - for _, output := range commonCustomOutputs { - expr := fmt.Sprintf("needs.%s.outputs.%s", jobName, output) - envVar := fmt.Sprintf("GH_AW_NEEDS_%s_OUTPUTS_%s", - normalizeJobNameForEnvVar(jobName), - normalizeJobNameForEnvVar(output)) - mappings = append(mappings, &ExpressionMapping{ - Original: fmt.Sprintf("${{ %s }}", expr), - EnvVar: envVar, - Content: expr, - }) - } - } +func preActivationNeedsMappings(data *WorkflowData) []*ExpressionMapping { + mappings := []*ExpressionMapping{ + knownNeedsExpressionMapping(string(constants.PreActivationJobName), constants.ActivatedOutput), } + if len(data.Command) > 0 { + mappings = append(mappings, knownNeedsExpressionMapping(string(constants.PreActivationJobName), constants.MatchedCommandOutput)) + } + return mappings +} - knownNeedsLog.Printf("Generated %d known needs.* expression mappings", len(mappings)) +func customJobNeedsMappings(data *WorkflowData) []*ExpressionMapping { + var mappings []*ExpressionMapping + for _, jobName := range getCustomJobsBeforeActivation(data) { + if shouldSkipGenericCustomJobOutput(data, jobName) { + knownNeedsLog.Printf("Skipping generic 'output' env var for job '%s': has explicit outputs without 'output'", jobName) + continue + } + mappings = append(mappings, knownNeedsExpressionMapping(jobName, "output")) + } return mappings } +func knownNeedsExpressionMapping(jobName, output string) *ExpressionMapping { + expr := fmt.Sprintf("needs.%s.outputs.%s", jobName, output) + envVar := fmt.Sprintf("GH_AW_NEEDS_%s_OUTPUTS_%s", + normalizeJobNameForEnvVar(jobName), + normalizeJobNameForEnvVar(output)) + return &ExpressionMapping{ + Original: fmt.Sprintf("${{ %s }}", expr), + EnvVar: envVar, + Content: expr, + } +} + +func shouldSkipGenericCustomJobOutput(data *WorkflowData, jobName string) bool { + jobConfig, ok := data.Jobs[jobName].(map[string]any) + if !ok { + return false + } + outputsField, hasOutputs := jobConfig["outputs"] + if !hasOutputs || outputsField == nil { + return false + } + outputsMap, ok := outputsField.(map[string]any) + if !ok { + return false + } + _, hasOutputKey := outputsMap["output"] + return !hasOutputKey +} + // filterExpressionsForActivation filters expression mappings to remove any that reference // custom jobs NOT in beforeActivationJobs. This prevents actionlint errors when a custom job // explicitly depends on activation (and therefore runs AFTER activation) but the markdown body diff --git a/pkg/workflow/lock_validation.go b/pkg/workflow/lock_validation.go index 6e1d7bb3b6a..5fff42b8aac 100644 --- a/pkg/workflow/lock_validation.go +++ b/pkg/workflow/lock_validation.go @@ -91,43 +91,47 @@ func ValidateActionSHAsInLockFile(ctx context.Context, lockFilePath string, cach // Check for updates checks := CheckActionSHAUpdates(ctx, actions, resolver) - // Count and report updates - updateCount := 0 - for _, check := range checks { - if check.NeedsUpdate { - updateCount++ - // Emit warning (FormatWarningMessage adds the warning emoji) - warningMsg := fmt.Sprintf("%s@%s has a newer SHA available: %s → %s", - check.Action.Repo, - check.Action.Version, - check.Action.SHA[:7], - check.LatestSHA[:7]) - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(warningMsg)) - - // Show full SHA in verbose mode - if verbose { - fmt.Fprintf(os.Stderr, " Current: %s\n", check.Action.SHA) - fmt.Fprintf(os.Stderr, " Latest: %s\n", check.LatestSHA) - } - } - } + updateCount := reportActionSHAUpdates(checks, verbose) if updateCount > 0 { - actionSHACheckerLog.Printf("Found %d actions that need updating", updateCount) - // Save the cache with updated SHAs so the next compilation will use them - if err := cache.Save(); err != nil { - actionSHACheckerLog.Printf("Warning: failed to save action cache: %v", err) - } else { - actionSHACheckerLog.Print("Saved updated action cache") - } - // Provide suggestion to fix the issue - fmt.Fprintln(os.Stderr, console.FormatInfoMessage("To apply updated action SHAs, recompile with: gh aw compile")) - if verbose { - fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d action(s) with available updates", updateCount))) - } + handleActionSHAUpdates(cache, updateCount, verbose) } else { actionSHACheckerLog.Print("All actions are up to date") } return nil } + +func reportActionSHAUpdates(checks []ActionUpdateCheck, verbose bool) int { + updateCount := 0 + for _, check := range checks { + if !check.NeedsUpdate { + continue + } + updateCount++ + warningMsg := fmt.Sprintf("%s@%s has a newer SHA available: %s → %s", + check.Action.Repo, + check.Action.Version, + check.Action.SHA[:7], + check.LatestSHA[:7]) + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(warningMsg)) + if verbose { + fmt.Fprintf(os.Stderr, " Current: %s\n", check.Action.SHA) + fmt.Fprintf(os.Stderr, " Latest: %s\n", check.LatestSHA) + } + } + return updateCount +} + +func handleActionSHAUpdates(cache *ActionCache, updateCount int, verbose bool) { + actionSHACheckerLog.Printf("Found %d actions that need updating", updateCount) + if err := cache.Save(); err != nil { + actionSHACheckerLog.Printf("Warning: failed to save action cache: %v", err) + } else { + actionSHACheckerLog.Print("Saved updated action cache") + } + fmt.Fprintln(os.Stderr, console.FormatInfoMessage("To apply updated action SHAs, recompile with: gh aw compile")) + if verbose { + fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %d action(s) with available updates", updateCount))) + } +} diff --git a/pkg/workflow/lsp_manager.go b/pkg/workflow/lsp_manager.go index f22e21653e4..92e8762f150 100644 --- a/pkg/workflow/lsp_manager.go +++ b/pkg/workflow/lsp_manager.go @@ -131,79 +131,69 @@ func (m *LSPManager) GenerateInstallSteps(workflowData *WorkflowData) []GitHubAc config := m.servers[language] effectiveVersion := strings.TrimPrefix(config.Version, "v") - var step GitHubActionStep if len(spec.NpmPackages) > 0 { - // npm-based LSP server: build install command from runtime-manager settings. - args := []string{"npm", "install", "-g"} - if !runInstallScripts { - args = append(args, "--ignore-scripts") - } - // Pin each npm package to its version. The primary (last) package is the - // LSP server binary itself; its version can be overridden via the frontmatter - // 'version' field. All other packages use their hardcoded default version. - primaryPkg := spec.NpmPackages[len(spec.NpmPackages)-1] - for _, pkg := range spec.NpmPackages { - ver := spec.NpmPackageVersions[pkg] - if pkg == primaryPkg && effectiveVersion != "" { - ver = effectiveVersion - } - if ver != "" { - args = append(args, pkg+"@"+ver) - } else { - args = append(args, pkg) - } - } - installCmd := strings.Join(args, " ") - step = GitHubActionStep{ - " - name: " + spec.StepName, - " run: " + installCmd, - } - if cooldownEnabled { - step = append(step, - " env:", - fmt.Sprintf(" NPM_CONFIG_MIN_RELEASE_AGE: '%d'", npmDefaultCooldownDays), - ) - } - step = append(step, " timeout-minutes: 10") + steps = append(steps, buildNpmLSPInstallStep(spec, effectiveVersion, runInstallScripts, cooldownEnabled)) } else { - // Non-npm LSP server (go install, gem install, rustup): build versioned command. - var installCmd string - switch language { - case "go": - ver := spec.DefaultVersion - if effectiveVersion != "" { - ver = effectiveVersion - } - if ver != "" { - installCmd = "go install golang.org/x/tools/gopls@v" + ver - } else { - installCmd = "go install golang.org/x/tools/gopls@latest" - } - case "ruby": - ver := spec.DefaultVersion - if effectiveVersion != "" { - ver = effectiveVersion - } - if ver != "" { - installCmd = "gem install solargraph -v " + ver - } else { - installCmd = "gem install solargraph" - } - default: - installCmd = "rustup component add rust-analyzer" - } - step = GitHubActionStep{ - " - name: " + spec.StepName, - " run: " + installCmd, - " timeout-minutes: 10", - } + steps = append(steps, buildNonNpmLSPInstallStep(language, spec, effectiveVersion)) } - steps = append(steps, step) } return steps } +func buildNpmLSPInstallStep(spec lspInstallSpec, effectiveVersion string, runInstallScripts bool, cooldownEnabled bool) GitHubActionStep { + args := []string{"npm", "install", "-g"} + if !runInstallScripts { + args = append(args, "--ignore-scripts") + } + primaryPkg := spec.NpmPackages[len(spec.NpmPackages)-1] + for _, pkg := range spec.NpmPackages { + ver := spec.NpmPackageVersions[pkg] + if pkg == primaryPkg && effectiveVersion != "" { + ver = effectiveVersion + } + if ver != "" { + args = append(args, pkg+"@"+ver) + } else { + args = append(args, pkg) + } + } + step := GitHubActionStep{" - name: " + spec.StepName, " run: " + strings.Join(args, " ")} + if cooldownEnabled { + step = append(step, " env:", fmt.Sprintf(" NPM_CONFIG_MIN_RELEASE_AGE: '%d'", npmDefaultCooldownDays)) + } + return append(step, " timeout-minutes: 10") +} + +func buildNonNpmLSPInstallStep(language string, spec lspInstallSpec, effectiveVersion string) GitHubActionStep { + return GitHubActionStep{ + " - name: " + spec.StepName, + " run: " + nonNpmLSPInstallCommand(language, spec.DefaultVersion, effectiveVersion), + " timeout-minutes: 10", + } +} + +func nonNpmLSPInstallCommand(language, defaultVersion, effectiveVersion string) string { + ver := defaultVersion + if effectiveVersion != "" { + ver = effectiveVersion + } + switch language { + case "go": + if ver != "" { + return "go install golang.org/x/tools/gopls@v" + ver + } + return "go install golang.org/x/tools/gopls@latest" + case "ruby": + if ver != "" { + return "gem install solargraph -v " + ver + } + return "gem install solargraph" + default: + return "rustup component add rust-analyzer" + } +} + // RuntimeRequirements returns the set of runtime requirements for all configured LSP // servers. These are returned as [RuntimeRequirement] values so that the caller can // feed them into the standard runtime manager (DetectRuntimeRequirements / diff --git a/pkg/workflow/maintenance_workflow.go b/pkg/workflow/maintenance_workflow.go index 8182081f3ee..16980b28426 100644 --- a/pkg/workflow/maintenance_workflow.go +++ b/pkg/workflow/maintenance_workflow.go @@ -136,110 +136,108 @@ func isNoOpReportAsIssueEnabled(reportAsIssue *string) bool { // opts.RepoSlug is the owner/repo slug used to determine the default branch for the push // trigger; pass an empty string to fall back to "main". func GenerateMaintenanceWorkflow(ctx context.Context, opts GenerateMaintenanceWorkflowOptions) error { - workflowDataList := opts.WorkflowDataList - workflowDir := opts.WorkflowDir - version := opts.Version - actionMode := opts.ActionMode - actionTag := opts.ActionTag - repoConfig := opts.RepoConfig - repoSlug := opts.RepoSlug maintenanceLog.Print("Checking if maintenance workflow is needed") - // Compute the resolver and setup action reference early — needed in all code - // paths including the maintenance-disabled early-exit path. - var resolver SHAResolver - for _, workflowData := range workflowDataList { - if workflowData != nil && workflowData.ActionResolver != nil { - resolver = workflowData.ActionResolver - break - } - } - setupActionRef := ResolveSetupActionReference(ctx, actionMode, version, actionTag, resolver) - githubScriptPin := getCachedActionPinFromResolver("actions/github-script", resolver) + generation := newMaintenanceWorkflowGeneration(ctx, opts) // Respect explicit opt-out from aw.json: maintenance: false - if repoConfig != nil && repoConfig.MaintenanceDisabled { - if err := handleMaintenanceDisabled(workflowDataList, workflowDir); err != nil { - return err - } - return GenerateAutoUpdateWorkflow(GenerateAutoUpdateWorkflowOptions{ - Context: ctx, - WorkflowDir: workflowDir, - Enabled: repoConfig.IsAutoUpgradeEnabled(), - RepoSlug: repoSlug, - SetupActionRef: setupActionRef, - GitHubScriptPin: githubScriptPin, - ActionMode: actionMode, - Version: version, - ActionTag: actionTag, - Resolver: resolver, - }) - } - - // Determine the runs-on value to use for all maintenance jobs. - const defaultRunsOn = "ubuntu-slim" - var configuredRunsOn RunsOnValue - disableLabelTrigger := true // default: disable label-triggered jobs (opt-in) - var maintenanceConfig *MaintenanceConfig - var compileGitHubTokenSecret string - enableCompileCreatePullRequest := false - if repoConfig != nil && repoConfig.Maintenance != nil { - maintenanceConfig = repoConfig.Maintenance - configuredRunsOn = maintenanceConfig.RunsOn - disableLabelTrigger = !maintenanceConfig.IsLabelTriggerEnabled() - if maintenanceConfig.Compile != nil { - compileGitHubTokenSecret = maintenanceConfig.Compile.CreatePullRequestGitHubToken - enableCompileCreatePullRequest = strings.TrimSpace(compileGitHubTokenSecret) != "" - } + if opts.RepoConfig != nil && opts.RepoConfig.MaintenanceDisabled { + return generateMaintenanceDisabledAutoUpdate(ctx, opts, generation) } - runsOnValue := FormatRunsOn(configuredRunsOn, defaultRunsOn) // Scan workflows for expires fields and track the minimum expires value - hasExpires, minExpires, triggerReason := scanWorkflowsForExpires(workflowDataList) + hasExpires, minExpires, triggerReason := scanWorkflowsForExpires(opts.WorkflowDataList) if !hasExpires { - maintenanceLog.Print("No workflows use expires field, skipping maintenance workflow generation") - - // Delete existing maintenance workflow file if it exists (no expires means no need for maintenance) - maintenanceFile := filepath.Join(workflowDir, "agentics-maintenance.yml") - if _, err := os.Stat(maintenanceFile); err == nil { - maintenanceLog.Printf("Deleting existing maintenance workflow: %s", maintenanceFile) - if err := os.Remove(maintenanceFile); err != nil { - return fmt.Errorf("failed to delete maintenance workflow: %w", err) - } - maintenanceLog.Print("Maintenance workflow deleted successfully") + return generateNoExpiresMaintenanceWorkflows(ctx, opts, generation) + } + + return generateExpiresMaintenanceWorkflow(ctx, opts, generation, minExpires, triggerReason) +} + +type maintenanceWorkflowGeneration struct { + resolver SHAResolver + setupActionRef string + githubScriptPin string + runsOnValue string + configuredRunsOn RunsOnValue + disableLabelTrigger bool + maintenanceConfig *MaintenanceConfig + compileGitHubTokenSecret string + enableCompileCreatePullRequest bool +} + +func newMaintenanceWorkflowGeneration(ctx context.Context, opts GenerateMaintenanceWorkflowOptions) maintenanceWorkflowGeneration { + resolver := firstWorkflowActionResolver(opts.WorkflowDataList) + generation := maintenanceWorkflowGeneration{ + resolver: resolver, + setupActionRef: ResolveSetupActionReference(ctx, opts.ActionMode, opts.Version, opts.ActionTag, resolver), + githubScriptPin: getCachedActionPinFromResolver("actions/github-script", resolver), + disableLabelTrigger: true, + } + if opts.RepoConfig != nil && opts.RepoConfig.Maintenance != nil { + generation.maintenanceConfig = opts.RepoConfig.Maintenance + generation.configuredRunsOn = generation.maintenanceConfig.RunsOn + generation.disableLabelTrigger = !generation.maintenanceConfig.IsLabelTriggerEnabled() + if generation.maintenanceConfig.Compile != nil { + generation.compileGitHubTokenSecret = generation.maintenanceConfig.Compile.CreatePullRequestGitHubToken + generation.enableCompileCreatePullRequest = strings.TrimSpace(generation.compileGitHubTokenSecret) != "" } + } + generation.runsOnValue = FormatRunsOn(generation.configuredRunsOn, "ubuntu-slim") + return generation +} - // Even without expires, side-repo targets still need maintenance workflows - // for safe_outputs, create_labels, and validate operations. - if err := generateAllSideRepoMaintenanceWorkflows(ctx, generateAllSideRepoMaintenanceWorkflowsOptions{ - workflowDataList: workflowDataList, - workflowDir: workflowDir, - version: version, - actionMode: actionMode, - actionTag: actionTag, - runsOnValue: runsOnValue, - resolver: resolver, - hasExpires: false, - minExpiresDays: 0, - }); err != nil { - return err +func firstWorkflowActionResolver(workflowDataList []*WorkflowData) SHAResolver { + for _, workflowData := range workflowDataList { + if workflowData != nil && workflowData.ActionResolver != nil { + return workflowData.ActionResolver } + } + return nil +} - return GenerateAutoUpdateWorkflow(GenerateAutoUpdateWorkflowOptions{ - Context: ctx, - WorkflowDir: workflowDir, - Enabled: repoConfig != nil && repoConfig.IsAutoUpgradeEnabled(), - RepoSlug: repoSlug, - SetupActionRef: setupActionRef, - GitHubScriptPin: githubScriptPin, - ActionMode: actionMode, - Version: version, - ActionTag: actionTag, - Resolver: resolver, - }) +func generateMaintenanceDisabledAutoUpdate(ctx context.Context, opts GenerateMaintenanceWorkflowOptions, generation maintenanceWorkflowGeneration) error { + if err := handleMaintenanceDisabled(opts.WorkflowDataList, opts.WorkflowDir); err != nil { + return err } + return generateMaintenanceAutoUpdate(ctx, opts, generation, opts.RepoConfig.IsAutoUpgradeEnabled()) +} +func generateNoExpiresMaintenanceWorkflows(ctx context.Context, opts GenerateMaintenanceWorkflowOptions, generation maintenanceWorkflowGeneration) error { + maintenanceLog.Print("No workflows use expires field, skipping maintenance workflow generation") + if err := deleteExistingMaintenanceWorkflow(opts.WorkflowDir); err != nil { + return err + } + if err := generateAllSideRepoMaintenanceWorkflows(ctx, generateAllSideRepoMaintenanceWorkflowsOptions{ + workflowDataList: opts.WorkflowDataList, + workflowDir: opts.WorkflowDir, + version: opts.Version, + actionMode: opts.ActionMode, + actionTag: opts.ActionTag, + runsOnValue: generation.runsOnValue, + resolver: generation.resolver, + hasExpires: false, + minExpiresDays: 0, + }); err != nil { + return err + } + return generateMaintenanceAutoUpdate(ctx, opts, generation, opts.RepoConfig != nil && opts.RepoConfig.IsAutoUpgradeEnabled()) +} + +func deleteExistingMaintenanceWorkflow(workflowDir string) error { + maintenanceFile := filepath.Join(workflowDir, "agentics-maintenance.yml") + if _, err := os.Stat(maintenanceFile); err == nil { + maintenanceLog.Printf("Deleting existing maintenance workflow: %s", maintenanceFile) + if err := os.Remove(maintenanceFile); err != nil { + return fmt.Errorf("failed to delete maintenance workflow: %w", err) + } + maintenanceLog.Print("Maintenance workflow deleted successfully") + } + return nil +} + +func generateExpiresMaintenanceWorkflow(ctx context.Context, opts GenerateMaintenanceWorkflowOptions, generation maintenanceWorkflowGeneration, minExpires int, triggerReason string) error { maintenanceLog.Printf("Maintenance workflow generation triggered: %s", triggerReason) maintenanceLog.Printf("Generating maintenance workflow for expired discussions, issues, and pull requests (minimum expires: %d hours)", minExpires) @@ -255,72 +253,87 @@ func GenerateMaintenanceWorkflow(ctx context.Context, opts GenerateMaintenanceWo // Fetch the default branch for the push trigger (dev mode only) // Resolved here to avoid passing it through multiple layers; empty slug falls back to "main" - defaultBranch := FetchDefaultBranch(repoSlug) + defaultBranch := FetchDefaultBranch(opts.RepoSlug) + + content := buildExpiresMaintenanceWorkflowContent(ctx, opts, generation, minExpiresDays, cronSchedule, scheduleDesc, defaultBranch) - // Generate the YAML content for the maintenance workflow + // Write the maintenance workflow file + if err := writeMaintenanceWorkflowFile(opts.WorkflowDir, content); err != nil { + return err + } + + // Generate side-repo maintenance workflows for any SideRepoOps targets detected. + if err := generateSideRepoMaintenanceForExpires(ctx, opts, generation, minExpiresDays); err != nil { + return err + } + + return generateMaintenanceAutoUpdate(ctx, opts, generation, opts.RepoConfig != nil && opts.RepoConfig.IsAutoUpgradeEnabled()) +} + +func buildExpiresMaintenanceWorkflowContent(ctx context.Context, opts GenerateMaintenanceWorkflowOptions, generation maintenanceWorkflowGeneration, minExpiresDays int, cronSchedule string, scheduleDesc string, defaultBranch string) string { maintenanceLog.Printf( "Maintenance compile configuration: createPullRequest=%v tokenSecretConfigured=%v", - enableCompileCreatePullRequest, - strings.TrimSpace(compileGitHubTokenSecret) != "", + generation.enableCompileCreatePullRequest, + strings.TrimSpace(generation.compileGitHubTokenSecret) != "", ) - copilotOrgBilling := allCopilotWorkflowsUseOrgBilling(workflowDataList) - content := buildMaintenanceWorkflowYAML(ctx, buildMaintenanceWorkflowYAMLOptions{ + return buildMaintenanceWorkflowYAML(ctx, buildMaintenanceWorkflowYAMLOptions{ cronSchedule: cronSchedule, scheduleDesc: scheduleDesc, minExpiresDays: minExpiresDays, - runsOnValue: runsOnValue, - actionMode: actionMode, - version: version, - actionTag: actionTag, - resolver: resolver, - configuredRunsOn: configuredRunsOn, + runsOnValue: generation.runsOnValue, + actionMode: opts.ActionMode, + version: opts.Version, + actionTag: opts.ActionTag, + resolver: generation.resolver, + configuredRunsOn: generation.configuredRunsOn, defaultBranch: defaultBranch, - disableLabelTrigger: disableLabelTrigger, - maintenanceConfig: maintenanceConfig, - compileGitHubToken: getEffectiveMaintenanceGitHubToken(compileGitHubTokenSecret), - createCompilePR: enableCompileCreatePullRequest, - copilotOrgBilling: copilotOrgBilling, + disableLabelTrigger: generation.disableLabelTrigger, + maintenanceConfig: generation.maintenanceConfig, + compileGitHubToken: getEffectiveMaintenanceGitHubToken(generation.compileGitHubTokenSecret), + createCompilePR: generation.enableCompileCreatePullRequest, + copilotOrgBilling: allCopilotWorkflowsUseOrgBilling(opts.WorkflowDataList), }) +} - // Write the maintenance workflow file +func writeMaintenanceWorkflowFile(workflowDir string, content string) error { maintenanceFile := filepath.Join(workflowDir, "agentics-maintenance.yml") maintenanceLog.Printf("Writing maintenance workflow to %s", maintenanceFile) - if err := fileutil.EnsureParentDir(maintenanceFile, constants.DirPermPublic); err != nil { return fmt.Errorf("failed to create maintenance workflow directory: %w", err) } if err := os.WriteFile(maintenanceFile, []byte(content), constants.FilePermPublic); err != nil { return fmt.Errorf("failed to write maintenance workflow: %w", err) } - maintenanceLog.Print("Maintenance workflow generated successfully") + return nil +} - // Generate side-repo maintenance workflows for any SideRepoOps targets detected. - if err := generateAllSideRepoMaintenanceWorkflows(ctx, generateAllSideRepoMaintenanceWorkflowsOptions{ - workflowDataList: workflowDataList, - workflowDir: workflowDir, - version: version, - actionMode: actionMode, - actionTag: actionTag, - runsOnValue: runsOnValue, - resolver: resolver, - hasExpires: hasExpires, +func generateSideRepoMaintenanceForExpires(ctx context.Context, opts GenerateMaintenanceWorkflowOptions, generation maintenanceWorkflowGeneration, minExpiresDays int) error { + return generateAllSideRepoMaintenanceWorkflows(ctx, generateAllSideRepoMaintenanceWorkflowsOptions{ + workflowDataList: opts.WorkflowDataList, + workflowDir: opts.WorkflowDir, + version: opts.Version, + actionMode: opts.ActionMode, + actionTag: opts.ActionTag, + runsOnValue: generation.runsOnValue, + resolver: generation.resolver, + hasExpires: true, minExpiresDays: minExpiresDays, - }); err != nil { - return err - } + }) +} +func generateMaintenanceAutoUpdate(ctx context.Context, opts GenerateMaintenanceWorkflowOptions, generation maintenanceWorkflowGeneration, enabled bool) error { return GenerateAutoUpdateWorkflow(GenerateAutoUpdateWorkflowOptions{ Context: ctx, - WorkflowDir: workflowDir, - Enabled: repoConfig != nil && repoConfig.IsAutoUpgradeEnabled(), - RepoSlug: repoSlug, - SetupActionRef: setupActionRef, - GitHubScriptPin: githubScriptPin, - ActionMode: actionMode, - Version: version, - ActionTag: actionTag, - Resolver: resolver, + WorkflowDir: opts.WorkflowDir, + Enabled: enabled, + RepoSlug: opts.RepoSlug, + SetupActionRef: generation.setupActionRef, + GitHubScriptPin: generation.githubScriptPin, + ActionMode: opts.ActionMode, + Version: opts.Version, + ActionTag: opts.ActionTag, + Resolver: generation.resolver, }) } @@ -384,70 +397,51 @@ func allCopilotWorkflowsUseOrgBilling(workflowDataList []*WorkflowData) bool { // whether any expires fields are set, the minimum expires value in hours, and the // first reason that triggered maintenance workflow generation. func scanWorkflowsForExpires(workflowDataList []*WorkflowData) (bool, int, string) { - hasExpires := false - minExpires := 0 // Track minimum expires value in hours - triggerReason := "" - - setTriggerReason := func(reason string) { - if triggerReason == "" { - triggerReason = reason - maintenanceLog.Printf("Maintenance workflow became required: %s", reason) - } - } + state := &expiresScanState{} for _, workflowData := range workflowDataList { if workflowData == nil || workflowData.SafeOutputs == nil { continue } - // Check for expired discussions - if workflowData.SafeOutputs.CreateDiscussions != nil { - if workflowData.SafeOutputs.CreateDiscussions.Expires > 0 { - hasExpires = true - expires := workflowData.SafeOutputs.CreateDiscussions.Expires - setTriggerReason(fmt.Sprintf("workflow %q sets safe_outputs.create_discussions.expires=%dh", workflowData.Name, expires)) - maintenanceLog.Printf("Workflow %s has expires field set to %d hours for discussions", workflowData.Name, expires) - if minExpires == 0 || expires < minExpires { - minExpires = expires - } - } - } - // Check for expired issues - if workflowData.SafeOutputs.CreateIssues != nil { - if workflowData.SafeOutputs.CreateIssues.Expires > 0 { - hasExpires = true - expires := workflowData.SafeOutputs.CreateIssues.Expires - setTriggerReason(fmt.Sprintf("workflow %q sets safe_outputs.create_issues.expires=%dh", workflowData.Name, expires)) - maintenanceLog.Printf("Workflow %s has expires field set to %d hours for issues", workflowData.Name, expires) - if minExpires == 0 || expires < minExpires { - minExpires = expires - } - } - } - // Check for expired pull requests - if workflowData.SafeOutputs.CreatePullRequests != nil { - if workflowData.SafeOutputs.CreatePullRequests.Expires > 0 { - hasExpires = true - expires := workflowData.SafeOutputs.CreatePullRequests.Expires - setTriggerReason(fmt.Sprintf("workflow %q sets safe_outputs.create_pull_requests.expires=%dh", workflowData.Name, expires)) - maintenanceLog.Printf("Workflow %s has expires field set to %d hours for pull requests", workflowData.Name, expires) - if minExpires == 0 || expires < minExpires { - minExpires = expires - } - } - } - // Check for no-op runs issue expiration (runtime defaults to 30 days) - if workflowData.SafeOutputs.NoOp != nil { - if isNoOpReportAsIssueEnabled(workflowData.SafeOutputs.NoOp.ReportAsIssue) { - hasExpires = true - expires := defaultNoOpIssueExpirationHours - setTriggerReason(fmt.Sprintf("workflow %q enables no-op issue reporting (default expiration %dh)", workflowData.Name, expires)) - maintenanceLog.Printf("Workflow %s has no-op report-as-issue enabled, using %d-hour no-op issue expiration", workflowData.Name, expires) - if minExpires == 0 || expires < minExpires { - minExpires = expires - } - } - } + scanWorkflowForExpires(workflowData, state) } - return hasExpires, minExpires, triggerReason + return state.hasExpires, state.minExpires, state.triggerReason +} + +type expiresScanState struct { + hasExpires bool + minExpires int + triggerReason string +} + +func (s *expiresScanState) record(workflowName string, expires int, reason string, logMessage string) { + s.hasExpires = true + if s.triggerReason == "" { + s.triggerReason = reason + maintenanceLog.Printf("Maintenance workflow became required: %s", reason) + } + maintenanceLog.Printf(logMessage, workflowName, expires) + if s.minExpires == 0 || expires < s.minExpires { + s.minExpires = expires + } +} + +func scanWorkflowForExpires(workflowData *WorkflowData, state *expiresScanState) { + if workflowData.SafeOutputs.CreateDiscussions != nil && workflowData.SafeOutputs.CreateDiscussions.Expires > 0 { + expires := workflowData.SafeOutputs.CreateDiscussions.Expires + state.record(workflowData.Name, expires, fmt.Sprintf("workflow %q sets safe_outputs.create_discussions.expires=%dh", workflowData.Name, expires), "Workflow %s has expires field set to %d hours for discussions") + } + if workflowData.SafeOutputs.CreateIssues != nil && workflowData.SafeOutputs.CreateIssues.Expires > 0 { + expires := workflowData.SafeOutputs.CreateIssues.Expires + state.record(workflowData.Name, expires, fmt.Sprintf("workflow %q sets safe_outputs.create_issues.expires=%dh", workflowData.Name, expires), "Workflow %s has expires field set to %d hours for issues") + } + if workflowData.SafeOutputs.CreatePullRequests != nil && workflowData.SafeOutputs.CreatePullRequests.Expires > 0 { + expires := workflowData.SafeOutputs.CreatePullRequests.Expires + state.record(workflowData.Name, expires, fmt.Sprintf("workflow %q sets safe_outputs.create_pull_requests.expires=%dh", workflowData.Name, expires), "Workflow %s has expires field set to %d hours for pull requests") + } + if workflowData.SafeOutputs.NoOp != nil && isNoOpReportAsIssueEnabled(workflowData.SafeOutputs.NoOp.ReportAsIssue) { + expires := defaultNoOpIssueExpirationHours + state.record(workflowData.Name, expires, fmt.Sprintf("workflow %q enables no-op issue reporting (default expiration %dh)", workflowData.Name, expires), "Workflow %s has no-op report-as-issue enabled, using %d-hour no-op issue expiration") + } } diff --git a/pkg/workflow/maintenance_workflow_yaml.go b/pkg/workflow/maintenance_workflow_yaml.go index 7f509c849d2..4c90a5145be 100644 --- a/pkg/workflow/maintenance_workflow_yaml.go +++ b/pkg/workflow/maintenance_workflow_yaml.go @@ -32,33 +32,7 @@ type buildMaintenanceWorkflowYAMLOptions struct { // buildMaintenanceWorkflowYAML generates the complete YAML content for the // agentics-maintenance.yml workflow. It is called by GenerateMaintenanceWorkflow // after the cron schedule and setup parameters have been resolved. -func buildMaintenanceWorkflowYAML( - ctx context.Context, - opts buildMaintenanceWorkflowYAMLOptions, -) string { - cronSchedule := opts.cronSchedule - scheduleDesc := opts.scheduleDesc - minExpiresDays := opts.minExpiresDays - runsOnValue := opts.runsOnValue - actionMode := opts.actionMode - version := opts.version - actionTag := opts.actionTag - resolver := opts.resolver - configuredRunsOn := opts.configuredRunsOn - defaultBranch := opts.defaultBranch - disableLabelTrigger := opts.disableLabelTrigger - maintenanceConfig := opts.maintenanceConfig - compileGitHubToken := opts.compileGitHubToken - createCompilePR := opts.createCompilePR - copilotOrgBilling := opts.copilotOrgBilling - maintenanceWorkflowYAMLLog.Printf("Building maintenance workflow YAML: actionMode=%s minExpiresDays=%d cronSchedule=%q defaultBranch=%q disableLabelTrigger=%v createCompilePR=%v copilotOrgBilling=%v", actionMode, minExpiresDays, cronSchedule, defaultBranch, disableLabelTrigger, createCompilePR, copilotOrgBilling) - labelDisableJobEnabled := !disableLabelTrigger && !maintenanceConfig.IsJobDisabled("label_disable_agentic_workflow") - labelApplySafeOutputsJobEnabled := !disableLabelTrigger && !maintenanceConfig.IsJobDisabled("label_apply_safe_outputs") - - var yaml strings.Builder - - // Add workflow header with logo and instructions - customInstructions := `This file defines the generated agentic maintenance workflow for this repository. +const maintenanceWorkflowCustomInstructions = `This file defines the generated agentic maintenance workflow for this repository. It runs scheduled cleanup for expiring safe outputs and supports manual maintenance operations. This workflow is generated automatically when workflows use expiring safe outputs @@ -70,43 +44,7 @@ To disable maintenance workflow generation, set in .github/workflows/aw.json: Agentic maintenance docs: https://github.github.com/gh-aw/reference/ephemerals/#manual-maintenance-operations` - header := GenerateWorkflowHeader("", "pkg/workflow/maintenance_workflow.go", customInstructions) - yaml.WriteString(header) - - yaml.WriteString(`name: Agentic Maintenance - -on: - schedule: - - cron: "` + cronSchedule + `" # ` + scheduleDesc + ` (based on minimum expires: ` + strconv.Itoa(minExpiresDays) + ` days) -`) - - // Add push trigger in dev mode so compile-workflows runs when workflow files change - if actionMode == ActionModeDev { - maintenanceWorkflowYAMLLog.Printf("Adding dev-mode push trigger for branch %q", defaultBranch) - yaml.WriteString(` push: - branches: - - ` + defaultBranch + ` - paths: - - '.github/workflows/*.md' -`) - } - - // Add label-event trigger only when the label-triggered jobs are enabled - if labelDisableJobEnabled || labelApplySafeOutputsJobEnabled { - maintenanceWorkflowYAMLLog.Print("Adding issues:labeled trigger for label-triggered maintenance jobs") - yaml.WriteString(` issues: - types: [labeled] -`) - } - - appliedRunURLValue := "${{ jobs.apply_safe_outputs.outputs.run_url }}" - appliedRunURLDescription := "The run URL that safe outputs were applied from" - if maintenanceConfig.IsJobDisabled("apply_safe_outputs") { - appliedRunURLValue = "${{ inputs.run_url }}" - appliedRunURLDescription = "The run URL that safe outputs were applied from (workflow_call falls back to inputs.run_url when apply_safe_outputs is disabled; other triggers leave this empty)" - } - - yaml.WriteString(` workflow_dispatch: +const maintenanceWorkflowEntrypointsPrefix = ` workflow_dispatch: inputs: operation: description: 'Optional maintenance operation to run' @@ -149,102 +87,127 @@ on: description: 'The maintenance operation that was completed (empty when none ran or a scheduled job ran)' value: ${{ jobs.run_operation.outputs.operation || inputs.operation }} applied_run_url: - description: '` + appliedRunURLDescription + `' - value: ` + appliedRunURLValue + ` +` + +func buildMaintenanceWorkflowYAML( + ctx context.Context, + opts buildMaintenanceWorkflowYAMLOptions, +) string { + maintenanceWorkflowYAMLLog.Printf("Building maintenance workflow YAML: actionMode=%s minExpiresDays=%d cronSchedule=%q defaultBranch=%q disableLabelTrigger=%v createCompilePR=%v copilotOrgBilling=%v", opts.actionMode, opts.minExpiresDays, opts.cronSchedule, opts.defaultBranch, opts.disableLabelTrigger, opts.createCompilePR, opts.copilotOrgBilling) + labelDisableJobEnabled := !opts.disableLabelTrigger && !opts.maintenanceConfig.IsJobDisabled("label_disable_agentic_workflow") + labelApplySafeOutputsJobEnabled := !opts.disableLabelTrigger && !opts.maintenanceConfig.IsJobDisabled("label_apply_safe_outputs") + setupActionRef := ResolveSetupActionReference(ctx, opts.actionMode, opts.version, opts.actionTag, opts.resolver) + + var yaml strings.Builder + writeMaintenanceWorkflowHeader(&yaml) + writeMaintenanceOnTriggers(&yaml, opts, labelDisableJobEnabled, labelApplySafeOutputsJobEnabled) + writeMaintenanceEntrypointsAndPermissions(&yaml, opts.maintenanceConfig) + writeMaintenanceCloseExpiredJobs(&yaml, opts, setupActionRef) + writeMaintenanceCleanupCacheMemoryJob(&yaml, opts, setupActionRef) + writeMaintenanceRunOperationJob(&yaml, ctx, opts, setupActionRef) + writeMaintenanceUpdatePullRequestBranchesJob(&yaml, opts, setupActionRef) + if !opts.maintenanceConfig.IsJobDisabled("apply_safe_outputs") { + writeMaintenanceApplySafeOutputsJob(&yaml, opts, setupActionRef) + } + writeMaintenanceCreateLabelsJob(&yaml, ctx, opts, setupActionRef) + writeMaintenanceActivityReportJob(&yaml, ctx, opts, setupActionRef) + writeMaintenanceForecastReportJob(&yaml, ctx, opts, setupActionRef) + writeMaintenanceCloseAgenticIssuesJob(&yaml, opts, setupActionRef) + writeMaintenanceValidateWorkflowsJob(&yaml, ctx, opts, setupActionRef) + writeMaintenanceLabelJobs(&yaml, opts, setupActionRef, labelDisableJobEnabled, labelApplySafeOutputsJobEnabled) + if opts.actionMode == ActionModeDev { + writeMaintenanceDevJobs(&yaml, ctx, opts, setupActionRef) + } + return yaml.String() +} -permissions: {} +func writeMaintenanceWorkflowHeader(yaml *strings.Builder) { + header := GenerateWorkflowHeader("", "pkg/workflow/maintenance_workflow.go", maintenanceWorkflowCustomInstructions) + yaml.WriteString(header) + yaml.WriteString("name: Agentic Maintenance\n\non:\n") +} -jobs: +func writeMaintenanceOnTriggers(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, labelDisableJobEnabled, labelApplySafeOutputsJobEnabled bool) { + yaml.WriteString(` schedule: + - cron: "` + opts.cronSchedule + `" # ` + opts.scheduleDesc + ` (based on minimum expires: ` + strconv.Itoa(opts.minExpiresDays) + ` days) +`) + if opts.actionMode == ActionModeDev { + maintenanceWorkflowYAMLLog.Printf("Adding dev-mode push trigger for branch %q", opts.defaultBranch) + yaml.WriteString(` push: + branches: + - ` + opts.defaultBranch + ` + paths: + - '.github/workflows/*.md' +`) + } + if labelDisableJobEnabled || labelApplySafeOutputsJobEnabled { + maintenanceWorkflowYAMLLog.Print("Adding issues:labeled trigger for label-triggered maintenance jobs") + yaml.WriteString(` issues: + types: [labeled] `) + } +} - setupActionRef := ResolveSetupActionReference(ctx, actionMode, version, actionTag, resolver) +func writeMaintenanceEntrypointsAndPermissions(yaml *strings.Builder, maintenanceConfig *MaintenanceConfig) { + appliedRunURLValue := "${{ jobs.apply_safe_outputs.outputs.run_url }}" + appliedRunURLDescription := "The run URL that safe outputs were applied from" + if maintenanceConfig.IsJobDisabled("apply_safe_outputs") { + appliedRunURLValue = "${{ inputs.run_url }}" + appliedRunURLDescription = "The run URL that safe outputs were applied from (workflow_call falls back to inputs.run_url when apply_safe_outputs is disabled; other triggers leave this empty)" + } + yaml.WriteString(maintenanceWorkflowEntrypointsPrefix) + yaml.WriteString(" description: '" + appliedRunURLDescription + "'\n") + yaml.WriteString(" value: " + appliedRunURLValue + "\n\npermissions: {}\n\njobs:\n") +} - writeCloseExpiredJob := func(jobName string, permissionLine string, stepName string, scriptName string) { - yaml.WriteString(` ` + jobName + `: +func writeMaintenanceCloseExpiredJobs(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { + if opts.maintenanceConfig.IsJobDisabled("close-expired-entities") { + return + } + writeMaintenanceCloseExpiredJob(yaml, opts, setupActionRef, "close-expired-discussions", "discussions: write", "Close expired discussions", "close_expired_discussions") + writeMaintenanceCloseExpiredJob(yaml, opts, setupActionRef, "close-expired-issues", "issues: write", "Close expired issues", "close_expired_issues") + writeMaintenanceCloseExpiredJob(yaml, opts, setupActionRef, "close-expired-pull-requests", "pull-requests: write", "Close expired pull requests", "close_expired_pull_requests") +} + +func writeMaintenanceCloseExpiredJob(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef, jobName, permissionLine, stepName, scriptName string) { + yaml.WriteString(` ` + jobName + `: if: ${{ ` + RenderCondition(buildNotForkAndScheduleOnly()) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: ` + permissionLine + ` steps: `) - if actionMode == ActionModeDev || actionMode == ActionModeScript { - maintenanceWorkflowYAMLLog.Printf("Adding checkout step for %s (actionMode=%s)", jobName, actionMode) - yaml.WriteString(" - name: Checkout actions folder\n") - yaml.WriteString(" uses: " + getActionPin("actions/checkout") + "\n") - yaml.WriteString(" with:\n") - yaml.WriteString(" sparse-checkout: |\n") - yaml.WriteString(" actions\n") - yaml.WriteString(" clean: false\n") - yaml.WriteString(" persist-credentials: false\n\n") - } - yaml.WriteString(` - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: ` + stepName + ` - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/` + scriptName + `.cjs'); - await main(); -`) - } - - if !maintenanceConfig.IsJobDisabled("close-expired-entities") { - writeCloseExpiredJob("close-expired-discussions", "discussions: write", "Close expired discussions", "close_expired_discussions") - writeCloseExpiredJob("close-expired-issues", "issues: write", "Close expired issues", "close_expired_issues") - writeCloseExpiredJob("close-expired-pull-requests", "pull-requests: write", "Close expired pull requests", "close_expired_pull_requests") + if opts.actionMode == ActionModeDev || opts.actionMode == ActionModeScript { + maintenanceWorkflowYAMLLog.Printf("Adding checkout step for %s (actionMode=%s)", jobName, opts.actionMode) + writeMaintenanceActionsCheckoutStep(yaml) } + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceScriptStep(yaml, opts.resolver, stepName, "", "", scriptName) +} - // Add cleanup-cache-memory job for scheduled runs and clean_cache_memories operation - // This job lists all caches starting with "memory-", groups them by key prefix, - // keeps the latest run ID per group, and deletes the rest. +func writeMaintenanceCleanupCacheMemoryJob(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { cleanupCacheCondition := buildNotForkAndScheduleOnlyOrOperation("clean_cache_memories") yaml.WriteString(` cleanup-cache-memory: if: ${{ ` + RenderCondition(cleanupCacheCondition) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: actions: write steps: `) - - // Add checkout step only in dev/script mode (for local action paths) - if actionMode == ActionModeDev || actionMode == ActionModeScript { - yaml.WriteString(" - name: Checkout actions folder\n") - yaml.WriteString(" uses: " + getActionPin("actions/checkout") + "\n") - yaml.WriteString(" with:\n") - yaml.WriteString(" sparse-checkout: |\n") - yaml.WriteString(" actions\n") - yaml.WriteString(" clean: false\n") - yaml.WriteString(" persist-credentials: false\n\n") + if opts.actionMode == ActionModeDev || opts.actionMode == ActionModeScript { + writeMaintenanceActionsCheckoutStep(yaml) } + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceScriptStep(yaml, opts.resolver, "Cleanup outdated cache-memory entries", "", "", "cleanup_cache_memory") +} - yaml.WriteString(` - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Cleanup outdated cache-memory entries - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/cleanup_cache_memory.cjs'); - await main(); -`) - - // Add unified run_operation job for all dispatch operations except those with dedicated jobs - // (safe_outputs, create_labels, activity_report, close_agentic_workflows_issues, clean_cache_memories, update_pull_request_branches, validate, forecast) +func writeMaintenanceRunOperationJob(yaml *strings.Builder, ctx context.Context, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { runOperationCondition := buildRunOperationCondition("safe_outputs", "create_labels", "activity_report", "close_agentic_workflows_issues", "clean_cache_memories", "update_pull_request_branches", "validate", "forecast") yaml.WriteString(` run_operation: if: ${{ ` + RenderCondition(runOperationCondition) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: actions: write contents: write @@ -252,35 +215,17 @@ jobs: outputs: operation: ${{ steps.record.outputs.operation }} steps: - - name: Checkout repository - uses: ` + getActionPin("actions/checkout") + ` - with: - persist-credentials: false - - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - `) - - yaml.WriteString(generateInstallCLISteps(ctx, actionMode, version, actionTag, resolver)) + writeMaintenanceRepositoryCheckoutStep(yaml) + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "") + yaml.WriteString(generateInstallCLISteps(ctx, opts.actionMode, opts.version, opts.actionTag, opts.resolver)) yaml.WriteString(` - name: Run operation - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` + uses: ` + getCachedActionPinFromResolver("actions/github-script", opts.resolver) + ` env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_AW_OPERATION: ${{ inputs.operation }} - GH_AW_CMD_PREFIX: ` + getCLICmdPrefix(actionMode) + ` + GH_AW_CMD_PREFIX: ` + getCLICmdPrefix(opts.actionMode) + ` with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -295,46 +240,25 @@ jobs: GH_AW_OPERATION: ${{ inputs.operation }} run: echo "operation=$GH_AW_OPERATION" >> "$GITHUB_OUTPUT" `) +} - // Add update_pull_request_branches job for workflow_dispatch with operation == 'update_pull_request_branches' +func writeMaintenanceUpdatePullRequestBranchesJob(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { yaml.WriteString(` update_pull_request_branches: if: ${{ ` + RenderCondition(buildDispatchOperationCondition("update_pull_request_branches")) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: contents: write pull-requests: write steps: `) - - // Add checkout step only in dev/script mode (for local action paths) - if actionMode == ActionModeDev || actionMode == ActionModeScript { - yaml.WriteString(" - name: Checkout actions folder\n") - yaml.WriteString(" uses: " + getActionPin("actions/checkout") + "\n") - yaml.WriteString(" with:\n") - yaml.WriteString(" sparse-checkout: |\n") - yaml.WriteString(" actions\n") - yaml.WriteString(" clean: false\n") - yaml.WriteString(" persist-credentials: false\n\n") + if opts.actionMode == ActionModeDev || opts.actionMode == ActionModeScript { + writeMaintenanceActionsCheckoutStep(yaml) } - - yaml.WriteString(` - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - - - name: Update pull request branches - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "") + yaml.WriteString(` - name: Update pull request branches + uses: ` + getCachedActionPinFromResolver("actions/github-script", opts.resolver) + ` env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -345,13 +269,13 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/update_pull_request_branches.cjs'); await main(); `) +} - // Add apply_safe_outputs job for workflow_dispatch with operation == 'safe_outputs' - if !maintenanceConfig.IsJobDisabled("apply_safe_outputs") { - yaml.WriteString(` +func writeMaintenanceApplySafeOutputsJob(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { + yaml.WriteString(` apply_safe_outputs: if: ${{ ` + RenderCondition(buildDispatchOperationCondition("safe_outputs")) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: actions: read contents: write @@ -361,31 +285,12 @@ jobs: outputs: run_url: ${{ steps.record.outputs.run_url }} steps: - - name: Checkout actions folder - uses: ` + getActionPin("actions/checkout") + ` - with: - sparse-checkout: | - actions - clean: false - persist-credentials: false - - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - - - name: Apply Safe Outputs - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` +`) + writeMaintenanceActionsCheckoutStep(yaml) + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "") + yaml.WriteString(` - name: Apply Safe Outputs + uses: ` + getCachedActionPinFromResolver("actions/github-script", opts.resolver) + ` env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_AW_RUN_URL: ${{ inputs.run_url }} @@ -403,87 +308,46 @@ jobs: GH_AW_RUN_URL: ${{ inputs.run_url }} run: echo "run_url=$GH_AW_RUN_URL" >> "$GITHUB_OUTPUT" `) - } +} - // Add create_labels job for workflow_dispatch with operation == 'create_labels' +func writeMaintenanceCreateLabelsJob(yaml *strings.Builder, ctx context.Context, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { yaml.WriteString(` create_labels: if: ${{ ` + RenderCondition(buildDispatchOperationCondition("create_labels")) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: contents: read issues: write steps: - - name: Checkout repository - uses: ` + getActionPin("actions/checkout") + ` - with: - persist-credentials: false - - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - -`) - - yaml.WriteString(generateInstallCLISteps(ctx, actionMode, version, actionTag, resolver)) - yaml.WriteString(` - name: Create missing labels - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - env: - GH_AW_CMD_PREFIX: ` + getCLICmdPrefix(actionMode) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/create_labels.cjs'); - await main(); `) + writeMaintenanceRepositoryCheckoutStep(yaml) + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "") + yaml.WriteString(generateInstallCLISteps(ctx, opts.actionMode, opts.version, opts.actionTag, opts.resolver)) + writeMaintenanceScriptStep(yaml, opts.resolver, "Create missing labels", " GH_AW_CMD_PREFIX: "+getCLICmdPrefix(opts.actionMode)+"\n", "", "create_labels") +} - // Add activity_report job for workflow_dispatch with operation == 'activity_report' +func writeMaintenanceActivityReportJob(yaml *strings.Builder, ctx context.Context, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { yaml.WriteString(` activity_report: if: ${{ ` + RenderCondition(buildDispatchOperationCondition("activity_report")) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` timeout-minutes: 120 permissions: actions: read contents: read issues: write steps: - - name: Checkout repository - uses: ` + getActionPin("actions/checkout") + ` - with: - persist-credentials: false - - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - `) + writeMaintenanceRepositoryCheckoutStep(yaml) + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "") + yaml.WriteString(generateInstallCLISteps(ctx, opts.actionMode, opts.version, opts.actionTag, opts.resolver)) + writeMaintenanceActivityReportCacheSteps(yaml, opts) + writeMaintenanceActivityReportIssueStep(yaml, opts.resolver) +} - yaml.WriteString(generateInstallCLISteps(ctx, actionMode, version, actionTag, resolver)) +func writeMaintenanceActivityReportCacheSteps(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions) { yaml.WriteString(` - name: Restore activity report logs cache id: activity_report_logs_cache uses: ` + getActionPin("actions/cache/restore") + ` @@ -499,7 +363,7 @@ jobs: shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_AW_CMD_PREFIX: ` + getCLICmdPrefix(actionMode) + ` + GH_AW_CMD_PREFIX: ` + getCLICmdPrefix(opts.actionMode) + ` run: | ${GH_AW_CMD_PREFIX} logs \ --repo "$GITHUB_REPOSITORY" \ @@ -516,7 +380,11 @@ jobs: path: ./.cache/gh-aw/activity-report-logs key: ${{ steps.activity_report_logs_cache.outputs.cache-primary-key }} - - name: Generate activity report issue +`) +} + +func writeMaintenanceActivityReportIssueStep(yaml *strings.Builder, resolver SHAResolver) { + yaml.WriteString(` - name: Generate activity report issue uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -556,41 +424,29 @@ jobs: }); core.info('Created issue #' + createdIssue.data.number + ': ' + createdIssue.data.html_url); `) +} - // Add forecast_report job for workflow_dispatch with operation == 'forecast' +func writeMaintenanceForecastReportJob(yaml *strings.Builder, ctx context.Context, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { yaml.WriteString(` forecast_report: if: ${{ ` + RenderCondition(buildDispatchOperationCondition("forecast")) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` timeout-minutes: 60 permissions: actions: read contents: read issues: write steps: - - name: Checkout repository - uses: ` + getActionPin("actions/checkout") + ` - with: - persist-credentials: false - - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - `) + writeMaintenanceRepositoryCheckoutStep(yaml) + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "") + yaml.WriteString(generateInstallCLISteps(ctx, opts.actionMode, opts.version, opts.actionTag, opts.resolver)) + writeMaintenanceForecastGenerateSteps(yaml, opts) + writeMaintenanceForecastIssueStep(yaml, opts.resolver) +} - yaml.WriteString(generateInstallCLISteps(ctx, actionMode, version, actionTag, resolver)) +func writeMaintenanceForecastGenerateSteps(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions) { yaml.WriteString(` - name: Restore forecast report logs cache id: forecast_report_logs_cache uses: ` + getActionPin("actions/cache/restore") + ` @@ -608,7 +464,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} DEBUG: "*" - GH_AW_CMD_PREFIX: ` + getCLICmdPrefix(actionMode) + ` + GH_AW_CMD_PREFIX: ` + getCLICmdPrefix(opts.actionMode) + ` run: | mkdir -p ./.cache/gh-aw/forecast set +e @@ -637,14 +493,23 @@ jobs: echo "Files under ./.github/aw/logs:" find ./.github/aw/logs -type f | sort - - name: Save forecast report logs cache +`) + writeMaintenanceForecastSaveCacheStep(yaml) +} + +func writeMaintenanceForecastSaveCacheStep(yaml *strings.Builder) { + yaml.WriteString(` - name: Save forecast report logs cache if: ${{ always() }} uses: ` + getActionPin("actions/cache/save") + ` with: path: ./.github/aw/logs key: ${{ runner.os }}-forecast-report-logs-${{ github.repository }}-${{ github.ref_name }}-${{ github.run_id }} - - name: Generate forecast issue +`) +} + +func writeMaintenanceForecastIssueStep(yaml *strings.Builder, resolver SHAResolver) { + yaml.WriteString(` - name: Generate forecast issue if: ${{ always() }} uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` env: @@ -657,57 +522,27 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/create_forecast_issue.cjs'); await main(); `) +} - // Add close_agentic_workflows_issues job for workflow_dispatch with operation == 'close_agentic_workflows_issues' +func writeMaintenanceCloseAgenticIssuesJob(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { yaml.WriteString(` close_agentic_workflows_issues: if: ${{ ` + RenderCondition(buildDispatchOperationCondition("close_agentic_workflows_issues")) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: issues: write steps: `) - - // Add checkout step only in dev/script mode (for local action paths) - if actionMode == ActionModeDev || actionMode == ActionModeScript { - yaml.WriteString(" - name: Checkout actions folder\n") - yaml.WriteString(" uses: " + getActionPin("actions/checkout") + "\n") - yaml.WriteString(" with:\n") - yaml.WriteString(" sparse-checkout: |\n") - yaml.WriteString(" actions\n") - yaml.WriteString(" clean: false\n") - yaml.WriteString(" persist-credentials: false\n\n") + if opts.actionMode == ActionModeDev || opts.actionMode == ActionModeScript { + writeMaintenanceActionsCheckoutStep(yaml) } + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "") + writeMaintenanceScriptStep(yaml, opts.resolver, "Close no-repro agentic-workflows issues", "", "", "close_agentic_workflows_issues") +} - yaml.WriteString(` - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - - - name: Close no-repro agentic-workflows issues - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/close_agentic_workflows_issues.cjs'); - await main(); -`) - - // Add validate_workflows job for workflow_dispatch with operation == 'validate' - // This job uses ubuntu-latest by default (needs full runner for CLI installation). - formattedRunsOn := FormatRunsOn(configuredRunsOn, "ubuntu-latest") +func writeMaintenanceValidateWorkflowsJob(yaml *strings.Builder, ctx context.Context, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { + formattedRunsOn := FormatRunsOn(opts.configuredRunsOn, "ubuntu-latest") yaml.WriteString(` validate_workflows: if: ${{ ` + RenderCondition(buildDispatchOperationCondition("validate")) + ` }} @@ -716,106 +551,51 @@ jobs: contents: read issues: write steps: - - name: Checkout repository - uses: ` + getActionPin("actions/checkout") + ` - with: - persist-credentials: false - - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - `) + writeMaintenanceRepositoryCheckoutStep(yaml) + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "") + yaml.WriteString(generateInstallCLISteps(ctx, opts.actionMode, opts.version, opts.actionTag, opts.resolver)) + writeMaintenanceScriptStep(yaml, opts.resolver, "Validate workflows and file issue on findings", " GH_AW_CMD_PREFIX: "+getCLICmdPrefix(opts.actionMode)+"\n", "", "run_validate_workflows") +} - yaml.WriteString(generateInstallCLISteps(ctx, actionMode, version, actionTag, resolver)) - - yaml.WriteString(` - name: Validate workflows and file issue on findings - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - env: - GH_AW_CMD_PREFIX: ` + getCLICmdPrefix(actionMode) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/run_validate_workflows.cjs'); - await main(); -`) +func writeMaintenanceLabelJobs(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string, labelDisableJobEnabled, labelApplySafeOutputsJobEnabled bool) { + if !labelDisableJobEnabled && !labelApplySafeOutputsJobEnabled { + return + } + maintenanceWorkflowYAMLLog.Print("Adding label-triggered jobs") + if labelDisableJobEnabled { + writeMaintenanceLabelDisableJob(yaml, opts, setupActionRef) + } + if labelApplySafeOutputsJobEnabled { + writeMaintenanceLabelApplySafeOutputsJob(yaml, opts, setupActionRef) + } +} - // Add label_disable_agentic_workflow job triggered by label "agentic-workflows:disable" on issues. - // This job reads the body of the labeled issue to extract the workflow_id from XML comment - // markers, disables the corresponding agentic workflow via the GitHub REST API, and posts - // a confirmation comment. - // Skipped when label_triggers is set to false in aw.json maintenance config. - if labelDisableJobEnabled || labelApplySafeOutputsJobEnabled { - maintenanceWorkflowYAMLLog.Print("Adding label-triggered jobs") - if labelDisableJobEnabled { - disableLabelCondition := buildLabeledDisableCondition() - yaml.WriteString(` +func writeMaintenanceLabelDisableJob(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { + disableLabelCondition := buildLabeledDisableCondition() + yaml.WriteString(` label_disable_agentic_workflow: if: ${{ ` + RenderCondition(disableLabelCondition) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: actions: write contents: read issues: write steps: - - name: Checkout actions folder - uses: ` + getActionPin("actions/checkout") + ` - with: - sparse-checkout: | - actions - clean: false - persist-credentials: false - - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - id: check_permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - - - name: Disable agentic workflow - if: ${{ steps.check_permissions.outcome == 'success' }} - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/disable_agentic_workflow.cjs'); - await main(); `) - } + writeMaintenanceActionsCheckoutStep(yaml) + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "check_permissions") + writeMaintenanceScriptStep(yaml, opts.resolver, "Disable agentic workflow", "", " if: ${{ steps.check_permissions.outcome == 'success' }}\n", "disable_agentic_workflow") +} - // Add label_apply_safe_outputs job triggered by "agentic-workflows:apply-safe-outputs" label on issues. - // This job extracts a workflow run URL from the issue body XML comments and re-applies the safe outputs. - if labelApplySafeOutputsJobEnabled { - applySafeOutputsCondition := buildLabeledApplySafeOutputsCondition() - yaml.WriteString(` +func writeMaintenanceLabelApplySafeOutputsJob(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { + applySafeOutputsCondition := buildLabeledApplySafeOutputsCondition() + yaml.WriteString(` label_apply_safe_outputs: if: ${{ ` + RenderCondition(applySafeOutputsCondition) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: actions: read contents: write @@ -823,56 +603,24 @@ jobs: issues: write pull-requests: write steps: - - name: Checkout actions folder - uses: ` + getActionPin("actions/checkout") + ` - with: - sparse-checkout: | - actions - clean: false - persist-credentials: false - - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check admin/maintainer permissions - id: check_permissions - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); - await main(); - - - name: Apply safe outputs from referenced run - if: ${{ steps.check_permissions.outcome == 'success' }} - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/label_apply_safe_outputs.cjs'); - await main(); `) - } - } + writeMaintenanceActionsCheckoutStep(yaml) + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceAdminCheckStep(yaml, opts.resolver, "check_permissions") + writeMaintenanceScriptStep(yaml, opts.resolver, "Apply safe outputs from referenced run", " GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n", " if: ${{ steps.check_permissions.outcome == 'success' }}\n", "label_apply_safe_outputs") +} + +func writeMaintenanceDevJobs(yaml *strings.Builder, ctx context.Context, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { + maintenanceWorkflowYAMLLog.Printf("Adding dev-only jobs: compile-workflows and secret-validation") + writeMaintenanceCompileWorkflowsJob(yaml, ctx, opts, setupActionRef) + writeMaintenanceSecretValidationJob(yaml, opts, setupActionRef) +} - // Add compile-workflows and zizmor-scan jobs only in dev mode - // These jobs are specific to the gh-aw repository and require go.mod, make build, etc. - // User repositories won't have these dependencies, so we skip them in release mode - if actionMode == ActionModeDev { - maintenanceWorkflowYAMLLog.Printf("Adding dev-only jobs: compile-workflows and secret-validation") - // Add compile-workflows job - yaml.WriteString(` +func writeMaintenanceCompileWorkflowsJob(yaml *strings.Builder, ctx context.Context, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { + yaml.WriteString(` compile-workflows: if: ${{ ` + RenderCondition(buildNotForkAndScheduled()) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` concurrency: group: ${{ github.workflow }}-compile-workflows-${{ github.repository }} cancel-in-progress: true @@ -881,91 +629,75 @@ jobs: issues: write steps: `) + writeMaintenanceDevRepositoryCheckoutStep(yaml) + yaml.WriteString(generateInstallCLISteps(ctx, opts.actionMode, opts.version, opts.actionTag, opts.resolver)) + writeMaintenanceCompileWorkflowsSteps(yaml, opts, setupActionRef) +} - // Dev mode: checkout entire repository (no sparse checkout, but no credentials) - yaml.WriteString(` - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - -`) - - yaml.WriteString(generateInstallCLISteps(ctx, actionMode, version, actionTag, resolver)) - yaml.WriteString(` - name: Pre-compile validation +func writeMaintenanceCompileWorkflowsSteps(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { + yaml.WriteString(` - name: Pre-compile validation run: | - ` + getCLICmdPrefix(actionMode) + ` compile --validate --no-emit --verbose + ` + getCLICmdPrefix(opts.actionMode) + ` compile --validate --no-emit --verbose echo "✓ Pre-compile validation passed" - name: Compile workflows run: | - ` + getCLICmdPrefix(actionMode) + ` compile --validate --verbose + ` + getCLICmdPrefix(opts.actionMode) + ` compile --validate --verbose echo "✓ All workflows compiled successfully" - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - - - name: Check for out-of-sync workflows and create issue or pull request if needed - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` `) - if compileGitHubToken != "" { - yaml.WriteString(` env: - GH_AW_MAINTENANCE_GITHUB_TOKEN: ` + compileGitHubToken + ` + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + yaml.WriteString(` - name: Check for out-of-sync workflows and create issue or pull request if needed + uses: ` + getCachedActionPinFromResolver("actions/github-script", opts.resolver) + ` `) - } - yaml.WriteString(` with: + if opts.compileGitHubToken != "" { + yaml.WriteString(` env: + GH_AW_MAINTENANCE_GITHUB_TOKEN: ` + opts.compileGitHubToken + ` +`) + } + yaml.WriteString(` with: `) - if compileGitHubToken != "" { - yaml.WriteString(` github-token: ${{ env.GH_AW_MAINTENANCE_GITHUB_TOKEN }} + if opts.compileGitHubToken != "" { + yaml.WriteString(` github-token: ${{ env.GH_AW_MAINTENANCE_GITHUB_TOKEN }} `) - } - yaml.WriteString(` script: | + } + yaml.WriteString(` script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io, getOctokit); const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_recompile_needed.cjs'); await main(); +`) +} +func writeMaintenanceSecretValidationJob(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions, setupActionRef string) { + yaml.WriteString(` secret-validation: if: ${{ ` + RenderCondition(buildNotForkAndScheduleOnly()) + ` }} - runs-on: ` + runsOnValue + ` + runs-on: ` + opts.runsOnValue + ` permissions: contents: read steps: `) - - // Add checkout step only in dev mode (for local action paths) - yaml.WriteString(` - name: Checkout actions folder - uses: ` + getActionPin("actions/checkout") + ` - with: - sparse-checkout: | - actions - clean: false - persist-credentials: false - -`) - - yaml.WriteString(` - name: Setup Node.js + writeMaintenanceActionsCheckoutStep(yaml) + yaml.WriteString(` - name: Setup Node.js uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 with: node-version: '22' - - name: Setup Scripts - uses: ` + setupActionRef + ` - with: - destination: ${{ runner.temp }}/gh-aw/actions - `) - // Build the Validate Secrets step, conditionally including the org billing flag. - // The line uses 10-space indentation to match the surrounding env block structure. - copilotOrgBillingLine := "" - if copilotOrgBilling { - maintenanceWorkflowYAMLLog.Print("Copilot org billing mode detected: adding GH_AW_COPILOT_ORG_BILLING=true to secret-validation step") - copilotOrgBillingLine = ` GH_AW_COPILOT_ORG_BILLING: "true" + writeMaintenanceSetupScriptsStep(yaml, setupActionRef) + writeMaintenanceValidateSecretsStep(yaml, opts) +} + +func writeMaintenanceValidateSecretsStep(yaml *strings.Builder, opts buildMaintenanceWorkflowYAMLOptions) { + copilotOrgBillingLine := "" + if opts.copilotOrgBilling { + maintenanceWorkflowYAMLLog.Print("Copilot org billing mode detected: adding GH_AW_COPILOT_ORG_BILLING=true to secret-validation step") + copilotOrgBillingLine = ` GH_AW_COPILOT_ORG_BILLING: "true" ` - } - yaml.WriteString(` - name: Validate Secrets - uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` + } + yaml.WriteString(` - name: Validate Secrets + uses: ` + getCachedActionPinFromResolver("actions/github-script", opts.resolver) + ` env: # GitHub tokens GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} @@ -993,8 +725,84 @@ jobs: path: secret-validation-report.md retention-days: 30 if-no-files-found: warn +`) +} + +func writeMaintenanceActionsCheckoutStep(yaml *strings.Builder) { + yaml.WriteString(` - name: Checkout actions folder + uses: ` + getActionPin("actions/checkout") + ` + with: + sparse-checkout: | + actions + clean: false + persist-credentials: false + +`) +} + +func writeMaintenanceRepositoryCheckoutStep(yaml *strings.Builder) { + yaml.WriteString(` - name: Checkout repository + uses: ` + getActionPin("actions/checkout") + ` + with: + persist-credentials: false + +`) +} + +func writeMaintenanceDevRepositoryCheckoutStep(yaml *strings.Builder) { + yaml.WriteString(` - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + +`) +} + +func writeMaintenanceSetupScriptsStep(yaml *strings.Builder, setupActionRef string) { + yaml.WriteString(` - name: Setup Scripts + uses: ` + setupActionRef + ` + with: + destination: ${{ runner.temp }}/gh-aw/actions + +`) +} + +func writeMaintenanceAdminCheckStep(yaml *strings.Builder, resolver SHAResolver, id string) { + yaml.WriteString(` - name: Check admin/maintainer permissions +`) + if id != "" { + yaml.WriteString(` id: ` + id + ` `) } + yaml.WriteString(` uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/check_team_member.cjs'); + await main(); - return yaml.String() +`) +} + +func writeMaintenanceScriptStep(yaml *strings.Builder, resolver SHAResolver, name, envLines, ifLine, scriptName string) { + yaml.WriteString(` - name: ` + name + ` +`) + if ifLine != "" { + yaml.WriteString(ifLine) + } + yaml.WriteString(` uses: ` + getCachedActionPinFromResolver("actions/github-script", resolver) + ` +`) + if envLines != "" { + yaml.WriteString(" env:\n" + envLines) + } + yaml.WriteString(` with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/` + scriptName + `.cjs'); + await main(); +`) } diff --git a/pkg/workflow/markdown_security_scanner.go b/pkg/workflow/markdown_security_scanner.go index d8d119eb69e..fcee6f22d38 100644 --- a/pkg/workflow/markdown_security_scanner.go +++ b/pkg/workflow/markdown_security_scanner.go @@ -226,70 +226,53 @@ func scanUnicodeAbuse(content string) []SecurityFinding { markdownSecurityLog.Printf("Scanning %d line(s) for unicode abuse", len(lines)) for lineNum, line := range lines { - lineNo := lineNum + 1 + findings = append(findings, scanUnicodeAbuseLine(line, lineNum+1)...) + } - // Check for zero-width and invisible characters - var prevRune rune - for i := 0; i < len(line); { - r, size := utf8.DecodeRuneInString(line[i:]) - if r == utf8.RuneError && size <= 1 { - i++ - continue - } + return findings +} - if name, ok := dangerousUnicodeRunes[r]; ok { - // U+200D (ZWJ) is a standard component of emoji sequences such as - // 🧑‍🤝‍🧑 (people holding hands) or 👨‍👩‍👧 (family). Only flag it when - // it is NOT flanked by emoji-range codepoints on both sides. - // prevRune is 0 (null) at the start of each line, so a ZWJ at - // the beginning of a line is always flagged (isEmojiLike(0)==false). - if r == '\u200D' { - var nextRune rune - if i+size < len(line) { - nextRune, _ = utf8.DecodeRuneInString(line[i+size:]) - } - if isEmojiLike(prevRune) && isEmojiLike(nextRune) { - prevRune = r - i += size - continue - } - } - findings = append(findings, SecurityFinding{ - Category: CategoryUnicodeAbuse, - Description: "contains invisible character: " + name, - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(line), 80), - }) - } +func scanUnicodeAbuseLine(line string, lineNo int) []SecurityFinding { + var findings []SecurityFinding + var prevRune rune + for i := 0; i < len(line); { + r, size := utf8.DecodeRuneInString(line[i:]) + if r == utf8.RuneError && size <= 1 { + i++ + continue + } - if name, ok := bidiOverrideRunes[r]; ok { - findings = append(findings, SecurityFinding{ - Category: CategoryUnicodeAbuse, - Description: "contains bidirectional override character: " + name, - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(line), 80), - }) - } + if finding, ok := unicodeInvisibleFinding(line, lineNo, r, prevRune, line[i+size:]); ok { + findings = append(findings, finding) + } + if name, ok := bidiOverrideRunes[r]; ok { + findings = append(findings, newSecurityFinding(CategoryUnicodeAbuse, "contains bidirectional override character: "+name, lineNo, line, 80)) + } + if unicode.IsControl(r) && r != '\n' && r != '\r' && r != '\t' && r != '\uFEFF' { + findings = append(findings, newSecurityFinding(CategoryUnicodeAbuse, fmt.Sprintf("contains control character U+%04X", r), lineNo, line, 80)) + } - // Check for C0/C1 control characters (except common whitespace) - if unicode.IsControl(r) && r != '\n' && r != '\r' && r != '\t' { - // Skip BOM which is already handled above - if r != '\uFEFF' { - findings = append(findings, SecurityFinding{ - Category: CategoryUnicodeAbuse, - Description: fmt.Sprintf("contains control character U+%04X", r), - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(line), 80), - }) - } - } + prevRune = r + i += size + } + return findings +} - prevRune = r - i += size +func unicodeInvisibleFinding(line string, lineNo int, r rune, prevRune rune, next string) (SecurityFinding, bool) { + name, ok := dangerousUnicodeRunes[r] + if !ok { + return SecurityFinding{}, false + } + if r == '\u200D' { + var nextRune rune + if next != "" { + nextRune, _ = utf8.DecodeRuneInString(next) + } + if isEmojiLike(prevRune) && isEmojiLike(nextRune) { + return SecurityFinding{}, false } } - - return findings + return newSecurityFinding(CategoryUnicodeAbuse, "contains invisible character: "+name, lineNo, line, 80), true } // --- Hidden Content Detection --- @@ -423,92 +406,58 @@ func scanObfuscatedLinks(content string) []SecurityFinding { for lineNum, line := range lines { lineNo := lineNum + 1 - // Check markdown links linkMatches := markdownLinkPattern.FindAllStringSubmatch(line, -1) for _, m := range linkMatches { - linkURL := m[2] - - // Check for data URIs - if dataURIPattern.MatchString(linkURL) { - findings = append(findings, SecurityFinding{ - Category: CategoryObfuscatedLinks, - Description: "markdown link uses a data: URI which can embed executable content", - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(m[0]), 80), - }) - } - - // Check for multiple URL encoding - if multipleEncodingPattern.MatchString(linkURL) { - findings = append(findings, SecurityFinding{ - Category: CategoryObfuscatedLinks, - Description: "markdown link URL is multiply-encoded (possible obfuscation)", - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(m[0]), 80), - }) - } - - // Check for IP address URLs - if ipAddressURLPattern.MatchString(linkURL) { - findings = append(findings, SecurityFinding{ - Category: CategoryObfuscatedLinks, - Description: "markdown link points to an IP address instead of a domain name", - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(m[0]), 80), - }) - } - - // Check for URL shorteners - if urlShortenerPattern.MatchString(linkURL) { - findings = append(findings, SecurityFinding{ - Category: CategoryObfuscatedLinks, - Description: "markdown link uses a URL shortener which hides the true destination", - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(m[0]), 80), - }) - } - - // Check for suspicious query parameters - if suspiciousQueryParamPattern.MatchString(linkURL) { - findings = append(findings, SecurityFinding{ - Category: CategoryObfuscatedLinks, - Description: "markdown link URL contains suspicious authentication parameters (token, key, secret)", - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(m[0]), 80), - }) - } - - // Check for javascript:, vbscript:, or data: protocols - lowerURL := strings.ToLower(strings.TrimSpace(linkURL)) - if strings.HasPrefix(lowerURL, "javascript:") || strings.HasPrefix(lowerURL, "vbscript:") || strings.HasPrefix(lowerURL, "data:") { - findings = append(findings, SecurityFinding{ - Category: CategoryObfuscatedLinks, - Description: "markdown link uses dangerous protocol: " + strings.SplitN(lowerURL, ":", 2)[0], - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(m[0]), 80), - }) - } + findings = append(findings, scanMarkdownLinkURL(m[0], m[2], lineNo)...) } - // Check markdown image links for data URIs imageMatches := markdownImagePattern.FindAllStringSubmatch(line, -1) for _, m := range imageMatches { - imageURL := m[2] - - if dataURIPattern.MatchString(imageURL) { - findings = append(findings, SecurityFinding{ - Category: CategoryObfuscatedLinks, - Description: "markdown image uses a data: URI which can embed executable content", - Line: lineNo, - Snippet: stringutil.Truncate(strings.TrimSpace(m[0]), 80), - }) - } + findings = append(findings, scanMarkdownImageURL(m[0], m[2], lineNo)...) } } return findings } +func scanMarkdownLinkURL(snippet, linkURL string, lineNo int) []SecurityFinding { + checks := []struct { + matches func(string) bool + desc string + }{ + {dataURIPattern.MatchString, "markdown link uses a data: URI which can embed executable content"}, + {multipleEncodingPattern.MatchString, "markdown link URL is multiply-encoded (possible obfuscation)"}, + {ipAddressURLPattern.MatchString, "markdown link points to an IP address instead of a domain name"}, + {urlShortenerPattern.MatchString, "markdown link uses a URL shortener which hides the true destination"}, + {suspiciousQueryParamPattern.MatchString, "markdown link URL contains suspicious authentication parameters (token, key, secret)"}, + } + var findings []SecurityFinding + for _, check := range checks { + if check.matches(linkURL) { + findings = append(findings, newSecurityFinding(CategoryObfuscatedLinks, check.desc, lineNo, snippet, 80)) + } + } + if protocol, ok := dangerousLinkProtocol(linkURL); ok { + findings = append(findings, newSecurityFinding(CategoryObfuscatedLinks, "markdown link uses dangerous protocol: "+protocol, lineNo, snippet, 80)) + } + return findings +} + +func scanMarkdownImageURL(snippet, imageURL string, lineNo int) []SecurityFinding { + if !dataURIPattern.MatchString(imageURL) { + return nil + } + return []SecurityFinding{newSecurityFinding(CategoryObfuscatedLinks, "markdown image uses a data: URI which can embed executable content", lineNo, snippet, 80)} +} + +func dangerousLinkProtocol(linkURL string) (string, bool) { + lowerURL := strings.ToLower(strings.TrimSpace(linkURL)) + if strings.HasPrefix(lowerURL, "javascript:") || strings.HasPrefix(lowerURL, "vbscript:") || strings.HasPrefix(lowerURL, "data:") { + return strings.SplitN(lowerURL, ":", 2)[0], true + } + return "", false +} + // --- HTML Abuse Detection --- var ( @@ -555,52 +504,34 @@ func scanHTMLAbuse(content string) []SecurityFinding { continue } - // Check for dangerous HTML elements - htmlChecks := []struct { - pattern *regexp.Regexp - desc string - }{ - {scriptTagPattern, "