From 115deb1fa1a99d933631d2b9cec26a4f4dff684c Mon Sep 17 00:00:00 2001 From: Shyam Kulkarni Date: Mon, 10 Aug 2026 23:01:33 +0530 Subject: [PATCH 1/2] feat(mcp): Add ECS instance diagnostics MCP --- mcp/ecs-instance-log-mcp/.gitignore | 44 + .../.kiro/specs/ecs-lambda-rewrite/spec.md | 54 + mcp/ecs-instance-log-mcp/LICENSE | 16 + mcp/ecs-instance-log-mcp/README.md | 389 + mcp/ecs-instance-log-mcp/bin/app.ts | 54 + mcp/ecs-instance-log-mcp/cdk.json | 22 + mcp/ecs-instance-log-mcp/deploy.sh | 604 ++ mcp/ecs-instance-log-mcp/docs/ARCHITECTURE.md | 308 + mcp/ecs-instance-log-mcp/get-config.sh | 52 + mcp/ecs-instance-log-mcp/package-lock.json | 470 ++ mcp/ecs-instance-log-mcp/package.json | 37 + mcp/ecs-instance-log-mcp/requirements-dev.txt | 4 + .../runbooks/A1-task-startup-resource-init.md | 78 + .../A2-task-startup-container-runtime.md | 52 + .../sops/runbooks/B1-ecr-image-pull-auth.md | 53 + .../sops/runbooks/B2-image-not-found.md | 48 + .../sops/runbooks/B3-docker-hub-rate-limit.md | 42 + .../C1-task-execution-role-permissions.md | 53 + .../runbooks/C2-secrets-manager-retrieval.md | 50 + .../sops/runbooks/D1-oom-kill-memory.md | 53 + .../sops/runbooks/D2-disk-space-exhaustion.md | 41 + .../sops/runbooks/D3-cpu-throttling.md | 40 + .../runbooks/E1-eni-allocation-subnet-ip.md | 49 + .../runbooks/E2-dns-resolution-failures.md | 44 + .../sops/runbooks/E3-connection-timeout.md | 45 + .../E4-human-approved-task-tcpdump.md | 55 + .../runbooks/F1-container-health-check.md | 47 + .../sops/runbooks/F2-elb-target-health.md | 43 + .../sops/runbooks/G1-agent-disconnected.md | 52 + .../G2-instance-registration-failure.md | 49 + .../sops/runbooks/H1-cloudwatch-log-driver.md | 49 + .../runbooks/I1-deployment-circuit-breaker.md | 48 + .../runbooks/J1-oci-runtime-entrypoint.md | 53 + .../K1-spot-interruption-instance-draining.md | 81 + .../K10-api-throttling-service-quotas.md | 86 + .../K11-fargate-metadata-credential-errors.md | 87 + .../K12-essential-container-exited-nonzero.md | 96 + .../runbooks/K13-windows-container-issues.md | 91 + .../runbooks/K14-task-latency-performance.md | 94 + .../K15-docker-daemon-agent-errors.md | 88 + .../runbooks/K2-task-placement-failures.md | 90 + .../K3-service-steady-state-failures.md | 87 + .../K4-service-auto-scaling-issues.md | 87 + .../sops/runbooks/K5-ecs-exec-failures.md | 83 + .../K6-service-connect-discovery-failures.md | 90 + .../K7-ebs-efs-volume-mount-failures.md | 86 + .../sops/runbooks/K8-task-stuck-pending.md | 84 + .../K9-fargate-platform-ephemeral-storage.md | 88 + .../runbooks/Z1-general-troubleshooting.md | 55 + .../src/ecs-log-gateway-construct-v2.ts | 1784 +++++ .../src/ecs-log-gateway-stack-v2.ts | 31 + mcp/ecs-instance-log-mcp/src/index.ts | 2 + .../src/lambda/ecs-log-automation.py | 6612 +++++++++++++++++ mcp/ecs-instance-log-mcp/tests/__init__.py | 0 mcp/ecs-instance-log-mcp/tests/conftest.py | 13 + .../tests/test_batch_approval.py | 201 + .../tests/test_collection_approval.py | 140 + .../tests/test_instance_validation.py | 144 + .../tests/test_region_validation.py | 111 + .../tests/test_security_parity.py | 171 + .../tests/test_tcpdump_approval.py | 184 + .../tests/test_tcpdump_security.py | 67 + .../tests/test_tool_validation_wiring.py | 142 + mcp/ecs-instance-log-mcp/tsconfig.json | 24 + 64 files changed, 14097 insertions(+) create mode 100644 mcp/ecs-instance-log-mcp/.gitignore create mode 100644 mcp/ecs-instance-log-mcp/.kiro/specs/ecs-lambda-rewrite/spec.md create mode 100644 mcp/ecs-instance-log-mcp/LICENSE create mode 100644 mcp/ecs-instance-log-mcp/README.md create mode 100644 mcp/ecs-instance-log-mcp/bin/app.ts create mode 100644 mcp/ecs-instance-log-mcp/cdk.json create mode 100755 mcp/ecs-instance-log-mcp/deploy.sh create mode 100644 mcp/ecs-instance-log-mcp/docs/ARCHITECTURE.md create mode 100755 mcp/ecs-instance-log-mcp/get-config.sh create mode 100644 mcp/ecs-instance-log-mcp/package-lock.json create mode 100644 mcp/ecs-instance-log-mcp/package.json create mode 100644 mcp/ecs-instance-log-mcp/requirements-dev.txt create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/A1-task-startup-resource-init.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/A2-task-startup-container-runtime.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/B1-ecr-image-pull-auth.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/B2-image-not-found.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/B3-docker-hub-rate-limit.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/C1-task-execution-role-permissions.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/C2-secrets-manager-retrieval.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/D1-oom-kill-memory.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/D2-disk-space-exhaustion.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/D3-cpu-throttling.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/E1-eni-allocation-subnet-ip.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/E2-dns-resolution-failures.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/E3-connection-timeout.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/E4-human-approved-task-tcpdump.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/F1-container-health-check.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/F2-elb-target-health.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/G1-agent-disconnected.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/G2-instance-registration-failure.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/H1-cloudwatch-log-driver.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/I1-deployment-circuit-breaker.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/J1-oci-runtime-entrypoint.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K1-spot-interruption-instance-draining.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K10-api-throttling-service-quotas.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K11-fargate-metadata-credential-errors.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K12-essential-container-exited-nonzero.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K13-windows-container-issues.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K14-task-latency-performance.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K15-docker-daemon-agent-errors.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K2-task-placement-failures.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K3-service-steady-state-failures.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K4-service-auto-scaling-issues.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K5-ecs-exec-failures.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K6-service-connect-discovery-failures.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K7-ebs-efs-volume-mount-failures.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K8-task-stuck-pending.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/K9-fargate-platform-ephemeral-storage.md create mode 100644 mcp/ecs-instance-log-mcp/sops/runbooks/Z1-general-troubleshooting.md create mode 100644 mcp/ecs-instance-log-mcp/src/ecs-log-gateway-construct-v2.ts create mode 100644 mcp/ecs-instance-log-mcp/src/ecs-log-gateway-stack-v2.ts create mode 100644 mcp/ecs-instance-log-mcp/src/index.ts create mode 100644 mcp/ecs-instance-log-mcp/src/lambda/ecs-log-automation.py create mode 100644 mcp/ecs-instance-log-mcp/tests/__init__.py create mode 100644 mcp/ecs-instance-log-mcp/tests/conftest.py create mode 100644 mcp/ecs-instance-log-mcp/tests/test_batch_approval.py create mode 100644 mcp/ecs-instance-log-mcp/tests/test_collection_approval.py create mode 100644 mcp/ecs-instance-log-mcp/tests/test_instance_validation.py create mode 100644 mcp/ecs-instance-log-mcp/tests/test_region_validation.py create mode 100644 mcp/ecs-instance-log-mcp/tests/test_security_parity.py create mode 100644 mcp/ecs-instance-log-mcp/tests/test_tcpdump_approval.py create mode 100644 mcp/ecs-instance-log-mcp/tests/test_tcpdump_security.py create mode 100644 mcp/ecs-instance-log-mcp/tests/test_tool_validation_wiring.py create mode 100644 mcp/ecs-instance-log-mcp/tsconfig.json diff --git a/mcp/ecs-instance-log-mcp/.gitignore b/mcp/ecs-instance-log-mcp/.gitignore new file mode 100644 index 0000000..9b3f8d3 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/.gitignore @@ -0,0 +1,44 @@ +# Dependencies +node_modules/ + +# Build output +lib/ +*.js +*.d.ts + +# CDK output +cdk.out/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log +npm-debug.log* + +# Test coverage and generated test state +coverage/ +.hypothesis/ +.pytest_cache/ + +# Python cache +__pycache__/ +*.pyc +.pytest_cache/ + +# Hypothesis test framework cache +.hypothesis/ + +# Deployment-specific outputs (contain account IDs, secrets) +mcp-config.txt +cdk-outputs.json + +# Lambda packaging +lambda.zip diff --git a/mcp/ecs-instance-log-mcp/.kiro/specs/ecs-lambda-rewrite/spec.md b/mcp/ecs-instance-log-mcp/.kiro/specs/ecs-lambda-rewrite/spec.md new file mode 100644 index 0000000..0cfd4be --- /dev/null +++ b/mcp/ecs-instance-log-mcp/.kiro/specs/ecs-lambda-rewrite/spec.md @@ -0,0 +1,54 @@ +# ECS Lambda Rewrite Spec + +## Goal +Rewrite `src/lambda/ecs-log-automation.py` from line 451 onwards to match EKS gold standard patterns. +Keep lines 1-449 (ECS_ERROR_PATTERNS, ECS_TRIAGE_CATEGORIES, ECS_LOG_TYPE_PATTERNS) exactly as-is. + +## Tasks (in order) + +### Task 1: Foundation Layer (lines 451-800) +- Compiled error patterns (pre-compile ECS_ERROR_PATTERNS regexes) +- Response helpers: success_response (5.5MB guard), error_response +- S3 safe helpers: safe_s3_read, safe_s3_head, safe_s3_list +- Utility helpers: get_regional_client, detect_instance_region, resolve_region +- Severity class, normalize_severity_filter, assign_finding_id +- format_bytes, parse_failure_reason (RunCommand), estimate_progress (RunCommand) +- Idempotency: store_execution_region, get_execution_region, find_execution_by_idempotency_token, store_idempotency_mapping + +### Task 2: Scan & Analysis Layer (lines 800-1200) +- Baseline helpers: load_baselines, update_baselines, annotate_findings_with_baselines +- Scan helpers: find_findings_index, scan_and_index_errors, scan_file_for_errors (false positive suppression, multi-signal) +- Read/search: read_by_lines, search_file_for_pattern (chunked), get_line_context, extract_timestamp +- categorize_log_source (ECS-specific), find_correlations (ECS-specific), generate_recommendations (ECS-specific) +- Triage: perform_ecs_triage (using ECS_TRIAGE_CATEGORIES) +- Temporal: _build_temporal_clusters, _build_root_cause_chain (ECS causal patterns) + +### Task 3: Core Tool Handlers (lines 1200-1800) +- lambda_handler with routing map (15 short names matching construct) +- collect (start_log_collection) - idempotency, cross-region, embedded bash script (PRESERVE existing) +- status (get_collection_status) - RunCommand APIs, progress estimation +- validate (validate_bundle_completeness) - manifest.json support +- errors (get_error_summary) - pre-indexed findings, pagination, baseline subtraction +- read (read_log_chunk) - byte-range + line-based, line-aligned +- search (search_logs_deep) - regex with S-NNN finding_ids, chunked +- correlate (correlate_events) - temporal clusters, root cause chains + +### Task 4: Advanced Tool Handlers (lines 1800-2400) +- artifact (get_artifact_reference) - presigned URLs +- summarize (generate_incident_summary) - grounded in finding_ids, triage +- history (list_collection_history) - cross-region S3 listing +- cluster_health - ECS cluster overview via ecs:ListContainerInstances + DescribeContainerInstances +- compare_instances - diff findings between instances +- batch_collect - smart batch with sampling, filter unhealthy/disconnected +- batch_status - poll multiple executions +- network_diagnostics - ECS-specific sections + +## Key ECS Differences from EKS +- SSM: start_automation_execution with AWSSupport-CollectECSInstanceLogs (same pattern as EKS) +- Parameters: ECSInstanceId, LogDestination, AutomationAssumeRole +- S3 prefix: ecs_{instance_id} +- Log types: ecs-agent, docker, containerd, system, kernel, networking, cgroups, metadata +- cluster_health uses ecs:ListContainerInstances + DescribeContainerInstances +- batch_collect filters: unhealthy/disconnected (not notready) +- network_diagnostics sections: iptables,docker,routes,dns,eni,security-groups +- Tool routing uses short names: collect, status, validate, errors, read, search, correlate, artifact, summarize, history, cluster_health, compare_instances, batch_collect, batch_status, network_diagnostics diff --git a/mcp/ecs-instance-log-mcp/LICENSE b/mcp/ecs-instance-log-mcp/LICENSE new file mode 100644 index 0000000..56a66b6 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/LICENSE @@ -0,0 +1,16 @@ +MIT No Attribution + +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/mcp/ecs-instance-log-mcp/README.md b/mcp/ecs-instance-log-mcp/README.md new file mode 100644 index 0000000..23e0cce --- /dev/null +++ b/mcp/ecs-instance-log-mcp/README.md @@ -0,0 +1,389 @@ +# ECS Instance Log MCP + +> **⚠️ Proof of Concept (POC):** This project is a proof of concept and should be tested in non-production environments first. Validate thoroughly in a staging or development account before using with production workloads. + +MCP Server for AWS DevOps Agent to collect and analyze diagnostic logs from ECS container instances using SSM Automation. Covers ECS agent, Docker/containerd, container logs, system logs, dmesg, networking, cgroups, instance metadata, and GPU diagnostics — artifacts that live on the instance OS and aren't accessible through the ECS API or CloudWatch. + +> **Want to understand the internals?** See [Architecture & Design](docs/ARCHITECTURE.md) for a deep dive into how the components work, data flows, tool design, and security model. + +--- + +## Prerequisites + +### 1. Node.js (v18.x or later) + +**macOS (Homebrew):** +```bash +brew install node +``` + +**Linux (Ubuntu/Debian):** +```bash +curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - +sudo apt-get install -y nodejs +``` + +### 2. AWS CLI v2 + +**macOS:** +```bash +brew install awscli +``` + +**Linux:** +```bash +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip awscliv2.zip +sudo ./aws/install +``` + +### 3. AWS CDK CLI + +```bash +npm install -g aws-cdk +``` + +### 4. Python 3 + +Most systems have it pre-installed: +```bash +python3 --version +``` + +### 5. AWS Credentials + +You need permissions to create IAM Roles, Lambda Functions, S3 Buckets, KMS Keys, Cognito User Pools, and BedrockAgentCore Gateways. + +```bash +aws configure +# Or use AWS SSO: +aws sso login --profile your-profile +export AWS_PROFILE=your-profile +``` + +--- + +## Deployment + +```bash +# Clone the repository +git clone +cd ecs-instance-log-mcp + +# Make the script executable +chmod +x deploy.sh + +# Deploy (defaults to us-east-1 and stack EcsInstanceLogMcpStack) +./deploy.sh + +# Update a specifically named existing stack +./deploy.sh EcsLogGatewayStack + +# Or deploy to a specific region +AWS_REGION=us-west-2 ./deploy.sh +``` + +### Interactive Deployment Flow + +The deploy script walks you through three interactive prompts: + +**Step 1 — Region selection:** +``` +Which AWS regions should be scanned for ECS clusters? + + 1) All enabled regions + 2) Current deploy region only (us-east-1) + 3) Enter a specific region + +Select [1/2/3] (default: 1): +``` + +**Step 2 — Cluster selection:** +``` +Found 4 ECS cluster(s): + + 1) prod-cluster (us-east-1) + 2) dev-cluster (us-east-1) + 3) analytics (us-west-2) + 4) eu-cluster (eu-west-1) + + a) All clusters + +Select clusters (comma-separated numbers, or 'a' for all) [default: a]: +``` + +**Step 3 — Instance role selection:** +``` +Found 3 unique container instance role(s): + + 1) arn:aws:iam::123456789012:role/ecsInstanceRole + └─ ecsInstanceRole (prod-cluster / us-east-1) + 2) arn:aws:iam::123456789012:role/ecs-dev-role + └─ ecs-dev-role (dev-cluster / us-east-1) + 3) arn:aws:iam::123456789012:role/ecs-eu-role + └─ ecs-eu-role (eu-cluster / eu-west-1) + + a) All roles + +Select instance roles (comma-separated numbers, or 'a' for all) [default: a]: +``` + +**Fail-closed discovery behavior:** + +If no ECS clusters are found, deployment stops because a nonempty exact cluster allowlist is mandatory. If selected clusters contain no EC2 container instances (for example, they are Fargate-only), the script permits manual entry of explicit ECS instance-role ARNs; it never grants account-wide upload access. + +### Non-Interactive / CI Mode + +Skip all prompts by providing the mandatory cluster, region, and role allowlists: + +```bash +export ALLOWED_CLUSTER_NAMES="prod-cluster" +export ALLOWED_REGIONS="us-east-1" +export ECS_INSTANCE_ROLE_ARNS="arn:aws:iam::123456789012:role/ecsInstanceRole" +./deploy.sh + +# Multiple values are comma-separated. +ALLOWED_CLUSTER_NAMES="prod-cluster,dev-cluster" \ +ALLOWED_REGIONS="us-east-1,us-west-2" \ +ECS_INSTANCE_ROLE_ARNS="arn:aws:iam::123456789012:role/Role1,arn:aws:iam::123456789012:role/Role2" \ +./deploy.sh +``` + +Deployment fails closed when any allowlist is empty. There is no account-scoped upload compatibility fallback. + +### Human approval and restricted tools + +Native Systems Manager approval is enabled by default. CDK synthesis fails closed unless `APPROVAL_APPROVER_ARNS` contains at least one IAM user or role ARN. The Automation documents embed the deployment-owned role, approvers, and SNS topic; callers cannot replace those values. Approvers need `ssm:SendAutomationSignal`. Email subscribers must confirm the SNS subscription before notifications are delivered. + +```bash +export APPROVAL_APPROVER_ARNS="arn:aws:iam::123456789012:role/EcsDiagnosticsApprover" +export APPROVAL_NOTIFICATION_EMAILS="oncall@example.com" +export APPROVAL_TTL_SECONDS=900 + +# tcpdump tools are hidden unless explicitly enabled. +export ENABLED_RESTRICTED_TOOLS="tcpdump_capture,tcpdump_analyze" +export PCAP_PRESIGNED_URL_EXPIRATION=60 +export MAX_PCAP_BYTES=209715200 + +./deploy.sh +``` + +`collect`, `batch_collect(dryRun=false)`, and new `tcpdump_capture` requests pause at a native `aws:approve` step. The Lambda has no `ssm:SendAutomationSignal` permission and, while approval is enabled, no direct `ssm:SendCommand` permission. `batch_collect` defaults to `dryRun=true`; one approval fans out to at most 15 sampled child collections, and `batch_status` reports partial fan-out failures. Approval wrapper documents exist only in the stack region, so approval-gated operations must target that region. Deploy a stack in each required region rather than disabling approval. + +Set `REQUIRE_COLLECTION_APPROVAL=false` only for an explicitly supervised test deployment. Direct tcpdump Run Command permission is added only when approval is disabled **and** `tcpdump_capture` is enabled. + +Packet capture is opt-in and limited to ECS tasks on EC2. A new capture requires `instanceId`, an exact task ID or ARN, and `confirmCapture=true`. `containerName` must match one RUNNING application container; it may be omitted only when exactly one eligible container exists. The node re-resolves the container PID immediately before `nsenter`, rejects PID/namespace changes and the host network namespace, and never installs tcpdump. Fargate and host-wide captures are unsupported. `tcpdump_analyze` requires the exact `commandId`; there is no latest-capture fallback. Packet data may contain sensitive payloads, so minimize filters/duration and use the short-lived pcap URL. + +### AppSec parity evidence + +| ID | Control | Enforcement and test evidence | +|---|---|---| +| M1 | Authenticated tool surface | Cognito OAuth2 protects AgentCore; restricted schemas/routes are absent unless explicitly enabled (`test_tcpdump_security.py`). | +| M2 | Mandatory encryption | KMS customer-managed key, TLS-only S3, public-access block; synthesis rejects disabled KMS (`test_security_parity.py`). | +| M3 | Least-privilege identities | Explicit ECS instance-role principals are mandatory; no `AnyPrincipal` or account fallback (`test_security_parity.py`). | +| M4 | Human authorization | Default-on native SSM `aws:approve`; Lambda cannot approve and has no direct Run Command permission in approval mode (`test_collection_approval.py`, `test_tcpdump_approval.py`). | +| E1 | Deployment scope | Exact cluster and region allowlists fail closed and are applied to all live API paths (`test_instance_validation.py`, `test_security_parity.py`). | +| E2 | Target identity | Exact EC2 ID plus ACTIVE ECS container-instance membership; tags and names are not trusted (`test_instance_validation.py`). | +| E3 | Artifact isolation | Generic reads require `instanceId` and canonical `ecs_{instanceId}/...` keys; metadata, cross-instance paths, traversal, and pcaps are rejected (`test_security_parity.py`). | +| E4 | Search resource bounds | Unsafe regex structures are rejected and every scanned file has an interruptible hard timeout (`test_security_parity.py`). | +| E5 | Polling provenance | Status and batch polling accept only gateway-created IDs bound to expected document, region, and instance; batch polling requires opaque `batchId` (`test_collection_approval.py`, `test_batch_approval.py`). | +| E6 | Invasive-operation containment | Tcpdump requires explicit opt-in, confirmation, approval, exact task/container/PID/netns, safe BPF, and short-lived artifacts; batch is dry-run by default and capped at 15 (`test_tcpdump_security.py`, `test_batch_approval.py`). | + +### What Gets Deployed + +| Resource | Purpose | +|----------|---------| +| S3 Bucket (KMS encrypted) | Stores collected log bundles | +| S3 Bucket (SOPs) | Stores 36 runbooks, auto-deployed via CDK | +| Lambda (ECS Log Automation) | Handles MCP tool invocations (17 by default; up to 19 with restricted tools enabled) | +| Lambda (Unzip) | Auto-extracts uploaded archives | +| Lambda (Findings Indexer) | Pre-indexes errors for fast retrieval | +| SSM Automation Role | Runs approval wrappers, log collection, approved task-scoped capture, and batch child automations | +| SNS Approval Topic | Notifies configured approvers of pending SSM Automation requests | +| SSM Automation Documents | Native approval wrappers for single collection, batch collection, and task-scoped tcpdump | +| Cognito User Pool | OAuth2 authentication for MCP Gateway | +| BedrockAgentCore Gateway | MCP protocol endpoint | +| KMS Key | Encrypts all data at rest | + +--- + +## Post-Deployment: ECS Instance IAM Setup + +### What's Automatic + +The CDK stack requires explicit `ECS_INSTANCE_ROLE_ARNS` and grants only those roles: + +- S3 bucket policy: bucket-level `s3:ListBucket`, `s3:GetBucketPolicyStatus`, and `s3:GetBucketAcl` for the support document's `HeadBucket` preflight; object-level `s3:PutObject` for uploads +- KMS key policy: `kms:GenerateDataKey`, `kms:Encrypt` on the encryption key + +After deployment, `deploy.sh` reads the live bucket policy and fails with an actionable error unless every configured ECS instance role has all required preflight and upload actions. This prevents a successful stack deployment from surfacing later as an SSM `HeadBucket` 403 during collection. + +Synthesis fails when no explicit instance role is configured; there is no account-scoped principal fallback. + +### What You May Still Need + +The only thing the CDK stack does not attach is the SSM Agent managed policy. ECS-optimized AMIs include SSM Agent by default, but the IAM role needs the policy: + +```bash +# Only needed if not already attached +aws iam attach-role-policy \ + --role-name \ + --policy-arn arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore +``` + +### Adding Instance Roles After Deployment + +If you add new ECS clusters later, re-run the deploy script — it will detect the new instance roles and update the S3 bucket and KMS key policies automatically. + +Alternatively, pass the new roles directly: + +```bash +ECS_INSTANCE_ROLE_ARNS="arn:aws:iam::123456789012:role/ExistingRole,arn:aws:iam::123456789012:role/NewRole" ./deploy.sh +``` + +### Checklist Per Cluster + +- [ ] Instance role was selected during deployment (or added via re-deploy) +- [ ] Instance role has `AmazonSSMManagedInstanceCore` managed policy (for SSM Agent) +- [ ] SSM Agent is running on the instances (default on ECS-optimized AMIs) +- [ ] `AWSSupport-CollectECSInstanceLogs` SSM document exists in the target region + +--- + +## Configuration in DevOps Agent + +After deployment, the script outputs the non-secret values needed for MCP Server configuration: + +| Setting | Value | +|---------|-------| +| MCP Server URL | `https://.gateway.bedrock-agentcore..amazonaws.com/mcp` | +| OAuth Client ID | Cognito Client ID from output | +| Token URL | `https://-.auth..amazoncognito.com/oauth2/token` | +| Scope | `ecs-log-gateway-id/gateway:read` | + +Non-secret values are saved to ignored `mcp-config.txt`. The OAuth client secret is deliberately not retrieved, printed, or written by `deploy.sh`; retrieve it only at the point of secure MCP client registration. + +--- + +## How It Works + +The server gives MCP-compatible agents the ability to collect full diagnostic bundles from ECS container instances, pre-index errors with severity classification, stream multi-GB log files without truncation, correlate events across log sources, run opt-in task-scoped tcpdump captures, compare instances, and follow structured runbooks — through 17 tools by default (up to 19 when the two restricted tcpdump tools are enabled), organized in 5 tiers. + +For a detailed walkthrough of the architecture, data flows, tool design, cross-region mechanics, security model, and anti-hallucination design, see: + +**[Architecture & Design →](docs/ARCHITECTURE.md)** + +### MCP Tools (Quick Reference) + +| Tier | Tools | Purpose | +|------|-------|---------| +| 1 — Core | `collect`, `status`, `validate`, `errors`, `read` | Log collection, findings, streaming | +| 2 — Analysis | `search`, `correlate`, `artifact`, `summarize`, `history` | Deep investigation, correlation, summaries | +| 3 — Cluster | `cluster_health`, `compare_instances`, `batch_collect`, `batch_status`, `network_diagnostics` | Multi-instance operations | +| 4 — Capture (opt-in) | `tcpdump_capture`, `tcpdump_analyze` | Human-approved, task-scoped packet capture on ECS EC2 only | +| 5 — SOPs | `list_sops`, `get_sop` | 36 structured runbooks | + +### Agent Workflow + +``` +collect → status (poll) → validate → errors → search → correlate → read → summarize +``` + +### Runbook Library (36 SOPs) + +| Category | Coverage | +|----------|----------| +| A — Task Startup | Resource initialization, container runtime failures | +| B — Image Pull | ECR auth, image not found, Docker Hub rate limits | +| C — IAM/Secrets | Task execution role, Secrets Manager retrieval | +| D — Resource Exhaustion | OOM kills, disk space, CPU throttling | +| E — Networking | ENI allocation, DNS resolution, connection timeouts | +| F — Health Checks | Container health checks, ELB target health | +| G — ECS Agent | Agent disconnected, instance registration failure | +| H — Logging | CloudWatch log driver configuration | +| I — Deployment | Circuit breaker triggers, rollbacks | +| J — Container Runtime | OCI runtime, entrypoint issues, architecture mismatch | +| K — Extended | Spot interruption, task placement, steady state, auto scaling, ECS Exec, Service Connect, volume mounts, stuck pending, Fargate, API throttling, Windows, performance, Docker daemon | +| Z — Catch-All | General troubleshooting | + +--- + +## Usage Examples + +### Basic Investigation +``` +Container instance i-0abc123def in us-west-2 has tasks failing to start. +Collect its logs and correlate what happened in the last 10 minutes. +``` + +### Cluster-Wide Triage +``` +We have a 50-instance ECS cluster and something is off. Do a dry run batch +collection first — show me which instances you'd sample. Then collect from +the unhealthy ones. +``` + +### Live Packet Capture +``` +Tasks on instance i-0abc123def can't reach the backend service. Run a 2-minute +tcpdump filtered on port 8080, then analyze — show me RST counts and retransmissions. +``` + +### Task-Level Capture +``` +Task abc123 on instance i-0abc123def has connection timeouts. Capture traffic +scoped to that task's network namespace for 60 seconds on port 443. +``` + +### SOP-Guided +``` +I don't know what's wrong — just investigate. List the available SOPs, run a +general triage, and follow whichever runbook matches. +``` + +--- + +## CloudFormation Outputs + +| Output | Description | +|--------|-------------| +| `GatewayId` | AgentCore Gateway ID | +| `GatewayUrl` | MCP Server URL | +| `CognitoUserPoolId` | Cognito User Pool ID | +| `CognitoClientId` | OAuth Client ID | +| `OAuthExchangeUrl` | OAuth Token URL | +| `OAuthScope` | OAuth Scope | +| `LogsBucketName` | S3 bucket for logs | +| `SOPBucketName` | S3 bucket for runbooks | +| `SSMAutomationRoleArn` | SSM Automation role ARN | +| `EncryptionKeyArn` | KMS key ARN | + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `collect` returns "document not found" | SSM document not in target region | Use a supported region or pass `region` explicitly | +| Upload step fails | Instance role missing S3/KMS permissions | Re-run deploy with the instance role selected | +| `status` returns wrong region | Region metadata not persisted | Pass `region` explicitly | +| Auto-detection times out | Instance in uncommon region | Pass `region` explicitly | +| `errors` returns empty | Findings indexer hasn't run yet | Wait a few seconds after `validate`, or use `search` | +| Collection succeeds but no extracted bundle appears | `.tgz` S3 notification or canonical extraction mapping is missing | Re-deploy the current stack; `deploy.sh` validates `.zip`, `.tar.gz`, and `.tgz` notifications | +| `tcpdump_capture` uploads fail | Instance role missing S3 PutObject | Re-run deploy with the instance role selected | + +--- + +## Cleanup + +```bash +cdk destroy +``` + +> The S3 bucket has `removalPolicy: DESTROY` with `autoDeleteObjects: true`, so it will be cleaned up with the stack. + +--- + +## License + +This project is licensed under the MIT No Attribution (MIT-0) License. See the [LICENSE](LICENSE) file. diff --git a/mcp/ecs-instance-log-mcp/bin/app.ts b/mcp/ecs-instance-log-mcp/bin/app.ts new file mode 100644 index 0000000..21205e7 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/bin/app.ts @@ -0,0 +1,54 @@ +#!/usr/bin/env node +import 'source-map-support/register'; +import * as cdk from 'aws-cdk-lib'; +import { EcsLogGatewayStackV2 } from '../src/ecs-log-gateway-stack-v2'; + +const app = new cdk.App(); +const stackName = process.env.CDK_STACK_NAME ?? 'EcsInstanceLogMcpStack'; + +const parseEnvironmentList = (name: string): string[] | undefined => { + const values = Array.from(new Set( + (process.env[name] ?? '').split(',').map(value => value.trim()).filter(Boolean), + )); + return values.length > 0 ? values : undefined; +}; + +new EcsLogGatewayStackV2(app, stackName, { + description: 'ECS Instance Log MCP Server - Production-grade log collection for DevOps Agent with byte-range streaming and incident analysis', + env: { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION, + }, + gatewayProps: { + gatewayName: 'EcsInstanceLogMcpGW', + logRetentionDays: 1, + enableKmsEncryption: process.env.ENABLE_KMS_ENCRYPTION + ? !['0', 'false', 'no'].includes(process.env.ENABLE_KMS_ENCRYPTION.toLowerCase()) + : true, + ssmDefaultHostRoleArn: process.env.SSM_DEFAULT_HOST_ROLE_ARN?.trim() || undefined, + allowedClusterNames: parseEnvironmentList('ALLOWED_CLUSTER_NAMES'), + allowedRegions: parseEnvironmentList('ALLOWED_REGIONS'), + ecsInstanceRoleArns: parseEnvironmentList('ECS_INSTANCE_ROLE_ARNS'), + requireCollectionApproval: process.env.REQUIRE_COLLECTION_APPROVAL + ? !['0', 'false', 'no'].includes(process.env.REQUIRE_COLLECTION_APPROVAL.toLowerCase()) + : undefined, + approvalApproverArns: process.env.APPROVAL_APPROVER_ARNS + ? process.env.APPROVAL_APPROVER_ARNS.split(',').map(value => value.trim()).filter(Boolean) + : undefined, + approvalNotificationEmails: process.env.APPROVAL_NOTIFICATION_EMAILS + ? process.env.APPROVAL_NOTIFICATION_EMAILS.split(',').map(value => value.trim()).filter(Boolean) + : undefined, + approvalTtlSeconds: process.env.APPROVAL_TTL_SECONDS + ? parseInt(process.env.APPROVAL_TTL_SECONDS, 10) + : undefined, + enableRestrictedTools: process.env.ENABLED_RESTRICTED_TOOLS + ? process.env.ENABLED_RESTRICTED_TOOLS.split(',').map(value => value.trim()).filter(Boolean) + : undefined, + pcapPresignedUrlExpirationSeconds: process.env.PCAP_PRESIGNED_URL_EXPIRATION + ? parseInt(process.env.PCAP_PRESIGNED_URL_EXPIRATION, 10) + : undefined, + maxPcapBytes: process.env.MAX_PCAP_BYTES + ? parseInt(process.env.MAX_PCAP_BYTES, 10) + : undefined, + }, +}); diff --git a/mcp/ecs-instance-log-mcp/cdk.json b/mcp/ecs-instance-log-mcp/cdk.json new file mode 100644 index 0000000..6760f7a --- /dev/null +++ b/mcp/ecs-instance-log-mcp/cdk.json @@ -0,0 +1,22 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/app.ts", + "watch": { + "include": ["**"], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": ["aws", "aws-cn"] + } +} diff --git a/mcp/ecs-instance-log-mcp/deploy.sh b/mcp/ecs-instance-log-mcp/deploy.sh new file mode 100755 index 0000000..27d09ba --- /dev/null +++ b/mcp/ecs-instance-log-mcp/deploy.sh @@ -0,0 +1,604 @@ +#!/bin/bash +set -euo pipefail + +# ECS Instance Log MCP - Deploy and Configure Script +# This script deploys the CDK stack and outputs all values needed for DevOps Agent configuration + +STACK_NAME="${1:-${CDK_STACK_NAME:-EcsInstanceLogMcpStack}}" +REGION="${AWS_REGION:-us-east-1}" +export CDK_STACK_NAME="$STACK_NAME" + +# Optional: pass ECS instance role ARNs directly (comma-separated) +# Usage: ./deploy.sh EcsInstanceLogMcpStack arn:aws:iam::123456789012:role/ecsInstanceRole +# Or: ECS_INSTANCE_ROLE_ARNS=arn:aws:iam::123456789012:role/ecsInstanceRole ./deploy.sh +if [ -n "${2:-}" ]; then + export ECS_INSTANCE_ROLE_ARNS="$2" +fi + +echo "==============================================" +echo "ECS Instance Log MCP - Deployment Script" +echo "==============================================" +echo "Stack Name: $STACK_NAME" +echo "Region: $REGION" +echo "" + +# Check prerequisites +command -v npm >/dev/null 2>&1 || { echo "Error: npm is required but not installed."; exit 1; } +command -v aws >/dev/null 2>&1 || { echo "Error: AWS CLI is required but not installed."; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "Error: python3 is required but not installed."; exit 1; } + +# Install dependencies +echo "Installing dependencies..." +npm install --silent + +# Build TypeScript +echo "Building TypeScript..." +npm run build + +# Bootstrap CDK (if needed) +echo "Bootstrapping CDK (if needed)..." +npx cdk bootstrap --quiet 2>/dev/null || true + +# ======================================================================== +# DETECT / CREATE SSM DEFAULT HOST MANAGEMENT ROLE +# ======================================================================== +echo "" +echo "Setting up SSM Default Host Management role..." + +SSM_ROLE_NAME="AWSSystemsManagerDefaultEC2InstanceManagementRole" +EPOXY_ROLE_NAME="EpoxyAWSSystemsManagerDefaultEC2InstanceManagementRole" + +# Check for existing role (standard or Epoxy-prefixed) +SSM_DEFAULT_HOST_ROLE_ARN=$(aws iam get-role \ + --role-name "$SSM_ROLE_NAME" \ + --query 'Role.Arn' --output text 2>/dev/null || true) + +if [ -z "$SSM_DEFAULT_HOST_ROLE_ARN" ] || [ "$SSM_DEFAULT_HOST_ROLE_ARN" = "None" ]; then + SSM_DEFAULT_HOST_ROLE_ARN=$(aws iam get-role \ + --role-name "$EPOXY_ROLE_NAME" \ + --query 'Role.Arn' --output text 2>/dev/null || true) +fi + +if [ -z "$SSM_DEFAULT_HOST_ROLE_ARN" ] || [ "$SSM_DEFAULT_HOST_ROLE_ARN" = "None" ]; then + echo "SSM Default Host Management role not found. Creating $SSM_ROLE_NAME..." + + TRUST_POLICY=$(cat <<'TRUST' +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ssm.amazonaws.com" }, + "Action": "sts:AssumeRole" + } + ] +} +TRUST +) + + aws iam create-role \ + --role-name "$SSM_ROLE_NAME" \ + --assume-role-policy-document "$TRUST_POLICY" \ + --description "Default EC2 instance management role for SSM" \ + --region "$REGION" >/dev/null + + aws iam attach-role-policy \ + --role-name "$SSM_ROLE_NAME" \ + --policy-arn "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + + SSM_DEFAULT_HOST_ROLE_ARN=$(aws iam get-role \ + --role-name "$SSM_ROLE_NAME" \ + --query 'Role.Arn' --output text) + + echo "Created role: $SSM_DEFAULT_HOST_ROLE_ARN" + echo "Waiting 10s for IAM propagation..." + sleep 10 +else + echo "Found existing role: $SSM_DEFAULT_HOST_ROLE_ARN" +fi + +export SSM_DEFAULT_HOST_ROLE_ARN + +# ======================================================================== +# AUTO-DETECT ECS CONTAINER INSTANCE ROLE ARNS (interactive) +# ======================================================================== +echo "" +if [ -n "${ECS_INSTANCE_ROLE_ARNS:-}" ]; then + if [ -z "${ALLOWED_CLUSTER_NAMES:-}" ]; then + echo "Error: ALLOWED_CLUSTER_NAMES is required in non-interactive role mode." + exit 1 + fi + export ALLOWED_REGIONS="${ALLOWED_REGIONS:-$REGION}" + echo "Using provided ECS instance role ARNs: $ECS_INSTANCE_ROLE_ARNS" + echo "Allowed clusters: $ALLOWED_CLUSTER_NAMES" + echo "Allowed regions: $ALLOWED_REGIONS" +else + # --- Step 1: Region selection --- + echo "Which AWS regions should be scanned for ECS clusters?" + echo "" + echo " 1) All enabled regions" + echo " 2) Current deploy region only ($REGION)" + echo " 3) Enter a specific region" + echo "" + read -rp "Select [1/2/3] (default: 1): " REGION_CHOICE + REGION_CHOICE="${REGION_CHOICE:-1}" + + case "$REGION_CHOICE" in + 1) + echo "" + echo "Fetching all enabled regions..." + SCAN_REGIONS=$(aws ec2 describe-regions --query 'Regions[].RegionName' --output text 2>/dev/null || echo "$REGION") + ;; + 2) + SCAN_REGIONS="$REGION" + ;; + 3) + read -rp "Enter region (e.g. us-west-2): " CUSTOM_REGION + if [ -z "$CUSTOM_REGION" ]; then + echo "No region entered, falling back to $REGION" + CUSTOM_REGION="$REGION" + fi + SCAN_REGIONS="$CUSTOM_REGION" + ;; + *) + echo "Invalid choice, falling back to all regions." + SCAN_REGIONS=$(aws ec2 describe-regions --query 'Regions[].RegionName' --output text 2>/dev/null || echo "$REGION") + ;; + esac + + # --- Step 2: Discover ECS clusters across selected regions --- + echo "" + echo "Scanning for ECS clusters..." + + CLUSTER_LIST=() + CLUSTER_DISPLAY=() + + for SCAN_REGION in $SCAN_REGIONS; do + CLUSTERS=$(aws ecs list-clusters --region "$SCAN_REGION" --query 'clusterArns[*]' --output text 2>/dev/null || true) + if [ -n "$CLUSTERS" ]; then + for CLUSTER_ARN in $CLUSTERS; do + CLUSTER_NAME="${CLUSTER_ARN##*/}" + CLUSTER_LIST+=("${SCAN_REGION}/${CLUSTER_NAME}") + done + fi + done + + if [ ${#CLUSTER_LIST[@]} -eq 0 ]; then + echo "Error: No ECS clusters found in the selected region(s)." + echo "A nonempty cluster allowlist and explicit ECS instance role ARNs are mandatory." + exit 1 + else + # --- Step 3: Display clusters and let user choose --- + echo "" + echo "Found ${#CLUSTER_LIST[@]} ECS cluster(s):" + echo "" + IDX=1 + for ENTRY in "${CLUSTER_LIST[@]}"; do + C_REGION="${ENTRY%%/*}" + C_NAME="${ENTRY#*/}" + echo " ${IDX}) ${C_NAME} (${C_REGION})" + IDX=$((IDX + 1)) + done + echo "" + echo " a) All clusters" + echo "" + read -rp "Select clusters (comma-separated numbers, or 'a' for all) [default: a]: " CLUSTER_CHOICE + CLUSTER_CHOICE="${CLUSTER_CHOICE:-a}" + + SELECTED_CLUSTERS=() + if [ "$CLUSTER_CHOICE" = "a" ] || [ "$CLUSTER_CHOICE" = "A" ]; then + SELECTED_CLUSTERS=("${CLUSTER_LIST[@]}") + else + IFS=',' read -ra PICKS <<< "$CLUSTER_CHOICE" + for PICK in "${PICKS[@]}"; do + PICK=$(echo "$PICK" | tr -d ' ') + if [[ "$PICK" =~ ^[0-9]+$ ]] && [ "$PICK" -ge 1 ] && [ "$PICK" -le ${#CLUSTER_LIST[@]} ]; then + SELECTED_CLUSTERS+=("${CLUSTER_LIST[$((PICK - 1))]}") + else + echo " Skipping invalid selection: $PICK" + fi + done + fi + + if [ ${#SELECTED_CLUSTERS[@]} -eq 0 ]; then + echo "Error: No valid clusters selected." + exit 1 + else + ALLOWED_CLUSTER_NAMES="" + ALLOWED_REGIONS="" + for ENTRY in "${SELECTED_CLUSTERS[@]}"; do + C_REGION="${ENTRY%%/*}" + C_NAME="${ENTRY#*/}" + case ",$ALLOWED_CLUSTER_NAMES," in + *",$C_NAME,"*) ;; + *) ALLOWED_CLUSTER_NAMES="${ALLOWED_CLUSTER_NAMES:+$ALLOWED_CLUSTER_NAMES,}$C_NAME" ;; + esac + case ",$ALLOWED_REGIONS," in + *",$C_REGION,"*) ;; + *) ALLOWED_REGIONS="${ALLOWED_REGIONS:+$ALLOWED_REGIONS,}$C_REGION" ;; + esac + done + export ALLOWED_CLUSTER_NAMES ALLOWED_REGIONS + echo "Allowed clusters: $ALLOWED_CLUSTER_NAMES" + echo "Allowed regions: $ALLOWED_REGIONS" + echo "" + echo "Detecting container instance roles for ${#SELECTED_CLUSTERS[@]} cluster(s)..." + + # --- Step 4: Collect unique instance role ARNs from selected clusters --- + ALL_ROLE_ARNS=() + ROLE_SOURCES=() + + for ENTRY in "${SELECTED_CLUSTERS[@]}"; do + C_REGION="${ENTRY%%/*}" + C_NAME="${ENTRY#*/}" + + # List container instances in the cluster + CI_ARNS=$(aws ecs list-container-instances --cluster "$C_NAME" --region "$C_REGION" \ + --query 'containerInstanceArns[*]' --output text 2>/dev/null || true) + + if [ -n "$CI_ARNS" ]; then + # Describe container instances to get EC2 instance IDs + CI_DETAILS=$(aws ecs describe-container-instances --cluster "$C_NAME" --region "$C_REGION" \ + --container-instances $CI_ARNS \ + --query 'containerInstances[].ec2InstanceId' --output text 2>/dev/null || true) + + for EC2_ID in $CI_DETAILS; do + # Get the IAM instance profile role from the EC2 instance + ROLE_ARN=$(aws ec2 describe-instances --instance-ids "$EC2_ID" --region "$C_REGION" \ + --query 'Reservations[0].Instances[0].IamInstanceProfile.Arn' --output text 2>/dev/null || true) + + if [ -n "$ROLE_ARN" ] && [ "$ROLE_ARN" != "None" ]; then + # Convert instance profile ARN to role ARN + PROFILE_NAME="${ROLE_ARN##*/}" + ACTUAL_ROLE_ARN=$(aws iam get-instance-profile --instance-profile-name "$PROFILE_NAME" \ + --query 'InstanceProfile.Roles[0].Arn' --output text 2>/dev/null || true) + + if [ -n "$ACTUAL_ROLE_ARN" ] && [ "$ACTUAL_ROLE_ARN" != "None" ]; then + # Deduplicate + ALREADY_ADDED=false + for EXISTING in "${ALL_ROLE_ARNS[@]}"; do + if [ "$EXISTING" = "$ACTUAL_ROLE_ARN" ]; then + ALREADY_ADDED=true + break + fi + done + if [ "$ALREADY_ADDED" = false ]; then + ALL_ROLE_ARNS+=("$ACTUAL_ROLE_ARN") + ROLE_NAME="${ACTUAL_ROLE_ARN##*/}" + ROLE_SOURCES+=("${ROLE_NAME} (${C_NAME} / ${C_REGION})") + fi + fi + fi + done + fi + done + + if [ ${#ALL_ROLE_ARNS[@]} -eq 0 ]; then + echo "WARNING: No container instance roles found in selected clusters." + echo " (Clusters may be empty or using Fargate launch type)" + echo "" + read -rp "Would you like to manually enter instance role ARN(s)? [y/N]: " MANUAL_ENTRY + if [ "$MANUAL_ENTRY" = "y" ] || [ "$MANUAL_ENTRY" = "Y" ]; then + echo "Enter comma-separated role ARNs (e.g. arn:aws:iam::123456789012:role/ecsInstanceRole):" + read -rp "> " MANUAL_ARNS + MANUAL_ARNS=$(echo "$MANUAL_ARNS" | tr -d ' ') + if [ -n "$MANUAL_ARNS" ]; then + ECS_INSTANCE_ROLE_ARNS="$MANUAL_ARNS" + echo "Using manually provided roles: $ECS_INSTANCE_ROLE_ARNS" + export ECS_INSTANCE_ROLE_ARNS + else + echo "Error: At least one explicit ECS instance role ARN is required." + exit 1 + fi + else + echo "Error: At least one explicit ECS instance role ARN is required." + exit 1 + fi + else + # --- Step 5: Let user choose which instance roles to include --- + echo "" + echo "Found ${#ALL_ROLE_ARNS[@]} unique container instance role(s):" + echo "" + IDX=1 + for i in "${!ALL_ROLE_ARNS[@]}"; do + echo " ${IDX}) ${ALL_ROLE_ARNS[$i]}" + echo " └─ ${ROLE_SOURCES[$i]}" + IDX=$((IDX + 1)) + done + echo "" + echo " a) All roles" + echo "" + read -rp "Select instance roles (comma-separated numbers, or 'a' for all) [default: a]: " ROLE_CHOICE + ROLE_CHOICE="${ROLE_CHOICE:-a}" + + SELECTED_ROLES=() + if [ "$ROLE_CHOICE" = "a" ] || [ "$ROLE_CHOICE" = "A" ]; then + SELECTED_ROLES=("${ALL_ROLE_ARNS[@]}") + else + IFS=',' read -ra PICKS <<< "$ROLE_CHOICE" + for PICK in "${PICKS[@]}"; do + PICK=$(echo "$PICK" | tr -d ' ') + if [[ "$PICK" =~ ^[0-9]+$ ]] && [ "$PICK" -ge 1 ] && [ "$PICK" -le ${#ALL_ROLE_ARNS[@]} ]; then + SELECTED_ROLES+=("${ALL_ROLE_ARNS[$((PICK - 1))]}") + else + echo " Skipping invalid selection: $PICK" + fi + done + fi + + # Build comma-separated string + ECS_INSTANCE_ROLE_ARNS="" + for ROLE in "${SELECTED_ROLES[@]}"; do + if [ -z "$ECS_INSTANCE_ROLE_ARNS" ]; then + ECS_INSTANCE_ROLE_ARNS="$ROLE" + else + ECS_INSTANCE_ROLE_ARNS="$ECS_INSTANCE_ROLE_ARNS,$ROLE" + fi + done + + if [ -n "$ECS_INSTANCE_ROLE_ARNS" ]; then + echo "" + echo "Using ECS instance roles: $ECS_INSTANCE_ROLE_ARNS" + export ECS_INSTANCE_ROLE_ARNS + else + echo "No roles selected." + read -rp "Would you like to manually enter instance role ARN(s) instead? [y/N]: " MANUAL_ENTRY + if [ "$MANUAL_ENTRY" = "y" ] || [ "$MANUAL_ENTRY" = "Y" ]; then + echo "Enter comma-separated role ARNs (e.g. arn:aws:iam::123456789012:role/ecsInstanceRole):" + read -rp "> " MANUAL_ARNS + MANUAL_ARNS=$(echo "$MANUAL_ARNS" | tr -d ' ') + if [ -n "$MANUAL_ARNS" ]; then + ECS_INSTANCE_ROLE_ARNS="$MANUAL_ARNS" + echo "Using manually provided roles: $ECS_INSTANCE_ROLE_ARNS" + export ECS_INSTANCE_ROLE_ARNS + else + echo "Error: At least one explicit ECS instance role ARN is required." + exit 1 + fi + else + echo "Error: At least one explicit ECS instance role ARN is required." + exit 1 + fi + fi + fi + fi + fi +fi + +if [ -z "${ALLOWED_CLUSTER_NAMES:-}" ]; then + echo "Error: ALLOWED_CLUSTER_NAMES must contain at least one ECS cluster." + exit 1 +fi +if [ -z "${ALLOWED_REGIONS:-}" ]; then + echo "Error: ALLOWED_REGIONS must contain at least one AWS region." + exit 1 +fi +if [ -z "${ECS_INSTANCE_ROLE_ARNS:-}" ]; then + echo "Error: ECS_INSTANCE_ROLE_ARNS must contain at least one explicit role ARN." + exit 1 +fi +export ALLOWED_CLUSTER_NAMES ALLOWED_REGIONS ECS_INSTANCE_ROLE_ARNS + +# Deploy the stack +echo "" +echo "Deploying CDK stack..." +npx cdk deploy "$STACK_NAME" --require-approval never --outputs-file cdk-outputs.json + +echo "" +echo "==============================================" +echo "Deployment Complete! Retrieving configuration..." +echo "==============================================" + +# Read from cdk-outputs.json using python3 for reliable JSON parsing +if [ ! -f cdk-outputs.json ]; then + echo "Error: cdk-outputs.json not found" + exit 1 +fi + +# Parse values from cdk-outputs.json +GATEWAY_URL=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'GatewayUrl' in k][0])" 2>/dev/null || echo "NOT_FOUND") +CLIENT_ID=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'CognitoClientId' in k][0])" 2>/dev/null || echo "NOT_FOUND") +USER_POOL_ID=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'CognitoUserPoolId' in k][0])" 2>/dev/null || echo "NOT_FOUND") +TOKEN_URL=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'OAuthExchangeUrl' in k][0])" 2>/dev/null || echo "NOT_FOUND") +OAUTH_SCOPE=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'OAuthScope' in k][0])" 2>/dev/null || echo "NOT_FOUND") +LOGS_BUCKET=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'LogsBucketName' in k][0])" 2>/dev/null || echo "NOT_FOUND") + +# Verify the deployed resource policy supports the preflight and upload calls made by +# AWSSupport-CollectECSInstanceLogs. This catches policy regressions before users run a collection. +if [ "$LOGS_BUCKET" = "NOT_FOUND" ]; then + echo "Error: Logs bucket output was not found; cannot validate instance upload permissions." + exit 1 +fi + +echo "Validating ECS instance upload permissions..." +if ! BUCKET_POLICY_JSON=$(aws s3api get-bucket-policy \ + --bucket "$LOGS_BUCKET" \ + --region "$REGION" \ + --query Policy \ + --output text 2>/dev/null); then + echo "Error: Unable to read the logs bucket policy for post-deployment validation." + exit 1 +fi +export BUCKET_POLICY_JSON + +python3 - "$LOGS_BUCKET" "$ECS_INSTANCE_ROLE_ARNS" <<'PY' +import json +import os +import sys + +bucket_name = sys.argv[1] +role_arns = [value.strip() for value in sys.argv[2].split(',') if value.strip()] +policy = json.loads(os.environ['BUCKET_POLICY_JSON']) +bucket_arn = f'arn:aws:s3:::{bucket_name}' +object_arn = f'{bucket_arn}/*' +required_bucket_actions = {'s3:GetBucketAcl', 's3:GetBucketPolicyStatus', 's3:ListBucket'} +required_object_actions = {'s3:PutObject'} + + +def values(value): + return value if isinstance(value, list) else [value] + + +failures = [] +for role_arn in role_arns: + bucket_actions = set() + object_actions = set() + for statement in policy.get('Statement', []): + if statement.get('Effect') != 'Allow': + continue + principals = values(statement.get('Principal', {}).get('AWS', [])) + if role_arn not in principals: + continue + actions = set(values(statement.get('Action', []))) + resources = set(values(statement.get('Resource', []))) + if bucket_arn in resources: + bucket_actions.update(actions) + if object_arn in resources: + object_actions.update(actions) + missing_bucket = sorted(required_bucket_actions - bucket_actions) + missing_object = sorted(required_object_actions - object_actions) + if missing_bucket or missing_object: + failures.append( + f'{role_arn}: missing bucket actions {missing_bucket or "none"}; ' + f'missing object actions {missing_object or "none"}' + ) + +if failures: + raise SystemExit('Error: ECS instance upload policy validation failed:\n ' + '\n '.join(failures)) + +print(f'Validated HeadBucket and PutObject permissions for {len(role_arns)} ECS instance role(s).') +PY +unset BUCKET_POLICY_JSON + +echo "Validating archive extraction notifications..." +if ! NOTIFICATIONS_JSON=$(aws s3api get-bucket-notification-configuration \ + --bucket "$LOGS_BUCKET" \ + --region "$REGION" \ + --output json 2>/dev/null); then + echo "Error: Unable to read the logs bucket notification configuration." + exit 1 +fi +export NOTIFICATIONS_JSON + +python3 - "$STACK_NAME" <<'PY' +import json +import os +import sys + +function_name = f'{sys.argv[1]}-unzip-function' +configuration = json.loads(os.environ['NOTIFICATIONS_JSON']) +configured_suffixes = set() +for item in configuration.get('LambdaFunctionConfigurations', []): + if not item.get('LambdaFunctionArn', '').endswith(f':function:{function_name}'): + continue + for rule in item.get('Filter', {}).get('Key', {}).get('FilterRules', []): + if rule.get('Name', '').lower() == 'suffix': + configured_suffixes.add(rule.get('Value')) + +required_suffixes = {'.zip', '.tar.gz', '.tgz'} +missing_suffixes = sorted(required_suffixes - configured_suffixes) +if missing_suffixes: + raise SystemExit( + f'Error: {function_name} is missing S3 ObjectCreated notifications for: ' + f'{", ".join(missing_suffixes)}' + ) +print(f'Validated archive notifications for {function_name}: {sorted(configured_suffixes)}') +PY +unset NOTIFICATIONS_JSON + +# The OAuth client secret is intentionally not retrieved, printed, or written to disk. +# Retrieve it only at the point of secure MCP registration using least-privilege credentials. + +echo "" +echo "==============================================" +echo "DEVOPS AGENT MCP SERVER CONFIGURATION" +echo "==============================================" +echo "" +echo "Copy these values to configure the MCP Server in DevOps Agent Console:" +echo "" +echo "┌─────────────────────────────────────────────────────────────────────┐" +echo "│ MCP Server URL: │" +echo "│ $GATEWAY_URL" +echo "├─────────────────────────────────────────────────────────────────────┤" +echo "│ OAuth Client ID: │" +echo "│ $CLIENT_ID" +echo "├─────────────────────────────────────────────────────────────────────┤" +echo "│ OAuth Client Secret: │" +echo "│ Not displayed or stored by this script. Retrieve it only during │" +echo "│ secure MCP client registration. │" +echo "├─────────────────────────────────────────────────────────────────────┤" +echo "│ Token URL: │" +echo "│ $TOKEN_URL" +echo "├─────────────────────────────────────────────────────────────────────┤" +echo "│ Scope (use only ONE): │" +echo "│ $OAUTH_SCOPE" +echo "└─────────────────────────────────────────────────────────────────────┘" +echo "" +echo "Additional Info:" +echo " Logs Bucket: $LOGS_BUCKET" +echo " Region: $REGION" +echo "" + +# Save configuration to file +CONFIG_FILE="mcp-config.txt" +cat > "$CONFIG_FILE" << EOF +# ECS Instance Log MCP - DevOps Agent Configuration +# Generated: $(date) +# Stack: $STACK_NAME +# Region: $REGION + +MCP_SERVER_URL=$GATEWAY_URL +OAUTH_CLIENT_ID=$CLIENT_ID +# OAUTH_CLIENT_SECRET is intentionally omitted. Retrieve it only during secure client registration. +TOKEN_URL=$TOKEN_URL +OAUTH_SCOPE=$OAUTH_SCOPE +LOGS_BUCKET=$LOGS_BUCKET +EOF + +echo "Configuration saved to: $CONFIG_FILE" +echo "" +echo "==============================================" +echo "AVAILABLE MCP TOOLS" +echo "==============================================" +echo "" +echo "TIER 1: CORE OPERATIONS" +echo "------------------------" +echo "1. collect - Start log collection from an ECS container instance" +echo "2. status - Get detailed status with progress tracking" +echo "3. validate - Verify all expected files were extracted" +echo "4. errors - Get pre-indexed error findings by severity" +echo "5. read - Byte-range streaming for multi-GB files" +echo "" +echo "TIER 2: ADVANCED ANALYSIS" +echo "-------------------------" +echo "6. search - Full-text regex search across all logs" +echo "7. correlate - Cross-file timeline correlation" +echo "8. artifact - Secure presigned URLs for large artifacts" +echo "9. summarize - Finding-grounded incident summary" +echo "10. history - Audit trail of past collections" +echo "" +echo "TIER 3: CLUSTER-LEVEL INTELLIGENCE" +echo "-----------------------------------" +echo "11. cluster_health - Health overview across all instances in a cluster" +echo "12. compare_instances - Diff findings between 2+ instances" +echo "13. batch_collect - Smart batch collection with sampling" +echo "14. batch_status - Poll status of multiple collections" +echo "15. network_diagnostics - Structured networking analysis" +echo "" +echo "TIER 4: LIVE PACKET CAPTURE" +echo "---------------------------" +echo "16. tcpdump_capture - Run tcpdump via SSM (supports task-scoped captures)" +echo "17. tcpdump_analyze - Decoded packets, stats, anomaly detection" +echo "" +echo "TIER 5: SOPs" +echo "------------" +echo "18. list_sops - List all 36 structured runbooks" +echo "19. get_sop - Get a specific runbook by name" +echo "" +echo "==============================================" +echo "EXAMPLE PROMPT FOR DEVOPS AGENT" +echo "==============================================" +echo "" +echo "\"I'm investigating an ECS container instance issue on i-0123456789abcdef0." +echo " Collect logs, find any critical errors, and give me a summary.\"" +echo "" diff --git a/mcp/ecs-instance-log-mcp/docs/ARCHITECTURE.md b/mcp/ecs-instance-log-mcp/docs/ARCHITECTURE.md new file mode 100644 index 0000000..bb5c3ad --- /dev/null +++ b/mcp/ecs-instance-log-mcp/docs/ARCHITECTURE.md @@ -0,0 +1,308 @@ +# ECS Instance Log MCP — Architecture & Design + +This document explains the internal design of the ECS Instance Log MCP server: how the components fit together, how data flows from an ECS container instance to an AI agent's context window, and the design decisions behind each layer. + +## Table of Contents + +- [System Overview](#system-overview) +- [Component Deep Dive](#component-deep-dive) +- [Data Flow](#data-flow) +- [Cross-Region Design](#cross-region-design) +- [Tool Architecture](#tool-architecture) +- [Time-Bounded Analysis](#time-bounded-analysis) +- [Anti-Hallucination Design](#anti-hallucination-design) +- [SOP Runbook System](#sop-runbook-system) +- [Security Model](#security-model) +- [CDK Construct Design](#cdk-construct-design) +- [Deploy Script Design](#deploy-script-design) + +--- + +## System Overview + +The server bridges the gap between AI agents and the OS-level diagnostic data on ECS container instances. The ECS API and CloudWatch don't expose Docker daemon config, iptables rules, cgroup memory events, container runtime state, or ECS agent internal logs — but these are exactly what's needed to diagnose task startup failures, agent disconnects, OOM kills, and networking issues. + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ MCP Client │────▶│ MCP Gateway │────▶│ Lambda │ +│ (DevOps Agent) │◀────│ (AgentCore) │◀────│ (19 tools) │ +└─────────────────┘ └──────────────────┘ └────────┬────────┘ + │ OAuth2 │ + ▼ │ + ┌──────────┐ ┌───────────┼───────────┐ + │ Cognito │ │ │ │ + │ User Pool│ ▼ ▼ ▼ + └──────────┘ ┌──────────┐ ┌──────────┐ ┌──────────┐ + │ SSM │ │ S3 │ │ S3 │ + │ Automati-│ │ Logs │ │ SOPs │ + │ on │ │ (KMS) │ │ Bucket │ + └────┬─────┘ └──────────┘ └──────────┘ + │ + ┌─────────┼─────────┐ + ▼ ▼ ▼ + ┌────────┐┌────────┐┌────────┐ + │ECS Inst││ECS Inst││ECS Inst│ + │Region A││Region B││Region C│ + └────────┘└────────┘└────────┘ +``` + +A deployment serves the regions allowed by IAM for read-only diagnostics. Native approval wrappers are account-owned regional SSM documents, so approval-gated collection and capture target the stack region. Deploy one stack per required region. + +--- + +## Component Deep Dive + +### MCP Gateway (Bedrock AgentCore) + +Entry point for all MCP tool calls. Handles MCP protocol (JSON-RPC over HTTP), OAuth2 token validation via Cognito, and request routing to Lambda. Tool names are kept short (e.g., `collect`, `errors`, `read`) to stay under the 64-character limit. + +### Lambda Function (Tool Router) + +A single Python Lambda implements 17 default tools and two opt-in packet-capture tools. Key design: +- **Single Lambda**: All tools share one function to avoid cold start multiplication +- **Regional clients**: `get_regional_client()` creates boto3 clients per region with caching +- **Auto-detection**: `detect_instance_region()` tries the default region, then allowed candidate regions +- **Exact ECS membership**: `validate_ecs_instance()` enumerates only configured `ALLOWED_CLUSTER_NAMES`, paginates their container instances, and requires an exact ACTIVE `ec2InstanceId` match; names and tags are not trusted as membership proof +- **Cluster and region allowlists**: empty cluster configuration fails closed, and `ALLOWED_REGIONS` restricts every live regional client path +- **Approval separation**: Lambda has no `ssm:SendAutomationSignal` and no direct `ssm:SendCommand` while approval is enabled + +### SSM Automation + +Approval is enabled by default. Three account-owned Automation documents pause at native `aws:approve` steps for single collection, one-approved batch fan-out, and task-scoped tcpdump. The Automation role, approvers, and SNS topic are embedded at deployment rather than supplied by callers. + +After approval, log collection invokes `AWSSupport-CollectECSInstanceLogs`, which: +- Runs on the target EC2 instance via SSM Agent +- Collects ECS agent logs, Docker/containerd, container logs, system logs, dmesg, networking, cgroups, metadata, and GPU info +- Packages an archive (`.tgz` on current Linux runbook versions; `.tar.gz` and `.zip` are also supported) and uploads it to the central S3 bucket + +The managed runbook writes Linux bundles at the bucket root using `ecs__.tgz`. The unzip Lambda validates that deployment-owned layout and maps extracted content into the canonical instance namespace used by analysis tools. + +The tcpdump wrapper invokes `AWS-RunShellScript` only after approval. The script never installs packages and rejects Fargate, host-wide, host-network, ambiguous-container, stale-PID, and changed-namespace targets. + +### S3 Log Storage + +Two S3 buckets: + +1. **Logs bucket** (KMS-encrypted): + ``` + ecs_{instance-id}_{execution-id}.tgz # Raw bundle from the managed SSM runbook + ecs_{instance-id}/{execution-id}/ + └── extracted/ # Canonical analysis namespace + ├── var/log/ecs/ecs-agent.log + ├── var/log/docker + ├── iptables-rules.txt + ├── manifest.json + └── findings_index.json + + idempotency/{instance-id}/ # Dedup mappings + _metadata/ # Execution and batch provenance + ``` + +2. **SOPs bucket**: 36 runbook markdown files, auto-deployed via CDK. + +### Findings Indexer + +The unzip Lambda invokes the Findings Indexer asynchronously after extraction and manifest generation. The indexer scans extracted files for ECS-specific error patterns (agent disconnects, image pull failures, OOM kills, etc.), assigns severity and stable finding IDs (F-001), and writes `findings_index.json` into the same canonical extraction prefix. + +### Unzip Lambda + +Triggered by S3 ObjectCreated notifications for `.zip`, `.tar.gz`, and `.tgz`. It URL-decodes the S3 event key, accepts only validated managed-runbook or canonical archive layouts, maps root-level managed bundles into `ecs_//extracted/`, generates `manifest.json`, and directly invokes the Findings Indexer. The deploy script verifies all three notification suffixes after deployment. + +### KMS Encryption + +Customer-managed key encrypts all S3 objects. S3 client uses SigV4 explicitly for presigned URL compatibility. Key and bucket policies require explicit ECS instance-role ARNs; synthesis fails instead of creating an account-scoped compatibility fallback. + +--- + +## Data Flow + +### Log Collection Flow + +``` +Agent calls collect(instanceId) + → Lambda validates allowed region and exact ACTIVE ECS membership + → Lambda starts the stack-region collection approval wrapper + → Native aws:approve waits; Lambda cannot approve its own request + → Approved wrapper invokes AWSSupport-CollectECSInstanceLogs + → SSM Agent collects and uploads the managed-runbook archive + → .zip/.tar.gz/.tgz ObjectCreated notification invokes Unzip Lambda + → Unzip Lambda maps the bundle into the canonical instance namespace and extracts it + → Unzip Lambda writes manifest.json and directly invokes Findings Indexer + → Agent polls status(executionId) until the child execution completes +``` + +### Batch Collection Flow + +``` +Agent calls batch_collect(clusterName) → dry-run plan by default + → Agent explicitly calls batch_collect(..., dryRun=false) + → One native approval covers at most 15 sampled instances + → Wrapper starts child collection automations and records per-instance errors + → Agent polls batch_status(batchId) for complete, failed, or partial_failure +``` + +### Live Packet Capture Flow + +``` +Agent calls tcpdump_capture(instanceId, taskId, containerName?, confirmCapture=true) + → Restricted-tool and same-region approval checks fail closed + → Native aws:approve waits with task/container scope in the request + → Approved wrapper dispatches the fixed Run Command path + → Script exactly resolves task and RUNNING application container + → Docker/containerd resolves PID; PID start time and net namespace are recorded + → Script rejects host namespace and re-resolves PID immediately before nsenter + → nsenter runs preinstalled tcpdump and uploads pcap/text/stats + → Agent polls executionId, then commandId + → Agent calls tcpdump_analyze(instanceId, commandId) +``` + +--- + +## Cross-Region Design + +The S3 data plane and read-only diagnostic APIs can cover configured `ALLOWED_REGIONS`. Approval wrappers are regional SSM documents and this implementation deliberately requires approval-gated operations to target the stack region. For multiple operational regions, deploy the stack separately in each region so every collection or capture retains native approval; do not disable approval as a regional workaround. + +- Region resolution: explicit parameter, then instance detection, then Lambda region +- Execution-region metadata is persisted for polling +- IAM resources and unsupported resource-level actions are constrained to allowed regions +- New approved collection, batch fan-out, and tcpdump requests are rejected when the target differs from the stack region + +--- + +## Tool Architecture + +| Tier | Tools | Purpose | +|------|-------|---------| +| 1 — Core | `collect`, `status`, `validate`, `errors`, `read` | Log collection, findings, streaming | +| 2 — Analysis | `search`, `correlate`, `artifact`, `summarize`, `history` | Deep investigation, correlation | +| 3 — Cluster | `cluster_health`, `compare_instances`, `batch_collect`, `batch_status`, `network_diagnostics` | Multi-instance ops | +| 4 — Restricted capture | `tcpdump_capture`, `tcpdump_analyze` | Opt-in, human-approved task packet capture | +| 5 — SOPs | `list_sops`, `get_sop` | Structured runbooks, including approved capture operations | + +--- + +## Time-Bounded Analysis + +`TimeWindowResolver` enforces time windows on all analysis: +1. Explicit `start_time` + `end_time` → used as-is +2. `incident_time` → ± 5 minutes +3. Nothing → last 10 minutes +4. Max: 24 hours (safety cap) + +--- + +## Anti-Hallucination Design + +1. **Finding IDs**: Every error gets a stable ID (F-001). `summarize` requires finding IDs — unresolved IDs are flagged. +2. **ECS instance validation**: `collect` verifies the target belongs to an ECS cluster before running SSM. +3. **Region allow-list**: Tools reject requests to disallowed regions. +4. **Baseline subtraction**: Known noise is annotated, not removed. +5. **Confidence scores**: `correlate` reports confidence and data gaps. +6. **Network diagnostics guardrails**: `network_diagnostics` returns an `ecsContext` block with domain-specific guardrails (Docker bridge vs awsvpc, ECS Agent vs container networking, SG per network mode, DNS per network mode, conntrack attribution, ENI limits, Service Connect vs Discovery, container vs ELB health checks). +7. **False positive suppression**: Error scanning filters out `error_count=0`, conditional error handling, etc. +8. **Configurable presigned URLs**: `PRESIGNED_URL_EXPIRATION_SECONDS` env var controls URL lifetime. +9. **S3 SigV4**: Explicit SigV4 signing for KMS-encrypted bucket compatibility. + +--- + +## SOP Runbook System + +Runbooks cover ECS-specific failure categories (A-K, Z) plus human-approved task packet capture. Each follows a 3-phase structure: +- Phase 1 — Triage (MUST): Check cluster/task state, collect logs, get findings +- Phase 2 — Enrich (SHOULD): Deep search, correlate, domain-specific diagnostics +- Phase 3 — Report (MUST): Grounded summary with finding IDs, root cause, remediation + +Auto-matched via `recommendedSOPs` in `errors`, `correlate`, and `summarize` responses. + +--- + +## Security Model + +| Layer | Mechanism | +|-------|-----------| +| Authentication | Cognito OAuth2 client credentials grant | +| Encryption at rest | KMS customer-managed key | +| Encryption in transit | HTTPS enforced on S3 | +| Public access | S3 Block Public Access | +| Native approval | Default-on `aws:approve`; synthesis fails without approvers | +| Approval ownership | Automation role, approvers, and SNS topic are embedded in documents | +| Approval separation | Lambda has no `ssm:SendAutomationSignal` and no direct `SendCommand` in approval mode | +| IAM | StartAutomation and SendCommand scoped to required documents and allowed regions | +| Cluster scope | Exact deployment-owned cluster allowlist; empty configuration fails closed | +| Restricted tools | Packet-capture schemas and runtime routes are absent unless explicitly enabled | +| Instance validation | Exact paginated ACTIVE ECS container-instance membership within allowed clusters | +| Generic artifacts | `instanceId`-bound canonical keys; cross-instance, metadata, traversal, and pcap access rejected | +| Regex search | Conservative unsafe-pattern rejection plus interruptible per-file wall-clock timeout | +| Execution polling | Gateway provenance binds execution ID to tool, document, region, and instance; batch requires `batchId` | +| Capture scope | Exact task/container; ambiguous, changed-PID, host-network, host-wide, and Fargate targets rejected | +| Packet artifacts | Short pcap URL lifetime; analysis requires the exact command UUID | +| Batch safety | Dry-run default, one approval, 15-child cap, explicit partial-failure reporting | +| Instance uploads | Explicit instance-role principals receive bucket-level `ListBucket`/policy/ACL reads for the support document's `HeadBucket` preflight and object-level `PutObject`; the deploy script validates the live policy after deployment | +| Region validation | `ALLOWED_REGIONS` plus same-region approval-wrapper enforcement | +| Idempotency | Instance-scoped token mapping | +| Audit | SSM Automation/Run Command history and CloudWatch Lambda logs | + +### AppSec control mapping + +| IDs | Architecture evidence | +|---|---| +| M1–M4 | OAuth authentication; mandatory KMS/TLS; explicit least-privilege principals; native human approval with approver separation. | +| E1–E3 | Exact cluster/region boundaries; exact ACTIVE ECS membership; canonical instance-bound S3 object access. | +| E4–E6 | Interruptible regex limits; provenance-bound polling; restricted task-scoped tcpdump and dry-run/capped batch collection. | + +Executable evidence is in `tests/test_security_parity.py`, `tests/test_instance_validation.py`, `tests/test_collection_approval.py`, `tests/test_batch_approval.py`, and the tcpdump security/approval suites. + +--- + +## CDK Construct Design + +Single CDK construct (`EcsLogGatewayConstructV2`) provisions everything. Relevant properties include: +- `allowedClusterNames`: mandatory exact ECS cluster allowlist +- `ecsInstanceRoleArns`: mandatory explicit principals for S3/KMS upload access; no account fallback +- `allowedRegions`: constrains regional IAM and Lambda validation +- `enableKmsEncryption`: retained for compatibility but must be `true`; synthesis rejects disabled encryption +- `requireCollectionApproval`: defaults to `true`; disabling it is an explicit supervised/test choice +- `approvalApproverArns`, `approvalNotificationEmails`, `approvalTtlSeconds`: configure native approval +- `enableRestrictedTools`: opt-in exposure of `tcpdump_capture` and/or `tcpdump_analyze` +- `pcapPresignedUrlExpirationSeconds`, `maxPcapBytes`: limit packet-artifact exposure +- `presignedUrlExpirationSeconds`, `ssmDefaultHostRoleArn`: configure general artifacts and SSM host uploads + +When approval is enabled, the construct creates all three wrappers and grants Lambda StartAutomation only on those account-owned documents. When disabled, wrappers are omitted and direct tcpdump SendCommand is granted only if capture is explicitly enabled. Explicit instance-role principals remain mandatory in both modes. + +--- + +## Deploy Script Design + +Interactive 5-step flow matching the EKS deploy pattern: + +``` +Step 1: Region Selection + ├── 1) All enabled regions + ├── 2) Current deploy region only + └── 3) Enter a specific region + +Step 2: ECS Cluster Discovery + └── Lists all ECS clusters across selected regions + +Step 3: Cluster Selection + ├── a) All clusters + └── 1,2,5) Comma-separated picks + +Step 4: Instance Role Detection + └── Discovers IAM roles from container instance profiles + +Step 5: Role Selection + ├── a) All roles + └── 1,3) Comma-separated picks +``` + +Automation mode requires all three deployment boundaries: + +```bash +ALLOWED_CLUSTER_NAMES="prod-cluster" \ +ALLOWED_REGIONS="us-east-1" \ +ECS_INSTANCE_ROLE_ARNS="arn:aws:iam::123456789012:role/ecsInstanceRole" \ +./deploy.sh +``` + +The script never retrieves, prints, or writes the OAuth client secret. The ignored `mcp-config.txt` contains non-secret connection metadata only. diff --git a/mcp/ecs-instance-log-mcp/get-config.sh b/mcp/ecs-instance-log-mcp/get-config.sh new file mode 100755 index 0000000..523025a --- /dev/null +++ b/mcp/ecs-instance-log-mcp/get-config.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# ECS Instance Log MCP - Get Configuration Values +# Run this after deployment to retrieve all configuration values + +STACK_NAME="${1:-EcsInstanceLogMcpStack}" +REGION="${AWS_REGION:-us-east-1}" + +if [ ! -f cdk-outputs.json ]; then + echo "Error: cdk-outputs.json not found. Run ./deploy.sh first." + exit 1 +fi + +# Parse values from cdk-outputs.json +GATEWAY_URL=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'GatewayUrl' in k][0])" 2>/dev/null) +CLIENT_ID=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'CognitoClientId' in k][0])" 2>/dev/null) +USER_POOL_ID=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'CognitoUserPoolId' in k][0])" 2>/dev/null) +TOKEN_URL=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'OAuthExchangeUrl' in k][0])" 2>/dev/null) +OAUTH_SCOPE=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'OAuthScope' in k][0])" 2>/dev/null) +LOGS_BUCKET=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'LogsBucketName' in k][0])" 2>/dev/null) + +# Get Cognito Client Secret +CLIENT_SECRET=$(aws cognito-idp describe-user-pool-client \ + --user-pool-id "$USER_POOL_ID" \ + --client-id "$CLIENT_ID" \ + --region "$REGION" \ + --query "UserPoolClient.ClientSecret" \ + --output text 2>/dev/null) + +echo "" +echo "==============================================" +echo "DEVOPS AGENT MCP SERVER CONFIGURATION" +echo "==============================================" +echo "" +echo "MCP Server URL:" +echo " $GATEWAY_URL" +echo "" +echo "OAuth Client ID:" +echo " $CLIENT_ID" +echo "" +echo "OAuth Client Secret:" +echo " $CLIENT_SECRET" +echo "" +echo "Token URL:" +echo " $TOKEN_URL" +echo "" +echo "Scope (use only ONE):" +echo " $OAUTH_SCOPE" +echo "" +echo "Logs Bucket:" +echo " $LOGS_BUCKET" +echo "" +echo "==============================================" diff --git a/mcp/ecs-instance-log-mcp/package-lock.json b/mcp/ecs-instance-log-mcp/package-lock.json new file mode 100644 index 0000000..2814c10 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/package-lock.json @@ -0,0 +1,470 @@ +{ + "name": "ecs-instance-log-mcp", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ecs-instance-log-mcp", + "version": "1.0.0", + "license": "MIT-0", + "dependencies": { + "aws-cdk-lib": "^2.170.0", + "constructs": "^10.0.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "aws-cdk": "^2.170.0", + "typescript": "^5.0.0" + } + }, + "node_modules/@aws-cdk/asset-awscli-v1": { + "version": "2.2.263", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.263.tgz", + "integrity": "sha512-X9JvcJhYcb7PHs8R7m4zMablO5C9PGb/hYfLnxds9h/rKJu6l7MiXE/SabCibuehxPnuO/vk+sVVJiUWrccarQ==", + "license": "Apache-2.0" + }, + "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.0.tgz", + "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==", + "license": "Apache-2.0" + }, + "node_modules/@aws-cdk/cloud-assembly-schema": { + "version": "48.20.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-48.20.0.tgz", + "integrity": "sha512-+eeiav9LY4wbF/EFuCt/vfvi/Zoxo8bf94PW5clbMraChEliq83w4TbRVy0jB9jE0v1ooFTtIjSQkowSPkfISg==", + "bundleDependencies": [ + "jsonschema", + "semver" + ], + "license": "Apache-2.0", + "dependencies": { + "jsonschema": "~1.4.1", + "semver": "^7.7.2" + }, + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": { + "version": "7.7.2", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@types/node": { + "version": "20.19.32", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.32.tgz", + "integrity": "sha512-Ez8QE4DMfhjjTsES9K2dwfV258qBui7qxUsoaixZDiTzbde4U12e1pXGNu/ECsUIOi5/zoCxAQxIhQnaUQ2VvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/aws-cdk": { + "version": "2.1105.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.1105.0.tgz", + "integrity": "sha512-1RY2UZJv31XYobEGFHQEb7c2HXNzDbHuHqdnfdYyygvZW4Nrm8MJCW42lqItQCn+wF52Ixc7r2VR5eR4YGtVhA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "cdk": "bin/cdk" + }, + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/aws-cdk-lib": { + "version": "2.237.1", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.237.1.tgz", + "integrity": "sha512-RH8mWHLBtc14stkeUF0gFLaWdS5iS2AHUHnoT5B5LLZfEkYP9G43LjJs0AMmpdlQ5/ZQVvCZ+VB83Q9vLxILRw==", + "bundleDependencies": [ + "@balena/dockerignore", + "case", + "fs-extra", + "ignore", + "jsonschema", + "minimatch", + "punycode", + "semver", + "table", + "yaml", + "mime-types" + ], + "license": "Apache-2.0", + "dependencies": { + "@aws-cdk/asset-awscli-v1": "2.2.263", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.0", + "@aws-cdk/cloud-assembly-schema": "^48.20.0", + "@balena/dockerignore": "^1.0.2", + "case": "1.6.3", + "fs-extra": "^11.3.3", + "ignore": "^5.3.2", + "jsonschema": "^1.5.0", + "mime-types": "^2.1.35", + "minimatch": "^3.1.2", + "punycode": "^2.3.1", + "semver": "^7.7.3", + "table": "^6.9.0", + "yaml": "1.10.2" + }, + "engines": { + "node": ">= 18.0.0" + }, + "peerDependencies": { + "constructs": "^10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { + "version": "1.0.2", + "inBundle": true, + "license": "Apache-2.0" + }, + "node_modules/aws-cdk-lib/node_modules/ajv": { + "version": "8.17.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-regex": { + "version": "5.0.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-styles": { + "version": "4.3.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/astral-regex": { + "version": "2.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/balanced-match": { + "version": "1.0.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/brace-expansion": { + "version": "1.1.12", + "inBundle": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/aws-cdk-lib/node_modules/case": { + "version": "1.6.3", + "inBundle": true, + "license": "(MIT OR GPL-3.0-or-later)", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/color-convert": { + "version": "2.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/color-name": { + "version": "1.1.4", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/concat-map": { + "version": "0.0.1", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/emoji-regex": { + "version": "8.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { + "version": "3.1.3", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fast-uri": { + "version": "3.1.0", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "inBundle": true, + "license": "BSD-3-Clause" + }, + "node_modules/aws-cdk-lib/node_modules/fs-extra": { + "version": "11.3.3", + "inBundle": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/aws-cdk-lib/node_modules/graceful-fs": { + "version": "4.2.11", + "inBundle": true, + "license": "ISC" + }, + "node_modules/aws-cdk-lib/node_modules/ignore": { + "version": "5.3.2", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { + "version": "1.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/jsonfile": { + "version": "6.2.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/jsonschema": { + "version": "1.5.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { + "version": "4.4.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/mime-db": { + "version": "1.52.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/mime-types": { + "version": "2.1.35", + "inBundle": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/minimatch": { + "version": "3.1.2", + "inBundle": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/punycode": { + "version": "2.3.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/aws-cdk-lib/node_modules/require-from-string": { + "version": "2.0.2", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/semver": { + "version": "7.7.3", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/aws-cdk-lib/node_modules/slice-ansi": { + "version": "4.0.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/string-width": { + "version": "4.2.3", + "inBundle": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/strip-ansi": { + "version": "6.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/table": { + "version": "6.9.0", + "inBundle": true, + "license": "BSD-3-Clause", + "dependencies": { + "ajv": "^8.0.1", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/universalify": { + "version": "2.0.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/yaml": { + "version": "1.10.2", + "inBundle": true, + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/constructs": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.4.5.tgz", + "integrity": "sha512-fOoP70YLevMZr5avJHx2DU3LNYmC6wM8OwdrNewMZou1kZnPGOeVzBrRjZNgFDHUlulYUjkpFRSpTE3D+n+ZSg==", + "license": "Apache-2.0" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/mcp/ecs-instance-log-mcp/package.json b/mcp/ecs-instance-log-mcp/package.json new file mode 100644 index 0000000..3c6f5cf --- /dev/null +++ b/mcp/ecs-instance-log-mcp/package.json @@ -0,0 +1,37 @@ +{ + "name": "ecs-instance-log-mcp", + "version": "1.0.0", + "description": "MCP Server for DevOps Agent to collect ECS container instance logs via SSM Automation", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "test": "jest", + "cdk": "cdk", + "synth": "cdk synth", + "deploy": "cdk deploy", + "destroy": "cdk destroy" + }, + "keywords": [ + "aws", + "cdk", + "ssm", + "automation", + "mcp", + "devops-agent", + "ecs", + "troubleshooting" + ], + "author": "AWS", + "license": "MIT-0", + "devDependencies": { + "@types/node": "^20.0.0", + "aws-cdk": "^2.170.0", + "typescript": "^5.0.0" + }, + "dependencies": { + "aws-cdk-lib": "^2.170.0", + "constructs": "^10.0.0" + } +} diff --git a/mcp/ecs-instance-log-mcp/requirements-dev.txt b/mcp/ecs-instance-log-mcp/requirements-dev.txt new file mode 100644 index 0000000..5aa3a1d --- /dev/null +++ b/mcp/ecs-instance-log-mcp/requirements-dev.txt @@ -0,0 +1,4 @@ +pytest>=7.0 +hypothesis>=6.0 +boto3>=1.26 +botocore>=1.29 diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/A1-task-startup-resource-init.md b/mcp/ecs-instance-log-mcp/sops/runbooks/A1-task-startup-resource-init.md new file mode 100644 index 0000000..0cb168d --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/A1-task-startup-resource-init.md @@ -0,0 +1,78 @@ +--- +title: "A1 — Task Startup / ResourceInitializationError" +description: "Diagnose and remediate ECS tasks failing to start due to ResourceInitializationError" +status: active +severity: CRITICAL +triggers: + - "ResourceInitializationError" + - "TaskFailedToStart" + - "CannotStartContainerError" + - "CannotCreateContainerError" +owner: devops-agent +objective: "Identify the root cause of task startup failure and restore task placement" +context: "ResourceInitializationError occurs when ECS cannot set up the required resources (ENI, secrets, volumes) before starting the container. Common in awsvpc networking mode and Fargate tasks." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to get pre-indexed findings +- Use `search` tool with instanceId and query=`ResourceInitializationError|TaskFailedToStart|CannotStartContainerError` to find startup failure evidence + +SHOULD: +- Use `search` tool with query=`unable to pull secrets|failed to retrieve|ENI.*timeout` to identify the specific resource that failed +- Use `network_diagnostics` tool with instanceId to check ENI and subnet IP availability + +MAY: +- Use `cluster_health` tool with clusterName to check if multiple instances are affected +- Use `compare_instances` tool to diff healthy vs failing instances + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline around the startup failure +- Determine which resource failed: ENI provisioning, secrets retrieval, or volume mount +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`subnet|InsufficientFreeAddresses` if ENI-related +- Use `search` tool with query=`secretsmanager|ssm:GetParameters|AccessDenied` if secrets-related + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the startup failure findings +- State root cause: which resource initialization step failed and why +- Recommend immediate mitigation based on failure type + +SHOULD: +- Include timeline from correlate showing the sequence of events +- Provide specific remediation steps (e.g., add IPs to subnet, fix IAM role) + +## Guardrails + +escalation_conditions: + - "All tasks in a service failing to start" + - "Subnet IP exhaustion affecting multiple services" + - "IAM role changes needed require approval" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "IAM role changes: YELLOW — operator action" + - "Subnet/VPC changes: RED — requires approval" + +## Common Issues + +- symptoms: "ResourceInitializationError: unable to pull secrets or registry auth" + diagnosis: "Task execution role lacks permissions for Secrets Manager or SSM Parameter Store" + resolution: "Add secretsmanager:GetSecretValue or ssm:GetParameters to task execution role" + +- symptoms: "ResourceInitializationError: failed to configure ENI" + diagnosis: "Subnet has no available IP addresses or ENI limit reached" + resolution: "Add IPs to subnet or use a larger subnet CIDR" + +- symptoms: "CannotStartContainerError: exec format error" + diagnosis: "Image architecture mismatch (e.g., ARM image on x86 instance)" + resolution: "Use correct image architecture or change instance type" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/A2-task-startup-container-runtime.md b/mcp/ecs-instance-log-mcp/sops/runbooks/A2-task-startup-container-runtime.md new file mode 100644 index 0000000..0d67cc5 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/A2-task-startup-container-runtime.md @@ -0,0 +1,52 @@ +--- +title: "A2 — Task Startup / Container Runtime Failure" +description: "Diagnose ECS tasks failing due to container runtime errors (Docker/containerd)" +status: active +severity: CRITICAL +triggers: + - "ContainerRuntimeError" + - "ContainerRuntimeTimeoutError" + - "CannotCreateContainerError" +owner: devops-agent +objective: "Identify container runtime issue preventing task startup and restore service" +context: "Container runtime errors occur when Docker or containerd cannot create or start the container process. May indicate daemon issues, resource exhaustion, or configuration problems." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical to find runtime errors +- Use `search` tool with instanceId and query=`ContainerRuntimeError|docker.*daemon|containerd.*error|OCI runtime` to find runtime evidence + +SHOULD: +- Use `search` tool with query=`no space left|disk full|inode` to check disk exhaustion +- Use `search` tool with query=`docker.*restart|containerd.*restart` to check daemon restarts + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline +- Use `validate` tool to confirm docker and containerd logs are present + +SHOULD: +- Use `search` tool with query=`docker.*version|containerd.*version` to check versions + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: Docker/containerd daemon issue, disk exhaustion, or config error +- Recommend restart of container runtime or instance replacement + +## Guardrails + +escalation_conditions: + - "Container runtime down on multiple instances" + - "Disk exhaustion requiring volume resize" + +safety_ratings: + - "Log collection, search: GREEN (read-only)" + - "Docker daemon restart: YELLOW — operator action" + - "Instance replacement: RED — requires drain first" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/B1-ecr-image-pull-auth.md b/mcp/ecs-instance-log-mcp/sops/runbooks/B1-ecr-image-pull-auth.md new file mode 100644 index 0000000..133cf80 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/B1-ecr-image-pull-auth.md @@ -0,0 +1,53 @@ +--- +title: "B1 — ECR Image Pull Authentication Failure" +description: "Diagnose and remediate ECR image pull failures due to authentication or authorization" +status: active +severity: CRITICAL +triggers: + - "CannotPullECRContainerError" + - "ecr:GetAuthorizationToken.*denied" + - "ecr:BatchGetImage.*not authorized" + - "pull.*access.*denied" +owner: devops-agent +objective: "Restore ECR image pull capability by fixing authentication/authorization" +context: "ECR image pulls require the task execution role to have ecr:GetAuthorizationToken, ecr:BatchGetImage, and ecr:GetDownloadUrlForLayer permissions. Cross-account pulls need additional ECR repository policy." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical to find image pull errors +- Use `search` tool with instanceId and query=`CannotPullECRContainerError|ecr.*denied|pull.*access.*denied|authorization.*token` to find auth evidence + +SHOULD: +- Use `search` tool with query=`ecr:GetAuthorizationToken|ecr:BatchGetImage|ecr:GetDownloadUrlForLayer` to check which permission is missing +- Use `search` tool with query=`cross-account|registry.*id` to check for cross-account pull attempts + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline +- Determine if this is a permission issue (IAM) or a network issue (VPC endpoint) + +SHOULD: +- Use `network_diagnostics` tool to check if VPC endpoints for ECR are configured +- Use `search` tool with query=`vpc.*endpoint|443.*timeout` to check ECR connectivity + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: missing IAM permissions, expired token, or network connectivity +- Recommend specific IAM policy additions or VPC endpoint configuration + +## Common Issues + +- symptoms: "ecr:GetAuthorizationToken denied" + diagnosis: "Task execution role missing ECR auth permissions" + resolution: "Attach AmazonEC2ContainerRegistryReadOnly policy to task execution role" + +- symptoms: "ecr:BatchGetImage not authorized for cross-account" + diagnosis: "ECR repository policy does not allow cross-account access" + resolution: "Add cross-account principal to ECR repository policy" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/B2-image-not-found.md b/mcp/ecs-instance-log-mcp/sops/runbooks/B2-image-not-found.md new file mode 100644 index 0000000..89f044a --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/B2-image-not-found.md @@ -0,0 +1,48 @@ +--- +title: "B2 — Image Not Found / Manifest Not Found" +description: "Diagnose image pull failures due to missing image or tag" +status: active +severity: HIGH +triggers: + - "manifest.*not.*found" + - "repository.*does.*not.*exist" + - "image.*not.*found" + - "failed to resolve ref.*not found" +owner: devops-agent +objective: "Identify why the container image cannot be found and fix the reference" +context: "Image not found errors occur when the image URI, tag, or digest in the task definition does not match any image in the registry." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`manifest.*not found|repository.*not exist|image.*not found|resolve ref` to find the exact image URI that failed + +SHOULD: +- Use `search` tool with query=`image=|imageUri|container.*image` to extract the full image URI from task definition logs + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if this is a new issue or recurring +- Confirm the exact image URI and tag that failed + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: image tag deleted, typo in URI, or repository does not exist +- Recommend verifying image exists in registry + +## Common Issues + +- symptoms: "manifest for :latest not found" + diagnosis: "Image tag was overwritten or deleted from registry" + resolution: "Use immutable tags or image digests instead of :latest" + +- symptoms: "repository does not exist" + diagnosis: "ECR repository name is wrong or repository was deleted" + resolution: "Verify ECR repository name matches task definition" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/B3-docker-hub-rate-limit.md b/mcp/ecs-instance-log-mcp/sops/runbooks/B3-docker-hub-rate-limit.md new file mode 100644 index 0000000..df2875b --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/B3-docker-hub-rate-limit.md @@ -0,0 +1,42 @@ +--- +title: "B3 — Docker Hub Rate Limit" +description: "Diagnose image pull failures due to Docker Hub rate limiting" +status: active +severity: HIGH +triggers: + - "toomanyrequests.*Too Many Requests" + - "You have reached your pull rate limit" +owner: devops-agent +objective: "Mitigate Docker Hub rate limiting and restore image pulls" +context: "Docker Hub enforces pull rate limits: 100 pulls/6h for anonymous, 200 pulls/6h for authenticated. ECS tasks pulling public images can hit these limits quickly." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`toomanyrequests|rate limit|Too Many Requests` to confirm rate limiting + +SHOULD: +- Use `search` tool with query=`docker.io|hub.docker.com|registry-1.docker.io` to identify which images are from Docker Hub + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check frequency of pull attempts +- Determine if multiple tasks/services are pulling the same public images + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- Recommend migrating public images to ECR (pull-through cache or manual copy) +- Recommend configuring Docker Hub authentication for higher limits + +## Common Issues + +- symptoms: "toomanyrequests: Too Many Requests" + diagnosis: "Anonymous Docker Hub pull rate limit exceeded" + resolution: "Use ECR pull-through cache for Docker Hub images, or authenticate with Docker Hub credentials via Secrets Manager" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/C1-task-execution-role-permissions.md b/mcp/ecs-instance-log-mcp/sops/runbooks/C1-task-execution-role-permissions.md new file mode 100644 index 0000000..e93ecb5 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/C1-task-execution-role-permissions.md @@ -0,0 +1,53 @@ +--- +title: "C1 — Task Execution Role Permission Issues" +description: "Diagnose IAM permission failures for ECS task execution role" +status: active +severity: CRITICAL +triggers: + - "AccessDeniedException" + - "is not authorized to perform" + - "No valid providers in chain" + - "AssumeRoleUnauthorizedAccess" + - "execution role.*does not have" +owner: devops-agent +objective: "Identify missing IAM permissions and restore task execution" +context: "The task execution role is used by the ECS agent to pull images, retrieve secrets, and send logs. Missing permissions cause ResourceInitializationError or AccessDenied errors." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`AccessDenied|not authorized|No valid providers|execution role` to find IAM errors + +SHOULD: +- Use `search` tool with query=`ecr:GetAuthorizationToken|secretsmanager:GetSecretValue|ssm:GetParameters|logs:CreateLogStream` to identify which specific API call is denied + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine if IAM errors are the root cause or a symptom +- Identify the exact IAM action and resource that was denied + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: specific missing IAM permission +- Recommend exact IAM policy statement to add + +## Common Issues + +- symptoms: "No valid providers in chain" + diagnosis: "Task execution role ARN is invalid or role does not exist" + resolution: "Verify task execution role ARN in task definition and ensure role exists" + +- symptoms: "is not authorized to perform ecr:GetAuthorizationToken" + diagnosis: "Task execution role missing ECR permissions" + resolution: "Attach AmazonEC2ContainerRegistryReadOnly managed policy" + +- symptoms: "AccessDenied for secretsmanager:GetSecretValue" + diagnosis: "Task execution role missing Secrets Manager permissions" + resolution: "Add secretsmanager:GetSecretValue permission for the specific secret ARN" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/C2-secrets-manager-retrieval.md b/mcp/ecs-instance-log-mcp/sops/runbooks/C2-secrets-manager-retrieval.md new file mode 100644 index 0000000..3794839 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/C2-secrets-manager-retrieval.md @@ -0,0 +1,50 @@ +--- +title: "C2 — Secrets Manager / SSM Parameter Retrieval Failure" +description: "Diagnose failures retrieving secrets or parameters during task startup" +status: active +severity: CRITICAL +triggers: + - "unable to pull secrets" + - "unable to retrieve secret from asm" + - "SecretNotFound" + - "ParameterNotFound" + - "secretsmanager:GetSecretValue.*denied" + - "ssm:GetParameters.*denied" +owner: devops-agent +objective: "Restore secret/parameter retrieval for ECS task startup" +context: "ECS tasks can reference Secrets Manager secrets and SSM parameters in container definitions. Failures occur due to missing permissions, deleted secrets, or VPC endpoint issues." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`unable to pull secrets|SecretNotFound|ParameterNotFound|retrieve secret|retrieve ecr registry auth` to find secret retrieval errors + +SHOULD: +- Use `search` tool with query=`secretsmanager|ssm:GetParameters|AccessDenied` to identify permission vs not-found issues + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if secrets were recently deleted or rotated +- Determine if this is a permission issue or a resource-not-found issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: secret deleted, permission denied, or VPC endpoint missing +- Recommend specific fix + +## Common Issues + +- symptoms: "SecretNotFound" + diagnosis: "Secret ARN in task definition references a deleted or non-existent secret" + resolution: "Verify secret exists in Secrets Manager and ARN matches task definition" + +- symptoms: "unable to pull secrets or registry auth: execution resource retrieval failed" + diagnosis: "VPC endpoint for Secrets Manager not configured in private subnet" + resolution: "Create VPC endpoint for secretsmanager or ensure NAT gateway is configured" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/D1-oom-kill-memory.md b/mcp/ecs-instance-log-mcp/sops/runbooks/D1-oom-kill-memory.md new file mode 100644 index 0000000..4de3e0f --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/D1-oom-kill-memory.md @@ -0,0 +1,53 @@ +--- +title: "D1 — OOM Kill / Memory Exhaustion" +description: "Diagnose container or instance OOM kills in ECS" +status: active +severity: CRITICAL +triggers: + - "OutOfMemoryError" + - "oom.*kill" + - "Memory cgroup out of memory" + - "invoked oom-killer" + - "exit code 137" +owner: devops-agent +objective: "Identify OOM-killed process, determine memory pressure source, and prevent recurrence" +context: "OOM kills occur when a container exceeds its memory limit (hard limit) or the instance runs out of memory. Exit code 137 indicates SIGKILL from OOM killer." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical to find OOM findings +- Use `search` tool with instanceId and query=`oom-killer|OOMKilled|out of memory|Memory cgroup|exit code 137` to find OOM evidence in dmesg + +SHOULD: +- Use `search` tool with query=`Killed process.*total-vm|memory.*limit|memory.*usage` to find which process was killed and memory stats +- Use `search` tool with query=`insufficient.*memory|memory.*pressure` to check for instance-level memory pressure + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId and pivotEvent=`oom-killer` to build timeline +- Determine if OOM is at container level (cgroup limit) or instance level (system OOM) + +SHOULD: +- Use `search` tool with query=`memory.*hard.*limit|memoryReservation|memory=` to check task definition memory settings + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: container memory limit too low or memory leak +- Recommend increasing memory limit or investigating memory leak + +## Common Issues + +- symptoms: "exit code 137, Memory cgroup out of memory" + diagnosis: "Container exceeded its hard memory limit" + resolution: "Increase container memory limit in task definition, or investigate memory leak" + +- symptoms: "invoked oom-killer on instance level, multiple containers affected" + diagnosis: "Instance total memory exhausted by sum of all containers" + resolution: "Use larger instance type or reduce number of tasks per instance" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/D2-disk-space-exhaustion.md b/mcp/ecs-instance-log-mcp/sops/runbooks/D2-disk-space-exhaustion.md new file mode 100644 index 0000000..efc1fa7 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/D2-disk-space-exhaustion.md @@ -0,0 +1,41 @@ +--- +title: "D2 — Disk Space Exhaustion" +description: "Diagnose disk full errors on ECS container instances" +status: active +severity: CRITICAL +triggers: + - "no.*space.*left.*device" + - "disk.*full" +owner: devops-agent +objective: "Identify disk space consumers and restore available space" +context: "Disk exhaustion on ECS instances prevents new container creation, image pulls, and log writing. Common causes: accumulated container images, large log files, or container writable layers." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`no space left|disk full|disk.*pressure` to find disk errors + +SHOULD: +- Use `search` tool with query=`docker.*prune|image.*size|overlay.*size` to check image storage usage + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine when disk filled up +- Use `validate` tool to check if disk-related logs are present + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- Recommend: docker system prune, increase EBS volume, or configure image cleanup + +## Common Issues + +- symptoms: "no space left on device during image pull" + diagnosis: "Docker storage driver partition full from accumulated images" + resolution: "Operator: run docker system prune, or enable ECS image cleanup (ECS_IMAGE_CLEANUP_INTERVAL)" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/D3-cpu-throttling.md b/mcp/ecs-instance-log-mcp/sops/runbooks/D3-cpu-throttling.md new file mode 100644 index 0000000..a510d9d --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/D3-cpu-throttling.md @@ -0,0 +1,40 @@ +--- +title: "D3 — CPU Throttling" +description: "Diagnose CPU throttling affecting ECS container performance" +status: active +severity: HIGH +triggers: + - "cpu.*throttl" + - "insufficient.*cpu" +owner: devops-agent +objective: "Identify CPU-throttled containers and optimize resource allocation" +context: "CPU throttling occurs when containers hit their CPU limit. Unlike memory, CPU throttling doesn't kill containers but degrades performance, causing health check failures and timeouts." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`cpu.*throttl|insufficient.*cpu|cpu.*limit` to find CPU throttling evidence + +SHOULD: +- Use `search` tool with query=`health.*check.*failed|timeout|slow` to check for symptoms of CPU throttling + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if CPU throttling correlates with health check failures + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- Recommend increasing CPU units in task definition or using larger instance type + +## Common Issues + +- symptoms: "cpu throttling detected, health checks failing intermittently" + diagnosis: "Container CPU limit too low for workload" + resolution: "Increase cpu units in task definition (e.g., 256 -> 512)" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/E1-eni-allocation-subnet-ip.md b/mcp/ecs-instance-log-mcp/sops/runbooks/E1-eni-allocation-subnet-ip.md new file mode 100644 index 0000000..61e9be1 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/E1-eni-allocation-subnet-ip.md @@ -0,0 +1,49 @@ +--- +title: "E1 — ENI Allocation / Subnet IP Exhaustion" +description: "Diagnose ENI provisioning failures and subnet IP exhaustion in ECS" +status: active +severity: CRITICAL +triggers: + - "ENI.*allocation.*failed" + - "InsufficientFreeAddressesInSubnet" + - "Timeout waiting for network interface" + - "failed.*create.*network.*interface" +owner: devops-agent +objective: "Restore ENI provisioning for ECS tasks using awsvpc networking" +context: "Tasks using awsvpc mode require an ENI per task. Failures occur when the subnet runs out of IPs, ENI limits are reached, or security group limits are exceeded." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`ENI.*alloc|InsufficientFreeAddresses|network interface.*fail|Timeout.*network` to find ENI errors +- Use `network_diagnostics` tool with instanceId and sections=eni to check ENI attachment status + +SHOULD: +- Use `search` tool with query=`subnet|available.*ip|ip.*address` to check subnet capacity + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine when ENI failures started +- Use `network_diagnostics` tool with sections=eni,security-groups to get full network picture + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: subnet IP exhaustion, ENI limit, or security group limit +- Recommend adding IPs to subnet or using ENI trunking + +## Common Issues + +- symptoms: "InsufficientFreeAddressesInSubnet" + diagnosis: "Subnet CIDR too small for number of tasks" + resolution: "Use larger subnet, enable ENI trunking, or reduce task count" + +- symptoms: "Timeout waiting for network interface provisioning" + diagnosis: "ENI creation taking too long, possible API throttling" + resolution: "Check AWS service health, reduce concurrent task launches" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/E2-dns-resolution-failures.md b/mcp/ecs-instance-log-mcp/sops/runbooks/E2-dns-resolution-failures.md new file mode 100644 index 0000000..9424efb --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/E2-dns-resolution-failures.md @@ -0,0 +1,44 @@ +--- +title: "E2 — DNS Resolution Failures" +description: "Diagnose DNS resolution failures affecting ECS containers" +status: active +severity: HIGH +triggers: + - "DNS.*failed" + - "name.*resolution.*failed" + - "no.*route.*host" +owner: devops-agent +objective: "Restore DNS resolution for ECS containers" +context: "DNS failures in ECS can affect service discovery, ECR image pulls, and application connectivity. VPC DNS settings, Docker DNS config, and Route 53 resolver rules all play a role." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`DNS.*failed|resolve.*fail|SERVFAIL|NXDOMAIN|no route to host` to find DNS errors +- Use `network_diagnostics` tool with instanceId and sections=dns to check resolv.conf + +SHOULD: +- Use `search` tool with query=`nameserver|resolv.conf|search.*domain` to check DNS configuration + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if DNS failures correlate with other network issues +- Use `network_diagnostics` tool with sections=dns,routes to verify network path to DNS server + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: VPC DNS disabled, security group blocking UDP 53, or DNS server unreachable +- Recommend enabling VPC DNS or fixing security group rules + +## Common Issues + +- symptoms: "DNS resolution failed for ecr.*.amazonaws.com" + diagnosis: "VPC DNS resolution disabled or security group blocks UDP/TCP 53" + resolution: "Enable enableDnsSupport and enableDnsHostnames on VPC, check security group outbound rules" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/E3-connection-timeout.md b/mcp/ecs-instance-log-mcp/sops/runbooks/E3-connection-timeout.md new file mode 100644 index 0000000..53bae7a --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/E3-connection-timeout.md @@ -0,0 +1,45 @@ +--- +title: "E3 — Connection Timeout / Network Unreachable" +description: "Diagnose connection timeouts and network unreachable errors in ECS" +status: active +severity: HIGH +triggers: + - "connection.*timeout" + - "network.*unreachable" + - "dial.*tcp.*timeout" + - "i/o timeout" +owner: devops-agent +objective: "Identify network connectivity issue and restore communication" +context: "Connection timeouts in ECS can be caused by security group rules, NACLs, route table misconfigurations, or VPC endpoint issues." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`connection.*timeout|network.*unreachable|dial.*tcp.*timeout|i/o timeout` to find timeout errors +- Use `network_diagnostics` tool with instanceId and sections=all to get full network picture + +SHOULD: +- Use `search` tool with query=`security.*group|nacl|route.*table` to check network config + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine if timeouts are intermittent or persistent +- Use `network_diagnostics` tool with sections=routes,security-groups to check routing and SG rules + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: security group, NACL, route table, or VPC endpoint issue +- Recommend specific network configuration fix + +## Common Issues + +- symptoms: "connection timeout to 443 for ECR/S3 endpoints" + diagnosis: "Missing VPC endpoints or NAT gateway for private subnet" + resolution: "Create VPC endpoints for ECR, S3, and CloudWatch Logs, or configure NAT gateway" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/E4-human-approved-task-tcpdump.md b/mcp/ecs-instance-log-mcp/sops/runbooks/E4-human-approved-task-tcpdump.md new file mode 100644 index 0000000..3ef2be4 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/E4-human-approved-task-tcpdump.md @@ -0,0 +1,55 @@ +--- +title: "E4 — Human-Approved Task-Scoped Packet Capture" +description: "Safely capture packets inside one ECS EC2 task network namespace" +status: active +severity: HIGH +triggers: + - "packet.*capture" + - "tcpdump" + - "task.*network.*namespace" +owner: devops-agent +objective: "Collect minimum necessary packet evidence without host-wide capture" +context: "Packet data can contain credentials and customer payloads. Capture requires explicit confirmation and native SSM human approval." +--- + +## Phase 1 — Authorize and Triage + +MUST: +- Confirm the requester is authorized to inspect traffic for the exact ECS task. +- Minimize `durationSeconds` and use the narrowest safe BPF `filter`; never collect unrelated traffic. +- Verify the task uses ECS EC2 launch type. Fargate and host-network tasks are unsupported. +- Verify tcpdump is already installed on the container instance; this tool never installs packages. +- Call `tcpdump_capture` with exact `instanceId`, task ID or ARN, and `confirmCapture=true`. +- Provide `containerName` when more than one RUNNING application container is eligible. +- Share `approvalConsoleUrl` with an authorized approver; do not submit duplicate capture requests. + +## Phase 2 — Approve, Poll, and Analyze + +MUST: +- Review task, container, interface, duration, and filter in the SSM approval request before approving. +- Poll `tcpdump_capture` with `executionId` until it returns a `commandId`, then poll that command. +- Stop if approval is denied or expires; request a fresh approval only if the capture is still necessary. +- Stop if task/container resolution, PID revalidation, or namespace validation fails; never fall back to host capture. +- Call `tcpdump_analyze` with the exact `instanceId` and `commandId`; latest-capture lookup is not allowed. + +SHOULD: +- Use decoded text/statistics first and download raw pcap only when necessary. +- Treat the short-lived pcap URL and downloaded file as sensitive data. + +## Phase 3 — Report and Retain + +MUST: +- Cite `executionId`, `commandId`, `instanceId`, task ID, resolved container, and capture time. +- Separate observed packet evidence from inference and document capture gaps. +- Store or delete downloaded pcap data according to the applicable retention policy. + +## Troubleshooting + +- `pending_approval`: approve in SSM or wait; do not create another request. +- denied/expired: no capture ran; obtain fresh authorization before retrying. +- parameter pattern rejection: verify exact task ID/ARN, container name, interface, and restricted BPF syntax. +- tcpdump missing: install it through the approved node-image or configuration-management process, then retry. +- ambiguous container: provide the exact RUNNING application `containerName`. +- PID or namespace changed: the task restarted during dispatch; verify placement and request a new capture. +- host network namespace: unsupported by design because it would expose host-wide traffic. +- stale analysis warning: network conditions may have changed; request a fresh approved capture if needed. diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/F1-container-health-check.md b/mcp/ecs-instance-log-mcp/sops/runbooks/F1-container-health-check.md new file mode 100644 index 0000000..6cae373 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/F1-container-health-check.md @@ -0,0 +1,47 @@ +--- +title: "F1 — Container Health Check Failures" +description: "Diagnose ECS container health check failures" +status: active +severity: HIGH +triggers: + - "health.*check.*failed" + - "UNHEALTHY" + - "failed container health checks" +owner: devops-agent +objective: "Identify why container health checks are failing and restore healthy state" +context: "ECS container health checks (HEALTHCHECK in Dockerfile or healthCheck in task definition) mark containers as UNHEALTHY when the check command fails consecutively. This can trigger task replacement." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`health.*check.*failed|UNHEALTHY|health.*status` to find health check failures + +SHOULD: +- Use `search` tool with query=`HEALTHCHECK|healthCheck|curl.*localhost|wget` to find health check command configuration +- Use `search` tool with query=`connection.*refused|timeout|exit code` to find why the check command fails + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if health check failures correlate with resource exhaustion or network issues + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: application not ready, port mismatch, or resource exhaustion +- Recommend fixing health check command, increasing startPeriod, or fixing application + +## Common Issues + +- symptoms: "container health check failed immediately after start" + diagnosis: "Health check startPeriod too short for application startup time" + resolution: "Increase startPeriod in health check configuration" + +- symptoms: "health check curl: connection refused" + diagnosis: "Application not listening on the expected port" + resolution: "Verify containerPort matches application listen port" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/F2-elb-target-health.md b/mcp/ecs-instance-log-mcp/sops/runbooks/F2-elb-target-health.md new file mode 100644 index 0000000..c1c0e08 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/F2-elb-target-health.md @@ -0,0 +1,43 @@ +--- +title: "F2 — ELB Target Health Check Failures" +description: "Diagnose ALB/NLB target health check failures for ECS services" +status: active +severity: HIGH +triggers: + - "target.*unhealthy" + - "failed ELB health checks" + - "Instance.*port.*is unhealthy" +owner: devops-agent +objective: "Restore ELB target health for ECS service" +context: "ELB health checks are separate from container health checks. Failed ELB health checks cause targets to be deregistered, reducing service capacity. Common causes: security group rules, health check path returning non-200, or application startup time." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`target.*unhealthy|ELB health|Instance.*unhealthy|deregistering.*target` to find ELB health failures + +SHOULD: +- Use `network_diagnostics` tool with instanceId and sections=security-groups to verify port access +- Use `search` tool with query=`health.*path|health.*check.*port` to check health check configuration + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if ELB failures correlate with deployments or resource issues + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: security group blocking health check port, wrong health check path, or slow startup +- Recommend specific fix + +## Common Issues + +- symptoms: "target unhealthy, health check on port 80 returning 502" + diagnosis: "Application not ready or returning errors on health check path" + resolution: "Verify health check path returns 200, increase deregistration delay and health check grace period" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/G1-agent-disconnected.md b/mcp/ecs-instance-log-mcp/sops/runbooks/G1-agent-disconnected.md new file mode 100644 index 0000000..4f3ac24 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/G1-agent-disconnected.md @@ -0,0 +1,52 @@ +--- +title: "G1 — ECS Agent Disconnected" +description: "Diagnose ECS agent disconnection from the cluster" +status: active +severity: CRITICAL +triggers: + - "agent.*connected.*false" + - "AGENT_DISCONNECTED" + - "websocket.*unable to dial" + - "Error getting ECS instance credentials" +owner: devops-agent +objective: "Restore ECS agent connectivity to the cluster" +context: "When the ECS agent disconnects, the instance cannot receive new task placements and existing tasks may become orphaned. Common causes: network issues, IAM credential expiry, or agent crash." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`AGENT_DISCONNECTED|agent.*connected.*false|websocket.*unable|ECS Agent failed` to find agent disconnection evidence + +SHOULD: +- Use `search` tool with query=`ecs-agent.*restart|ecs-init|agent.*start` to check agent restart history +- Use `network_diagnostics` tool with instanceId to check network connectivity + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine when agent disconnected and what else happened +- Determine if this is a network issue, IAM issue, or agent crash + +SHOULD: +- Use `search` tool with query=`credential|token.*expir|sts.*assume` to check IAM credential issues + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: network connectivity, IAM credentials, or agent crash +- Recommend restarting ECS agent or replacing instance + +## Common Issues + +- symptoms: "AGENT_DISCONNECTED, websocket unable to dial" + diagnosis: "Network connectivity to ECS service endpoint lost" + resolution: "Check VPC endpoints, NAT gateway, and security group outbound rules for HTTPS (443)" + +- symptoms: "Error getting ECS instance credentials" + diagnosis: "Instance profile or IAM role issue" + resolution: "Verify instance profile is attached and has AmazonEC2ContainerServiceforEC2Role policy" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/G2-instance-registration-failure.md b/mcp/ecs-instance-log-mcp/sops/runbooks/G2-instance-registration-failure.md new file mode 100644 index 0000000..e017382 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/G2-instance-registration-failure.md @@ -0,0 +1,49 @@ +--- +title: "G2 — Instance Registration Failure" +description: "Diagnose ECS container instance registration failures" +status: active +severity: CRITICAL +triggers: + - "failed.*register.*container.*instance" + - "No container instances were found" + - "ECS Agent failed to start" + - "client version.*is too old" +owner: devops-agent +objective: "Restore instance registration with ECS cluster" +context: "Instance registration failures prevent the instance from joining the cluster. Causes include wrong cluster name in ECS config, outdated agent version, or IAM permission issues." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`failed.*register|No container instances|Agent failed to start|client version.*too old` to find registration errors + +SHOULD: +- Use `search` tool with query=`ECS_CLUSTER|ecs.config|cluster.*name` to check cluster configuration +- Use `search` tool with query=`agent.*version|ecs-init.*version` to check agent version + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of registration attempts +- Determine if this is a config issue, version issue, or permission issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: wrong cluster name, outdated agent, or missing permissions +- Recommend specific fix + +## Common Issues + +- symptoms: "failed to register container instance: cluster not found" + diagnosis: "ECS_CLUSTER in /etc/ecs/ecs.config points to wrong or non-existent cluster" + resolution: "Fix ECS_CLUSTER value in /etc/ecs/ecs.config and restart ECS agent" + +- symptoms: "client version is too old" + diagnosis: "ECS agent version incompatible with cluster features" + resolution: "Update ECS agent: sudo yum update -y ecs-init && sudo systemctl restart ecs" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/H1-cloudwatch-log-driver.md b/mcp/ecs-instance-log-mcp/sops/runbooks/H1-cloudwatch-log-driver.md new file mode 100644 index 0000000..2a7f3e4 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/H1-cloudwatch-log-driver.md @@ -0,0 +1,49 @@ +--- +title: "H1 — CloudWatch Log Driver Issues" +description: "Diagnose CloudWatch logging failures for ECS containers" +status: active +severity: HIGH +triggers: + - "log.*driver.*error" + - "failed.*send.*logs" + - "awslogs.*error" + - "failed to initialize logging driver" + - "logs:CreateLogStream.*denied" +owner: devops-agent +objective: "Restore CloudWatch log delivery for ECS containers" +context: "The awslogs log driver sends container stdout/stderr to CloudWatch Logs. Failures can be caused by missing IAM permissions, log group not existing, or network connectivity to CloudWatch endpoint." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`log.*driver.*error|awslogs.*error|CreateLogStream.*denied|initialize logging driver` to find logging errors + +SHOULD: +- Use `search` tool with query=`logs:CreateLogGroup|logs:CreateLogStream|logs:PutLogEvents` to check which permission is missing + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if logging failures cause task failures +- Determine if this is a permission issue, log group issue, or network issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: missing IAM permissions, log group not created, or VPC endpoint missing +- Recommend specific fix + +## Common Issues + +- symptoms: "failed to initialize logging driver: AccessDeniedException" + diagnosis: "Task execution role missing CloudWatch Logs permissions" + resolution: "Add logs:CreateLogGroup, logs:CreateLogStream, logs:PutLogEvents to task execution role" + +- symptoms: "awslogs error: ResourceNotFoundException" + diagnosis: "Log group does not exist and auto-create is not enabled" + resolution: "Create log group or set awslogs-create-group=true in log configuration" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/I1-deployment-circuit-breaker.md b/mcp/ecs-instance-log-mcp/sops/runbooks/I1-deployment-circuit-breaker.md new file mode 100644 index 0000000..33afd13 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/I1-deployment-circuit-breaker.md @@ -0,0 +1,48 @@ +--- +title: "I1 — Deployment Circuit Breaker Triggered" +description: "Diagnose ECS deployment failures that triggered the circuit breaker" +status: active +severity: CRITICAL +triggers: + - "deployment circuit breaker.*triggered" + - "ECS Deployment Circuit Breaker was triggered" + - "deployment circuit breaker.*rolling back" +owner: devops-agent +objective: "Identify why the deployment failed and fix the underlying issue before redeploying" +context: "The ECS deployment circuit breaker automatically rolls back a deployment when tasks repeatedly fail to reach RUNNING state. This prevents bad deployments from taking down the entire service." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`circuit breaker|rolling back|deployment.*fail|unable to place a task` to find deployment failure evidence + +SHOULD: +- Use `search` tool with query=`TaskFailedToStart|ResourceInitializationError|CannotPullContainerError` to find the underlying task failure +- Use `cluster_health` tool with clusterName to check overall cluster state + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of deployment events +- Identify the root cause: image pull failure, resource exhaustion, health check failure, or permission issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: the underlying task failure that triggered the circuit breaker +- Recommend fixing the root cause before redeploying + +## Common Issues + +- symptoms: "circuit breaker triggered, tasks failing with CannotPullContainerError" + diagnosis: "New image tag does not exist or ECR permissions changed" + resolution: "Verify image exists in registry, fix permissions, then redeploy" + +- symptoms: "circuit breaker triggered, tasks failing health checks" + diagnosis: "New application version has a bug or misconfiguration" + resolution: "Check application logs, fix the issue, then redeploy" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/J1-oci-runtime-entrypoint.md b/mcp/ecs-instance-log-mcp/sops/runbooks/J1-oci-runtime-entrypoint.md new file mode 100644 index 0000000..2be35a2 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/J1-oci-runtime-entrypoint.md @@ -0,0 +1,53 @@ +--- +title: "J1 — OCI Runtime / Entrypoint Failures" +description: "Diagnose container runtime failures related to OCI, entrypoint, or architecture mismatch" +status: active +severity: CRITICAL +triggers: + - "OCI runtime create failed" + - "exec format error" + - "no such file or directory.*entrypoint" + - "permission denied.*entrypoint" + - "container_linux.go.*starting container process" +owner: devops-agent +objective: "Fix container startup failure at the runtime level" +context: "OCI runtime errors occur at the lowest level of container creation. Common causes: wrong CPU architecture (ARM vs x86), missing entrypoint binary, or permission issues on the entrypoint script." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`OCI runtime|exec format error|entrypoint.*not found|permission denied.*entrypoint|container_linux.go` to find runtime errors + +SHOULD: +- Use `search` tool with query=`architecture|platform|amd64|arm64` to check for architecture mismatch + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if this affects all tasks or specific images +- Determine if this is an architecture mismatch, missing binary, or permission issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: architecture mismatch, missing entrypoint, or permission denied +- Recommend building correct image architecture or fixing Dockerfile + +## Common Issues + +- symptoms: "exec format error" + diagnosis: "Image built for different CPU architecture (e.g., ARM image on x86 instance)" + resolution: "Build multi-arch image or use correct platform: docker build --platform linux/amd64" + +- symptoms: "no such file or directory: /app/entrypoint.sh" + diagnosis: "Entrypoint binary missing from image" + resolution: "Verify ENTRYPOINT/CMD in Dockerfile, ensure binary is included in image" + +- symptoms: "permission denied: /app/entrypoint.sh" + diagnosis: "Entrypoint script not executable" + resolution: "Add RUN chmod +x /app/entrypoint.sh in Dockerfile" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K1-spot-interruption-instance-draining.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K1-spot-interruption-instance-draining.md new file mode 100644 index 0000000..271efd0 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K1-spot-interruption-instance-draining.md @@ -0,0 +1,81 @@ +--- +title: "K1 — Spot Interruption / Instance Draining" +description: "Diagnose and remediate ECS task disruptions caused by Spot Instance interruptions or container instance draining" +status: active +severity: HIGH +triggers: + - "SpotInterruption" + - "DRAINING" + - "instance is being terminated" + - "Spot Instance interruption notice" + - "managed instance draining" + - "ECS_ENABLE_SPOT_INSTANCE_DRAINING" +owner: devops-agent +objective: "Identify Spot interruption or draining events and ensure tasks are gracefully rescheduled" +context: "When EC2 Spot capacity is reclaimed or an instance enters DRAINING state, ECS stops scheduling new tasks on it and attempts to replace running service tasks. Standalone tasks are NOT automatically replaced. Two-minute warning for Spot terminations." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=high to find Spot/draining related errors +- Use `search` tool with instanceId and query=`SpotInterruption|DRAINING|instance.*terminat|spot.*interrupt` to find interruption evidence + +SHOULD: +- Use `search` tool with query=`ECS_ENABLE_SPOT_INSTANCE_DRAINING|managed.*draining|lifecycle.*hook` to check draining configuration +- Use `cluster_health` tool with clusterName to check if multiple instances are affected + +MAY: +- Use `compare_instances` tool to diff healthy vs draining instances +- Use `search` tool with query=`standalone.*task|PENDING.*stop` to find standalone tasks that won't auto-replace + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of draining/interruption events +- Determine if tasks were successfully rescheduled to other instances +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`replacement.*task|new.*placement|capacity.*provider` to verify replacement task launches +- Check if managed termination protection is enabled for the capacity provider + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the interruption findings +- State root cause: Spot reclamation, manual draining, or ASG lifecycle event +- Recommend enabling ECS_ENABLE_SPOT_INSTANCE_DRAINING if not configured + +SHOULD: +- Include timeline from correlate showing draining sequence +- Recommend using multiple capacity providers with Spot and On-Demand mix +- Recommend enabling managed instance draining on capacity providers + +## Guardrails + +escalation_conditions: + - "Multiple instances draining simultaneously causing capacity shortage" + - "Standalone tasks lost without replacement" + - "Service unable to place replacement tasks" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Capacity provider configuration changes: YELLOW — operator action" + - "ASG/instance type changes: RED — requires approval" + +## Common Issues + +- symptoms: "SpotInterruption: Spot capacity is no longer available" + diagnosis: "EC2 reclaimed Spot capacity in the Availability Zone" + resolution: "Use multiple AZs, diversify instance types, enable Spot Instance draining, consider On-Demand fallback" + +- symptoms: "Container instance set to DRAINING but tasks not replaced" + diagnosis: "No available capacity on other instances or standalone tasks not auto-replaced" + resolution: "Ensure sufficient capacity in other AZs, manually restart standalone tasks, enable managed instance draining" + +- symptoms: "Tasks stuck in STOPPING state during draining" + diagnosis: "stopTimeout too long or application not handling SIGTERM gracefully" + resolution: "Reduce stopTimeout in task definition, implement graceful shutdown handler in application" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K10-api-throttling-service-quotas.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K10-api-throttling-service-quotas.md new file mode 100644 index 0000000..c0d9bb5 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K10-api-throttling-service-quotas.md @@ -0,0 +1,86 @@ +--- +title: "K10 — API Throttling & Service Quota Exceeded" +description: "Diagnose and remediate ECS API throttling, rate limiting, and service quota exhaustion" +status: active +severity: HIGH +triggers: + - "throttl" + - "rate limit" + - "Rate exceeded" + - "TooManyRequestsException" + - "Limit exceeded" + - "service quota" + - "RequestLimitExceeded" + - "Operations are being throttled" +owner: devops-agent +objective: "Identify throttled APIs and quota limits, restore normal API throughput and task scheduling" +context: "ECS integrates with ELB, Cloud Map, EC2, and other services that each have independent API rate limits. Synchronous throttling returns immediate errors. Asynchronous throttling occurs when ECS invokes APIs on behalf of the user (e.g., ENI provisioning, target registration). At scale, RegisterTarget/DeregisterTarget, EC2 ENI APIs, and Cloud Map APIs are common throttle points. Service quotas limit concurrent Fargate tasks, registered instances per cluster (5000), and tasks in PROVISIONING state." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId (any active instance) to gather cluster-level logs +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find throttling errors +- Use `search` tool with instanceId and query=`throttl|rate.*limit|Rate exceeded|TooManyRequests|Limit exceeded|RequestLimitExceeded` to find evidence + +SHOULD: +- Use `search` tool with query=`Operations are being throttled|Will try again later|Cloud Map|RegisterTarget|DeregisterTarget` to identify async throttling +- Use `cluster_health` tool with clusterName to check cluster-wide impact + +MAY: +- Use `search` tool with query=`CloudTrail|ErrorCode.*Throttling|exponential.*backoff` to find CloudTrail throttle evidence +- Use `search` tool with query=`service.*quota|concurrent.*tasks|PROVISIONING.*quota` to check quota limits + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of throttling events +- Determine which API is being throttled: ECS, ELB, EC2, or Cloud Map +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`DescribeTargetHealth|CreateNetworkInterface|RegisterInstance` to identify specific throttled operations +- Use `search` tool with query=`desired.*count|running.*count|pending.*count` to assess scale of deployment + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the throttling findings +- State root cause: which API/service is throttled and current vs allowed rate +- Recommend specific remediation to reduce throttling + +SHOULD: +- Include deployment scaling recommendations (stagger deployments, reduce batch size) +- Recommend quota increase request if hard limits are hit + +## Guardrails + +escalation_conditions: + - "Sustained throttling blocking all deployments" + - "Service quota hard limit reached" + - "Multiple services affected by cascading throttle" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Deployment configuration changes: YELLOW — operator action" + - "Service quota increase request: YELLOW — requires AWS Support" + +## Common Issues + +- symptoms: "Service event: 'Operations are being throttled. Will try again later'" + diagnosis: "Cloud Map API throttling during service discovery registration at scale" + resolution: "Reduce concurrent deployments. Stagger service updates. Contact AWS Support for Cloud Map API limit increase." + +- symptoms: "Slow task launches, tasks stuck in PROVISIONING" + diagnosis: "EC2 ENI API throttling when using awsvpc network mode at scale" + resolution: "Enable ENI trunking to reduce ENI API calls. Use bridge network mode where possible. Stagger deployments." + +- symptoms: "'Limit exceeded' error when creating tasks" + diagnosis: "Fargate concurrent task quota or tasks-in-PROVISIONING-per-cluster quota exceeded" + resolution: "Delete unused task definition revisions. Request Fargate quota increase via Service Quotas console." + +- symptoms: "ELB target registration delays during deployment" + diagnosis: "RegisterTarget/DeregisterTarget API throttled with many services behind load balancers" + resolution: "Reduce number of concurrent service deployments. Use deployment circuit breaker to prevent rapid retry loops." diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K11-fargate-metadata-credential-errors.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K11-fargate-metadata-credential-errors.md new file mode 100644 index 0000000..822e178 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K11-fargate-metadata-credential-errors.md @@ -0,0 +1,87 @@ +--- +title: "K11 — Fargate Task Metadata & Credential Retrieval Errors" +description: "Diagnose and remediate task metadata endpoint failures and credential retrieval errors on Fargate" +status: active +severity: HIGH +triggers: + - "metadata" + - "credential" + - "Missing credentials" + - "could not load credentials" + - "instance metadata" + - "IMDS" + - "ECS_CONTAINER_METADATA" + - "timeout.*metadata" + - "provider chain" +owner: devops-agent +objective: "Restore task metadata endpoint access and credential retrieval for Fargate tasks" +context: "Fargate tasks use the task metadata endpoint (v3/v4) for container metadata, Docker stats, and task-level information. AWS SDK credential retrieval uses the container credential provider (169.254.170.2) injected via AWS_CONTAINER_CREDENTIALS_RELATIVE_URI. Failures occur when the metadata endpoint is unreachable, credentials expire, the task execution role is misconfigured, or network issues block the link-local address range. Intermittent failures may indicate container startup race conditions or SDK version issues." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected task/instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find credential/metadata errors +- Use `search` tool with instanceId and query=`Missing credentials|could not load credentials|metadata.*error|credential.*provider|169\.254\.170` to find evidence + +SHOULD: +- Use `search` tool with query=`ECS_CONTAINER_METADATA_URI|AWS_CONTAINER_CREDENTIALS|timeout.*metadata|IMDS` to identify metadata endpoint issues +- Use `search` tool with query=`AssumeRole|sts:AssumeRole|expired.*token|security.*token` to check credential expiry + +MAY: +- Use `search` tool with query=`SDK.*version|boto3|aws-sdk|retry.*credential` to check SDK-related issues +- Use `network_diagnostics` tool with instanceId to check network path to metadata endpoint + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of credential/metadata failures +- Determine failure type: metadata endpoint unreachable, credentials expired, or role misconfigured +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`task.*execution.*role|taskRoleArn|executionRoleArn` to verify role configuration +- Use `search` tool with query=`platform.*version|1\.4\.0|LATEST` to check platform version compatibility + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the credential/metadata findings +- State root cause: which credential/metadata mechanism failed and why +- Recommend specific remediation + +SHOULD: +- Include timeline showing when credentials started failing +- Recommend SDK upgrade if version-related + +## Guardrails + +escalation_conditions: + - "All tasks in a service unable to retrieve credentials" + - "Credential failures causing cascading application errors" + - "Metadata endpoint completely unreachable" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "IAM role changes: YELLOW — operator action" + - "Network/VPC changes: RED — requires approval" + +## Common Issues + +- symptoms: "'Missing credentials in config, or could not load credentials from any provider'" + diagnosis: "AWS SDK cannot find credentials. Task role not configured or container credential provider not available." + resolution: "Ensure taskRoleArn is set in task definition. Verify AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable is present. Upgrade AWS SDK to latest version." + +- symptoms: "Intermittent metadata errors on Fargate" + diagnosis: "Race condition during container startup — metadata endpoint not ready when application starts" + resolution: "Add retry logic with exponential backoff for metadata/credential calls at application startup. Use SDK built-in retry mechanisms." + +- symptoms: "Timeout errors from instance metadata service on Fargate" + diagnosis: "Network path to 169.254.170.2 blocked or task metadata endpoint overloaded" + resolution: "Verify task is on platform version 1.4.0+. Check that no custom iptables rules block link-local addresses. Reduce metadata polling frequency." + +- symptoms: "'Unable to retrieve instance metadata' in application logs" + diagnosis: "Application using EC2 IMDS (169.254.169.254) instead of ECS container credential provider" + resolution: "Configure application to use ECS container credential provider (AWS_CONTAINER_CREDENTIALS_RELATIVE_URI) instead of EC2 IMDS. Update SDK configuration." diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K12-essential-container-exited-nonzero.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K12-essential-container-exited-nonzero.md new file mode 100644 index 0000000..9be7318 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K12-essential-container-exited-nonzero.md @@ -0,0 +1,96 @@ +--- +title: "K12 — Essential Container Exited with Non-Zero Exit Code" +description: "Diagnose and remediate ECS tasks stopped due to essential container exit with non-zero codes" +status: active +severity: CRITICAL +triggers: + - "EssentialContainerExited" + - "exit code" + - "non-zero" + - "exit 1" + - "exit 137" + - "exit 139" + - "exit 143" + - "exit 255" + - "stopped" + - "essential container" + - "SIGKILL" + - "SIGTERM" + - "segfault" +owner: devops-agent +objective: "Identify why essential containers are exiting and restore task stability" +context: "When an essential container in an ECS task exits, the entire task is stopped. The exit code indicates the failure type: 0=normal, 1=application error, 137=OOM kill or SIGKILL (128+9), 139=segfault SIGSEGV (128+11), 143=SIGTERM (128+15), 255=container runtime error. The DescribeTasks API provides stoppedReason and container exit codes. Common causes include application crashes, OOM kills, signal handling issues, and entrypoint/command errors." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find exit-related errors +- Use `search` tool with instanceId and query=`EssentialContainerExited|exit code|non-zero|stopped.*reason|SIGKILL|SIGTERM|segfault` to find evidence + +SHOULD: +- Use `search` tool with query=`exit.*137|oom.*kill|invoked oom-killer|Memory cgroup` to check for OOM kills +- Use `search` tool with query=`exit.*139|SIGSEGV|segmentation fault|core dump` to check for segfaults +- Use `search` tool with query=`exit.*143|SIGTERM|graceful.*shutdown|signal.*15` to check for termination signals + +MAY: +- Use `search` tool with query=`exit.*1|error|exception|fatal|panic` to check for application errors +- Use `search` tool with query=`entrypoint|CMD|command.*not.*found|exec format error` to check entrypoint issues + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline around the container exit +- Determine exit code and map to failure category (OOM, signal, application error, runtime error) +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`memory.*limit|memory.*reservation|memoryReservation` to check memory configuration +- Use `search` tool with query=`health.*check|UNHEALTHY|health.*status` to check if health check failure triggered stop + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the exit code findings +- State root cause: specific exit code, signal, and triggering condition +- Recommend specific remediation based on exit code category + +SHOULD: +- Include container logs leading up to the exit +- Provide exit code reference table in summary + +## Guardrails + +escalation_conditions: + - "Essential container repeatedly exiting (crash loop)" + - "Exit code 137 (OOM) across multiple tasks" + - "Exit code 139 (segfault) indicating memory corruption" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Task definition memory/CPU changes: YELLOW — operator action" + - "Application code changes: RED — requires development team" + +## Common Issues + +- symptoms: "Task stopped: Essential container exited, exit code 137" + diagnosis: "Container killed by OOM killer — memory usage exceeded container memory limit" + resolution: "Increase memory limit in task definition. Profile application memory usage. Check for memory leaks. Set memoryReservation (soft limit) below memory (hard limit)." + +- symptoms: "Task stopped: Essential container exited, exit code 1" + diagnosis: "Application error — unhandled exception, configuration error, or dependency failure" + resolution: "Check container logs for error messages. Verify environment variables and secrets. Test container locally with same configuration." + +- symptoms: "Task stopped: Essential container exited, exit code 139" + diagnosis: "Segmentation fault (SIGSEGV) — memory corruption, null pointer, or binary incompatibility" + resolution: "Check for architecture mismatch (ARM vs x86). Update application dependencies. Enable core dumps for debugging." + +- symptoms: "Task stopped: Essential container exited, exit code 143" + diagnosis: "Container received SIGTERM — graceful shutdown requested by ECS (deployment, scaling, spot interruption)" + resolution: "Implement SIGTERM handler in application for graceful shutdown. Increase stopTimeout in task definition to allow more time for cleanup." + +- symptoms: "Task stopped: Essential container exited, exit code 255" + diagnosis: "Container runtime error — Docker/containerd failed to start or manage the container" + resolution: "Check container image validity. Verify entrypoint and CMD. Check for exec format errors (wrong architecture). Review container runtime logs." diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K13-windows-container-issues.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K13-windows-container-issues.md new file mode 100644 index 0000000..81dde86 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K13-windows-container-issues.md @@ -0,0 +1,91 @@ +--- +title: "K13 — Windows Container Issues" +description: "Diagnose and remediate Windows-specific ECS container failures including OS mismatch, IAM role bootstrap, and awslogs driver" +status: active +severity: HIGH +triggers: + - "Windows" + - "windows" + - "OS mismatch" + - "operating system does not match" + - "EnableTaskIAMRole" + - "ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE" + - "No valid providers in chain" + - "Unable to assume the role" + - "Windows Server" +owner: devops-agent +objective: "Resolve Windows-specific ECS task failures and configuration issues" +context: "Windows containers on ECS have unique requirements: the container base image OS version must match the host OS version, IAM roles for tasks require explicit bootstrap configuration (-EnableTaskIAMRole), the awslogs driver needs ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE=true, and several Linux task definition parameters are unsupported (linuxParameters, privileged, readonlyRootFilesystem, user, ulimits). Windows and Linux tasks must run in separate clusters. Windows Server 2016 is deprecated and cannot run the latest Docker version." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected Windows instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find Windows-specific errors +- Use `search` tool with instanceId and query=`operating system does not match|OS mismatch|Windows.*error|EnableTaskIAMRole|AWSLOGS_EXECUTIONROLE` to find evidence + +SHOULD: +- Use `search` tool with query=`No valid providers in chain|Unable to assume.*role|credential.*provider` to check IAM role issues +- Use `search` tool with query=`Windows Server 2016|Windows Server 2019|Windows Server 2022|Windows Server 2025` to identify OS version + +MAY: +- Use `search` tool with query=`user data|bootstrap|Initialize-ECSAgent|Set-Variable` to check instance bootstrap configuration +- Use `cluster_health` tool with clusterName to check cluster-wide Windows instance health + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of Windows-specific failures +- Determine failure category: OS mismatch, IAM bootstrap, awslogs driver, or unsupported parameter +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`container.*image|base.*image|nanoserver|servercore|ltsc` to check image OS version +- Use `search` tool with query=`awslogs|log.*driver|logConfiguration|CreateLogStream` to check logging configuration + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the Windows-specific findings +- State root cause: which Windows-specific configuration is incorrect +- Recommend specific remediation + +SHOULD: +- Include OS version compatibility matrix +- Recommend migration path if on deprecated Windows Server version + +## Guardrails + +escalation_conditions: + - "All Windows tasks failing across the cluster" + - "OS version mismatch requiring AMI update" + - "Windows Server 2016 deprecation blocking updates" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "User data / bootstrap changes: YELLOW — requires instance replacement" + - "AMI update: RED — requires approval and rolling replacement" + +## Common Issues + +- symptoms: "'The container operating system does not match the host operating system'" + diagnosis: "Container base image OS version does not match the EC2 host or Fargate platform OS version" + resolution: "Ensure container image base (e.g., ltsc2022) matches host OS. Use matching ECS-optimized Windows AMI. For Fargate, use compatible Windows platform version." + +- symptoms: "'No valid providers in chain' error on Windows tasks" + diagnosis: "ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE not set on Windows container instance" + resolution: "Add 'ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE=true' to instance user data bootstrap script." + +- symptoms: "'Unable to assume the role' on Windows EC2 tasks" + diagnosis: "IAM roles for tasks not enabled — missing -EnableTaskIAMRole in bootstrap" + resolution: "Add '-EnableTaskIAMRole' flag to Initialize-ECSAgent in instance user data. Ensure Windows instance meets IAM role configuration requirements." + +- symptoms: "Task definition validation fails with unsupported parameters" + diagnosis: "Linux-only parameters used in Windows task definition (linuxParameters, privileged, readonlyRootFilesystem, user, ulimits)" + resolution: "Remove unsupported parameters from task definition. Specify container-level CPU and memory instead of task-level for Windows EC2 tasks." + +- symptoms: "Windows container logs not appearing in CloudWatch" + diagnosis: "awslogs driver requires ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE on Windows instances" + resolution: "Set ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE=true in instance user data. Verify task execution role has logs:CreateLogStream and logs:PutLogEvents permissions." diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K14-task-latency-performance.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K14-task-latency-performance.md new file mode 100644 index 0000000..d6d23f4 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K14-task-latency-performance.md @@ -0,0 +1,94 @@ +--- +title: "K14 — Task Latency & Performance Degradation" +description: "Diagnose and remediate ECS task latency, EBS throttling, network interface throttling, and slow DNS" +status: active +severity: HIGH +triggers: + - "latency" + - "slow" + - "performance" + - "response time" + - "TargetResponseTime" + - "TTFB" + - "throttl" + - "EBS" + - "network.*throughput" + - "DNS.*slow" + - "timeout" +owner: devops-agent +objective: "Identify performance bottlenecks and restore acceptable task latency" +context: "ECS task latency can stem from multiple layers: application code, container resource limits (CPU/memory), EBS volume throttling (IOPS/throughput), network interface throttling (bandwidth/PPS limits), DNS resolution delays, load balancer configuration, or external dependency latency. For EC2 launch type, instance-level metrics (CPU, memory, network, EBS) are critical. For Fargate, container-level CloudWatch Container Insights metrics and network sidecar diagnostics help isolate the bottleneck." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=warning to find performance-related warnings +- Use `search` tool with instanceId and query=`latency|slow|timeout|performance|throttl|response.*time|TTFB` to find evidence + +SHOULD: +- Use `search` tool with query=`CPU.*utilization|memory.*utilization|cpu.*throttl|oom` to check resource saturation +- Use `search` tool with query=`EBS.*throttl|VolumeReadOps|VolumeWriteOps|BurstBalance` to check EBS throttling +- Use `network_diagnostics` tool with instanceId to check network performance + +MAY: +- Use `search` tool with query=`DNS.*resolution|resolve.*time|nslookup|dig` to check DNS latency +- Use `search` tool with query=`TargetResponseTime|HealthyHostCount|RequestCount` to check ALB metrics + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of performance degradation +- Determine bottleneck layer: CPU, memory, EBS, network, DNS, or application +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`instance.*type|vCPU|network.*bandwidth|baseline` to check instance capabilities +- Use `compare_instances` tool to compare performance across healthy vs degraded instances + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the performance findings +- State root cause: which layer is the bottleneck and specific metric evidence +- Recommend specific remediation to restore performance + +SHOULD: +- Include before/after metric comparison if baseline data available +- Recommend right-sizing based on observed resource utilization + +## Guardrails + +escalation_conditions: + - "P99 latency exceeding SLA thresholds" + - "EBS volume consistently throttled" + - "Network bandwidth saturated on instance" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Task definition resource changes: YELLOW — operator action" + - "Instance type change: YELLOW — requires rolling replacement" + +## Common Issues + +- symptoms: "High response times, TargetResponseTime spikes in ALB metrics" + diagnosis: "Application CPU or memory saturation causing slow request processing" + resolution: "Increase task CPU/memory limits. Enable Application Auto Scaling. Profile application for hot paths." + +- symptoms: "Intermittent latency spikes on EC2 launch type" + diagnosis: "EBS volume IOPS or throughput throttling — burst credits exhausted" + resolution: "Upgrade to gp3 volume with provisioned IOPS. Monitor BurstBalance metric. Reduce disk I/O or use instance store." + +- symptoms: "Network throughput degradation on EC2 instances" + diagnosis: "Instance network bandwidth limit reached or network interface PPS throttling" + resolution: "Use larger instance type with higher network bandwidth. Enable enhanced networking (ENA). Distribute traffic across more instances." + +- symptoms: "Slow DNS resolution causing connection timeouts" + diagnosis: "VPC DNS resolver throttled or DNS cache not configured" + resolution: "Enable DNS caching in application or use a local DNS cache sidecar. Check Route 53 Resolver query limits. Reduce DNS TTL churn." + +- symptoms: "Fargate task latency with no obvious resource saturation" + diagnosis: "Network interface throttling on Fargate — each task gets a single ENI with bandwidth limits" + resolution: "Use larger Fargate task size (higher vCPU = higher network bandwidth). Optimize payload sizes. Use connection pooling." diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K15-docker-daemon-agent-errors.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K15-docker-daemon-agent-errors.md new file mode 100644 index 0000000..e32b428 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K15-docker-daemon-agent-errors.md @@ -0,0 +1,88 @@ +--- +title: "K15 — Docker Daemon & Container Runtime Errors" +description: "Diagnose and remediate Docker API 500 errors, Docker daemon issues, containerd failures, and ECS agent runtime problems" +status: active +severity: CRITICAL +triggers: + - "Docker" + - "docker" + - "containerd" + - "API error 500" + - "devmapper" + - "daemon" + - "thin pool" + - "storage driver" + - "docker.sock" + - "container runtime" +owner: devops-agent +objective: "Restore Docker daemon and container runtime health on ECS container instances" +context: "Docker API 500 errors typically indicate the thin pool storage on a container instance is full, preventing new container creation. The default ECS-optimized AMI provides 8 GiB for OS and 22 GiB for images/metadata. When storage fills up, the Docker daemon cannot create containers. Other runtime issues include containerd failures, docker.sock permission errors, and stale container cleanup. The ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION agent variable controls how long stopped containers remain." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find Docker/runtime errors +- Use `search` tool with instanceId and query=`API error.*500|devmapper|thin pool|docker.*daemon|containerd.*error|docker\.sock` to find evidence + +SHOULD: +- Use `search` tool with query=`disk.*full|no space left|storage.*driver|overlay2|devicemapper` to check storage driver issues +- Use `search` tool with query=`ECS_ENGINE_TASK_CLEANUP|cleanup.*wait|stopped.*container|dead.*container` to check container cleanup + +MAY: +- Use `search` tool with query=`docker.*version|containerd.*version|runc.*version` to check runtime versions +- Use `cluster_health` tool with clusterName to check if multiple instances are affected + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of Docker/runtime failures +- Determine failure type: storage exhaustion, daemon crash, containerd failure, or permission error +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`docker.*info|Storage Driver|Data Space|Metadata Space|Thin Pool` to check Docker storage info +- Use `search` tool with query=`systemctl.*docker|service.*docker|restart.*docker|docker.*start` to check daemon restart attempts + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the Docker/runtime findings +- State root cause: storage exhaustion, daemon failure, or configuration issue +- Recommend specific remediation to restore container runtime + +SHOULD: +- Include storage utilization data +- Recommend preventive measures (cleanup duration, larger volumes, monitoring) + +## Guardrails + +escalation_conditions: + - "Docker daemon completely unresponsive" + - "All container launches failing on instance" + - "Multiple instances with same Docker error" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Docker daemon restart: YELLOW — causes brief task disruption" + - "Instance replacement: RED — requires draining and replacement" + +## Common Issues + +- symptoms: "Docker API error (500): devmapper — thin pool full" + diagnosis: "Container instance thin pool storage exhausted by accumulated images and stopped containers" + resolution: "Terminate instance and launch new one with larger data volume. Reduce ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION (default 3h). Remove unused images with 'docker image prune'. Use fstrim to reclaim space." + +- symptoms: "Cannot connect to Docker daemon at unix:///var/run/docker.sock" + diagnosis: "Docker daemon not running or crashed" + resolution: "Check Docker daemon status: systemctl status docker. Review /var/log/docker for crash logs. Restart Docker: systemctl restart docker. If persistent, replace instance." + +- symptoms: "containerd: runtime error during container creation" + diagnosis: "containerd runtime failure — may be caused by corrupted container state or resource exhaustion" + resolution: "Restart containerd service. Check system memory and disk. If persistent, drain and replace the instance." + +- symptoms: "Stale containers consuming disk space" + diagnosis: "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION too long, stopped containers accumulating" + resolution: "Set ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION to a shorter value (e.g., 15m). Manually clean: docker container prune. Monitor disk usage with CloudWatch agent." diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K2-task-placement-failures.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K2-task-placement-failures.md new file mode 100644 index 0000000..3e5c62b --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K2-task-placement-failures.md @@ -0,0 +1,90 @@ +--- +title: "K2 — Task Placement Failures" +description: "Diagnose and remediate ECS tasks failing to place due to insufficient resources, missing attributes, or constraint violations" +status: active +severity: CRITICAL +triggers: + - "no container instance met all of its requirements" + - "unable to place a task" + - "insufficient CPU" + - "insufficient memory" + - "AGENT" + - "MemberOf placement constraint unsatisfied" + - "SERVICE_TASK_PLACEMENT_FAILURE" + - "PROVISIONING" +owner: devops-agent +objective: "Identify why tasks cannot be placed and restore task scheduling" +context: "Task placement failures occur when no container instance in the cluster meets the task's CPU, memory, port, attribute, or constraint requirements. Tasks remain in PROVISIONING/PENDING state." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId (any active instance) to gather cluster-level logs +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find placement-related errors +- Use `search` tool with instanceId and query=`no container instance|unable to place|insufficient.*CPU|insufficient.*memory|placement.*constraint` to find placement failure evidence + +SHOULD: +- Use `cluster_health` tool with clusterName to check overall cluster capacity and instance count +- Use `search` tool with query=`AGENT|agent.*disconnected|agent.*not.*connected` to check for disconnected agents + +MAY: +- Use `compare_instances` tool to diff instances that can vs cannot accept tasks +- Use `search` tool with query=`port.*already.*use|required.*port` to check port conflicts + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of placement failures +- Determine the specific constraint that failed: CPU, memory, port, attribute, or placement constraint +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`attribute.*missing|ecs.capability|ecs.instance-type` to check missing attributes +- Use `search` tool with query=`desired.*count|running.*count|capacity.*provider` to check capacity vs demand + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the placement failure findings +- State root cause: which requirement could not be satisfied +- Recommend specific remediation based on failure type + +SHOULD: +- Include cluster capacity analysis showing available vs required resources +- Recommend right-sizing task definitions or adding capacity + +## Guardrails + +escalation_conditions: + - "All tasks in a service stuck in PENDING/PROVISIONING" + - "Cluster has zero registered container instances" + - "Capacity provider unable to scale out" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Task definition changes: YELLOW — operator action" + - "Instance type or ASG changes: RED — requires approval" + +## Common Issues + +- symptoms: "no container instance met all of its requirements — insufficient CPU units" + diagnosis: "Task requires more CPU than any instance has available" + resolution: "Reduce task CPU, use larger instance types, or add more instances" + +- symptoms: "no container instance met all of its requirements — insufficient memory" + diagnosis: "Task requires more memory than any instance has available" + resolution: "Reduce task memory, use larger instance types, or terminate unused tasks" + +- symptoms: "closest matching container-instance encountered error AGENT" + diagnosis: "ECS agent on the instance is disconnected" + resolution: "SSH to instance and restart ecs agent: sudo systemctl restart ecs" + +- symptoms: "MemberOf placement constraint unsatisfied" + diagnosis: "No instances match the placement constraint expression" + resolution: "Add custom attributes to instances or relax placement constraints" + +- symptoms: "closest matching container instance already uses a required port" + diagnosis: "Host port conflict — another task already bound to the required port" + resolution: "Use dynamic port mapping with ALB, or add more container instances" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K3-service-steady-state-failures.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K3-service-steady-state-failures.md new file mode 100644 index 0000000..fd1a2ec --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K3-service-steady-state-failures.md @@ -0,0 +1,87 @@ +--- +title: "K3 — Service Steady State Failures" +description: "Diagnose and remediate ECS services that fail to reach or maintain steady state" +status: active +severity: CRITICAL +triggers: + - "service.*unable to reach steady state" + - "SERVICE_TASK_START_IMPAIRED" + - "tasks failed to start" + - "deployment failed" + - "SERVICE_DEPLOYMENT_FAILED" + - "rolling back" + - "not healthy in target-group" +owner: devops-agent +objective: "Identify why a service cannot reach steady state and restore service stability" +context: "A service fails to reach steady state when tasks repeatedly fail health checks, exit with errors, cannot be placed, or fail to start. The service scheduler continuously tries to replace failed tasks, creating a restart loop." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from an affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find service-level errors +- Use `search` tool with instanceId and query=`steady state|deployment failed|tasks failed to start|SERVICE_TASK_START_IMPAIRED|rolling back` to find steady state failure evidence + +SHOULD: +- Use `search` tool with query=`health check|unhealthy|target.*not.*found|deregistered` to check health check failures +- Use `search` tool with query=`exit code|non-zero|EssentialContainerExited|OOMKilled` to check task exit reasons +- Use `cluster_health` tool with clusterName to check overall service health + +MAY: +- Use `compare_instances` tool to diff instances running healthy vs failing tasks +- Use `network_diagnostics` tool with instanceId to check network-related causes + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of service events +- Determine the specific failure mode: health check, task crash, placement, or configuration +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`HealthCheckGracePeriod|healthCheckPath|deregistration.*delay` to check health check configuration +- Use `search` tool with query=`minimumHealthyPercent|maximumPercent|desiredCount` to check deployment configuration + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the steady state failure findings +- State root cause: which component is preventing steady state +- Recommend specific remediation based on failure mode + +SHOULD: +- Include deployment timeline showing task start/stop cycles +- Recommend adjusting HealthCheckGracePeriodSeconds if tasks need more startup time + +## Guardrails + +escalation_conditions: + - "Service in continuous restart loop for more than 30 minutes" + - "All tasks in service failing simultaneously" + - "Deployment circuit breaker triggered" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Health check parameter changes: YELLOW — operator action" + - "Service/task definition changes: YELLOW — operator action" + - "Rollback to previous task definition: RED — requires approval" + +## Common Issues + +- symptoms: "service unable to reach steady state — tasks failing health checks" + diagnosis: "Application not responding on health check path within timeout" + resolution: "Increase HealthCheckGracePeriodSeconds, verify health check path returns 200, check application startup time" + +- symptoms: "service unable to reach steady state — ELB health checks failing" + diagnosis: "Security group blocking health check traffic or wrong port/path" + resolution: "Verify security group allows ALB to reach container port, confirm health check path and expected response code" + +- symptoms: "tasks exiting with non-zero exit code" + diagnosis: "Application crashing due to configuration error, missing env vars, or dependency failure" + resolution: "Check CloudWatch Logs for application errors, verify environment variables and secrets, test container locally" + +- symptoms: "SERVICE_DEPLOYMENT_FAILED with circuit breaker" + diagnosis: "Too many consecutive task failures triggered the deployment circuit breaker" + resolution: "Fix the underlying task failure, then create a new deployment with the corrected task definition" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K4-service-auto-scaling-issues.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K4-service-auto-scaling-issues.md new file mode 100644 index 0000000..60cef0f --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K4-service-auto-scaling-issues.md @@ -0,0 +1,87 @@ +--- +title: "K4 — Service Auto Scaling / Capacity Provider Issues" +description: "Diagnose and remediate ECS service auto scaling failures and capacity provider scaling problems" +status: active +severity: HIGH +triggers: + - "CapacityProviderReservation" + - "scaling policy" + - "desired count" + - "instance count discrepancy" + - "Limit exceeded" + - "InsufficientCapacity" + - "VcpuLimitExceeded" + - "managed scaling" +owner: devops-agent +objective: "Identify auto scaling or capacity provider issues and restore proper scaling behavior" +context: "ECS uses Application Auto Scaling for service task count and cluster auto scaling (capacity providers) for EC2 instance count. Failures can occur at either level — tasks not scaling, or instances not launching to support task demand." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from a container instance in the cluster +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=high to find scaling-related errors +- Use `search` tool with instanceId and query=`scaling|capacity.*provider|desired.*count|Limit exceeded|InsufficientCapacity|VcpuLimitExceeded` to find scaling failure evidence + +SHOULD: +- Use `cluster_health` tool with clusterName to check cluster capacity metrics +- Use `search` tool with query=`CapacityProviderReservation|targetCapacity|minimumScalingStepSize|maximumScalingStepSize` to check capacity provider configuration + +MAY: +- Use `search` tool with query=`CloudWatch.*alarm|target.*tracking|scaling.*policy` to check scaling policy triggers +- Use `compare_instances` tool to check instance distribution across AZs + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of scaling events +- Determine if the issue is at service level (task count) or cluster level (instance count) +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`Auto Scaling group|ASG|launch.*template|instance.*type` to check ASG configuration +- Use `search` tool with query=`service quota|rate limit|throttl` to check for API throttling + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the scaling failure findings +- State root cause: service scaling policy, capacity provider, ASG limits, or service quotas +- Recommend specific remediation + +SHOULD: +- Include scaling timeline showing demand vs capacity +- Recommend capacity provider configuration adjustments + +## Guardrails + +escalation_conditions: + - "Tasks stuck in PENDING due to no available capacity" + - "Service quota limits reached" + - "Capacity provider unable to launch instances in any AZ" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Scaling policy adjustments: YELLOW — operator action" + - "Service quota increase requests: YELLOW — operator action" + - "ASG/capacity provider changes: RED — requires approval" + +## Common Issues + +- symptoms: "Tasks stuck in PENDING, no new instances launching" + diagnosis: "Capacity provider managed scaling not enabled or ASG at MaxSize" + resolution: "Enable managed scaling on capacity provider, increase ASG MaxSize, or add instances manually" + +- symptoms: "Instance count discrepancy between ASG and ECS cluster" + diagnosis: "Instances launched but not registering with ECS cluster" + resolution: "Check ECS agent connectivity, verify instance user data sets correct cluster name, check security groups allow ECS agent traffic" + +- symptoms: "VcpuLimitExceeded error" + diagnosis: "EC2 vCPU service quota reached for the instance type family" + resolution: "Request service quota increase, use different instance types, or terminate unused instances" + +- symptoms: "Service not scaling despite high CPU/memory" + diagnosis: "Auto scaling policy not configured or CloudWatch alarm not triggering" + resolution: "Verify target tracking policy exists, check CloudWatch metrics are being published, ensure Container Insights is enabled" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K5-ecs-exec-failures.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K5-ecs-exec-failures.md new file mode 100644 index 0000000..2a80ae7 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K5-ecs-exec-failures.md @@ -0,0 +1,83 @@ +--- +title: "K5 — ECS Exec Failures" +description: "Diagnose and remediate failures when using ECS Exec to run commands in containers" +status: active +severity: MEDIUM +triggers: + - "execute command failed" + - "TargetNotConnectedException" + - "ExecuteCommandAgent" + - "SSM agent" + - "session manager" + - "ecs exec" +owner: devops-agent +objective: "Identify why ECS Exec cannot connect to a container and restore interactive access" +context: "ECS Exec uses AWS Systems Manager (SSM) Session Manager to establish connections to containers. Failures occur due to missing IAM permissions, SSM agent issues, VPC endpoint gaps, or the feature not being enabled on the service." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=medium to find exec-related errors +- Use `search` tool with instanceId and query=`execute command|ExecuteCommandAgent|TargetNotConnected|SSM.*agent|session.*manager` to find exec failure evidence + +SHOULD: +- Use `search` tool with query=`enableExecuteCommand|executeCommandConfiguration|task.*role|ssmmessages` to check ECS Exec configuration +- Use `network_diagnostics` tool with instanceId to check VPC endpoint connectivity + +MAY: +- Use `search` tool with query=`vpc.*endpoint|com.amazonaws.*ssmmessages|com.amazonaws.*ssm` to check SSM VPC endpoints +- Use `cluster_health` tool with clusterName to check if exec works on other tasks + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline around exec failures +- Determine the specific failure: IAM permissions, SSM agent, VPC endpoints, or feature not enabled +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`task.*IAM.*role|iam:PassRole|ssm:StartSession` to check IAM role configuration +- Use `search` tool with query=`managed.*agent|RUNNING|STOPPED` to check ExecuteCommandAgent status + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the exec failure findings +- State root cause: which component is preventing ECS Exec +- Recommend specific remediation steps + +SHOULD: +- Recommend running the ECS Exec Checker script for comprehensive validation +- Include IAM policy requirements for task role + +## Guardrails + +escalation_conditions: + - "ECS Exec needed for production incident debugging but unavailable" + - "SSM agent not running on any container instances" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "IAM role policy changes: YELLOW — operator action" + - "VPC endpoint creation: RED — requires approval" + +## Common Issues + +- symptoms: "The execute command failed because execute command was not enabled" + diagnosis: "ECS Exec not enabled on the service or task" + resolution: "Update service with --enable-execute-command flag, then force new deployment" + +- symptoms: "TargetNotConnectedException" + diagnosis: "SSM agent in the container cannot reach SSM endpoints" + resolution: "Create VPC endpoints for ssmmessages, ssm, and ec2messages, or ensure NAT gateway for internet access" + +- symptoms: "The execute command failed — missing permissions" + diagnosis: "Task IAM role lacks SSM permissions" + resolution: "Add ssmmessages:CreateControlChannel, ssmmessages:CreateDataChannel, ssmmessages:OpenControlChannel, ssmmessages:OpenDataChannel to task role" + +- symptoms: "ExecuteCommandAgent status is STOPPED" + diagnosis: "SSM agent crashed or container restarted" + resolution: "Force new deployment to restart tasks with fresh SSM agent, check container has enough memory for SSM agent overhead" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K6-service-connect-discovery-failures.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K6-service-connect-discovery-failures.md new file mode 100644 index 0000000..95b167e --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K6-service-connect-discovery-failures.md @@ -0,0 +1,90 @@ +--- +title: "K6 — Service Connect / Service Discovery Failures" +description: "Diagnose and remediate ECS Service Connect and AWS Cloud Map service discovery issues" +status: active +severity: HIGH +triggers: + - "service connect" + - "service discovery" + - "Cloud Map" + - "namespace" + - "SERVICE_DISCOVERY_INSTANCE_UNHEALTHY" + - "SERVICE_DISCOVERY_OPERATION_THROTTLED" + - "envoy" + - "proxy" +owner: devops-agent +objective: "Identify service-to-service communication failures and restore service discovery" +context: "ECS Service Connect uses Envoy proxy sidecars for service mesh. Service Discovery uses AWS Cloud Map DNS. Failures include namespace mismatches, proxy crashes, DNS resolution failures, and Cloud Map API throttling." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from an affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=high to find service connect/discovery errors +- Use `search` tool with instanceId and query=`service connect|service discovery|Cloud Map|namespace|envoy|proxy.*crash|UNHEALTHY` to find evidence + +SHOULD: +- Use `search` tool with query=`SERVICE_DISCOVERY_INSTANCE_UNHEALTHY|SERVICE_DISCOVERY_OPERATION_THROTTLED|port.*mapping|ingressPortOverride` to check specific failure types +- Use `network_diagnostics` tool with instanceId to check inter-service connectivity + +MAY: +- Use `search` tool with query=`DNS.*resolution|NXDOMAIN|SERVFAIL|resolve.*fail` to check DNS issues +- Use `cluster_health` tool with clusterName to check if discovery works for other services + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of discovery failures +- Determine if the issue is Service Connect (proxy), Cloud Map (DNS), or namespace configuration +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`security group|network ACL|port.*block|connection.*refused` to check network access +- Use `search` tool with query=`sidecar|ecs-service-connect|container.*definition` to check proxy container status + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the discovery failure findings +- State root cause: namespace mismatch, proxy failure, DNS issue, or permissions +- Recommend specific remediation + +SHOULD: +- Include service mesh topology showing which services can/cannot communicate +- Recommend verifying namespace configuration across all services + +## Guardrails + +escalation_conditions: + - "All inter-service communication broken" + - "Cloud Map API throttling affecting multiple services" + - "Envoy proxy crashing in a loop" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Namespace/service configuration changes: YELLOW — operator action" + - "Security group/network ACL changes: RED — requires approval" + +## Common Issues + +- symptoms: "Services cannot discover each other" + diagnosis: "Services not in the same Cloud Map namespace" + resolution: "Verify all services use the same namespace, update service configuration if needed" + +- symptoms: "SERVICE_DISCOVERY_INSTANCE_UNHEALTHY" + diagnosis: "Container health check failing, causing Cloud Map to mark instance unhealthy" + resolution: "Fix container health check, verify application responds correctly" + +- symptoms: "Service Connect proxy container crashing" + diagnosis: "Insufficient CPU/memory allocated for the Envoy sidecar" + resolution: "Add 256 CPU units and 64+ MiB memory to task definition for the proxy container" + +- symptoms: "SERVICE_DISCOVERY_OPERATION_THROTTLED" + diagnosis: "Too many Cloud Map API calls from rapid task churn" + resolution: "Reduce deployment frequency, increase task stability, contact AWS Support if persistent" + +- symptoms: "Connection refused between services using Service Connect" + diagnosis: "Security group not allowing traffic on containerPort or ingressPortOverride" + resolution: "Update security groups to allow inbound traffic from client service subnets on the Service Connect port" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K7-ebs-efs-volume-mount-failures.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K7-ebs-efs-volume-mount-failures.md new file mode 100644 index 0000000..079b85f --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K7-ebs-efs-volume-mount-failures.md @@ -0,0 +1,86 @@ +--- +title: "K7 — EBS/EFS Volume Mount Failures" +description: "Diagnose and remediate ECS task failures related to EBS volume attachment or EFS file system mounting" +status: active +severity: HIGH +triggers: + - "CannotCreateVolumeError" + - "volume mount" + - "EFS" + - "EBS" + - "mount failed" + - "file system" + - "access point" + - "transit encryption" +owner: devops-agent +objective: "Identify volume mount failures and restore persistent storage access for ECS tasks" +context: "ECS tasks can use EBS volumes (configuredAtLaunch), EFS file systems, bind mounts, and Docker volumes. Failures include IAM permission issues, security group blocking NFS traffic, EFS access point misconfiguration, and EBS attachment limits." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=high to find volume-related errors +- Use `search` tool with instanceId and query=`CannotCreateVolume|volume.*mount|EFS|EBS|mount.*fail|file.*system|access.*point` to find volume failure evidence + +SHOULD: +- Use `search` tool with query=`nfs|port 2049|security group|transit.*encryption|authorization` to check EFS connectivity +- Use `search` tool with query=`configuredAtLaunch|infrastructure.*role|ebs.*attach|volume.*type` to check EBS configuration + +MAY: +- Use `network_diagnostics` tool with instanceId to check NFS port connectivity +- Use `search` tool with query=`disk.*space|inode|throughput|burst.*credit` to check EFS performance issues + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of volume mount failures +- Determine if the issue is EBS attachment, EFS mount, permissions, or network +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`iam.*role|elasticfilesystem|ebs:CreateVolume|ebs:AttachVolume` to check IAM permissions +- Use `search` tool with query=`subnet|availability.*zone|mount.*target` to check EFS mount target availability + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the volume failure findings +- State root cause: IAM, security group, mount target, or configuration issue +- Recommend specific remediation + +SHOULD: +- Include storage architecture showing volume configuration +- Recommend security group rules for NFS (port 2049) if EFS-related + +## Guardrails + +escalation_conditions: + - "All tasks failing to mount shared EFS volume" + - "EBS volume attachment limit reached on instance" + - "Data loss risk from volume configuration changes" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Security group rule changes: YELLOW — operator action" + - "EFS/EBS configuration changes: RED — requires approval" + +## Common Issues + +- symptoms: "CannotCreateVolumeError: failed to create EBS volume" + diagnosis: "ECS infrastructure IAM role lacks ebs:CreateVolume permission or volume quota exceeded" + resolution: "Add EBS permissions to infrastructure role, check EBS volume limits in the region" + +- symptoms: "ResourceInitializationError: failed to invoke EFS utils commands" + diagnosis: "EFS mount target not available in the task's AZ or security group blocking port 2049" + resolution: "Create EFS mount target in the task's subnet AZ, allow inbound TCP 2049 from task security group" + +- symptoms: "EFS mount timeout" + diagnosis: "Security group or network ACL blocking NFS traffic between task ENI and EFS mount target" + resolution: "Allow TCP port 2049 inbound on EFS security group from task security group" + +- symptoms: "EFS access denied" + diagnosis: "EFS access point IAM authorization failing or task role missing elasticfilesystem:ClientMount" + resolution: "Add elasticfilesystem:ClientMount and ClientWrite to task role, verify access point configuration" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K8-task-stuck-pending.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K8-task-stuck-pending.md new file mode 100644 index 0000000..da395ea --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K8-task-stuck-pending.md @@ -0,0 +1,84 @@ +--- +title: "K8 — Task Stuck in PENDING State" +description: "Diagnose and remediate ECS tasks that remain in PENDING or PROVISIONING state" +status: active +severity: CRITICAL +triggers: + - "PENDING" + - "PROVISIONING" + - "stuck" + - "task not starting" + - "waiting for capacity" + - "timed out waiting" +owner: devops-agent +objective: "Identify why tasks are stuck in PENDING and restore task scheduling" +context: "Tasks stuck in PENDING/PROVISIONING indicate the scheduler cannot find suitable placement. Causes include no available instances, insufficient resources, ENI limits, subnet IP exhaustion, or capacity provider scaling delays." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId (any active instance) to gather cluster-level logs +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find pending-related errors +- Use `search` tool with instanceId and query=`PENDING|PROVISIONING|stuck|waiting.*capacity|timed out|unable to place` to find evidence + +SHOULD: +- Use `cluster_health` tool with clusterName to check cluster capacity and registered instances +- Use `search` tool with query=`ENI.*limit|network.*interface|subnet.*IP|InsufficientFreeAddresses` to check ENI/IP exhaustion + +MAY: +- Use `search` tool with query=`capacity.*provider|managed.*scaling|Auto Scaling|launch.*template` to check scaling pipeline +- Use `compare_instances` tool to check resource availability across instances + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of pending events +- Determine the bottleneck: CPU, memory, ENI, IP addresses, or instance availability +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`desired.*capacity|running.*count|pending.*count` to check demand vs supply +- Use `search` tool with query=`awsvpc|bridge|host|network.*mode` to check if awsvpc mode is causing ENI limits + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the pending task findings +- State root cause: which resource is the bottleneck +- Recommend specific remediation to unblock task scheduling + +SHOULD: +- Include capacity analysis showing available vs required resources +- Recommend ENI trunking if awsvpc mode is hitting ENI limits + +## Guardrails + +escalation_conditions: + - "Tasks stuck in PENDING for more than 15 minutes" + - "All new deployments blocked" + - "Capacity provider unable to scale out" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Instance type or count changes: YELLOW — operator action" + - "Subnet/VPC changes: RED — requires approval" + +## Common Issues + +- symptoms: "Tasks stuck in PROVISIONING — Fargate" + diagnosis: "Fargate capacity not available in the selected AZ or subnet has no IPs" + resolution: "Add subnets in multiple AZs, ensure subnets have available IP addresses" + +- symptoms: "Tasks stuck in PENDING — EC2 launch type" + diagnosis: "No container instances with sufficient CPU/memory" + resolution: "Add instances, use larger instance types, or enable capacity provider managed scaling" + +- symptoms: "Tasks stuck due to ENI limit on EC2 instance" + diagnosis: "awsvpc network mode requires one ENI per task, instance ENI limit reached" + resolution: "Enable ENI trunking (ECS_AWSVPC_TRUNKING=true), use instances with higher ENI limits, or switch to bridge network mode" + +- symptoms: "Task timed out waiting for capacity" + diagnosis: "Capacity provider scaling too slow or instances failing to register" + resolution: "Reduce instanceWarmupPeriod, check instance user data for correct cluster name, verify ECS agent connectivity" diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/K9-fargate-platform-ephemeral-storage.md b/mcp/ecs-instance-log-mcp/sops/runbooks/K9-fargate-platform-ephemeral-storage.md new file mode 100644 index 0000000..71c3c95 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/K9-fargate-platform-ephemeral-storage.md @@ -0,0 +1,88 @@ +--- +title: "K9 — Fargate Platform Version & Ephemeral Storage Issues" +description: "Diagnose and remediate Fargate platform version incompatibilities and ephemeral storage exhaustion" +status: active +severity: HIGH +triggers: + - "platform version" + - "ephemeral storage" + - "disk space" + - "no space left" + - "platform 1.3" + - "platform 1.4" + - "LATEST" + - "storage exceeded" + - "EFS" + - "volume mount" +owner: devops-agent +objective: "Resolve Fargate platform version mismatches and ephemeral storage exhaustion" +context: "Fargate tasks on platform version 1.4.0+ receive 20 GiB ephemeral storage (expandable to 200 GiB). Older platform versions (1.3.0 and earlier) only get 10 GiB for Docker layers plus 4 GiB for volume mounts. Features like EFS, Secrets Manager injection, task metadata v4, and containerd runtime require platform 1.4.0+. Using LATEST resolves to the newest version but explicit pinning can cause drift." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected task/instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find storage or platform errors +- Use `search` tool with instanceId and query=`no space left|disk.*full|ephemeral.*storage|platform.*version|LATEST` to find evidence + +SHOULD: +- Use `search` tool with query=`EFS|volume.*mount|bind.*mount|storage.*exceeded` to check volume-related failures +- Use `search` tool with query=`containerd|docker|Fargate.*agent|platform.*1\.[0-3]` to detect old platform version symptoms + +MAY: +- Use `cluster_health` tool with clusterName to check if multiple tasks are affected +- Use `search` tool with query=`encryption|KMS|AES-256` to verify ephemeral storage encryption status + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of storage/platform events +- Determine whether the issue is ephemeral storage exhaustion or platform version incompatibility +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`ephemeralStorage|sizeInGiB|20.*GiB|200.*GiB` to check configured storage +- Use `search` tool with query=`task metadata|ECS_CONTAINER_METADATA_URI` to verify metadata endpoint availability + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the storage/platform findings +- State root cause: ephemeral storage limit hit or platform version too old for required features +- Recommend specific remediation + +SHOULD: +- Include storage utilization data if available +- Recommend platform version upgrade path if on older version + +## Guardrails + +escalation_conditions: + - "All Fargate tasks failing due to storage exhaustion" + - "Platform version change required across multiple services" + - "EFS mount failures blocking critical workloads" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Task definition ephemeralStorage update: YELLOW — operator action" + - "Platform version change: YELLOW — requires deployment" + +## Common Issues + +- symptoms: "Task fails with 'no space left on device' on Fargate" + diagnosis: "Default 20 GiB ephemeral storage exhausted by large container images or runtime data" + resolution: "Increase ephemeralStorage in task definition (up to 200 GiB). Reduce container image size. Clean temp files in entrypoint." + +- symptoms: "EFS volume mount fails on Fargate" + diagnosis: "Task using platform version older than 1.4.0 which does not support EFS" + resolution: "Update service to use platform version 1.4.0 or LATEST. Verify EFS security group allows NFS (port 2049) from task security group." + +- symptoms: "Secrets Manager injection fails on Fargate" + diagnosis: "Platform version 1.3.0 or earlier does not support Secrets Manager environment variable injection" + resolution: "Upgrade to platform version 1.4.0 or LATEST. Ensure task execution role has secretsmanager:GetSecretValue permission." + +- symptoms: "Task metadata endpoint v4 not available" + diagnosis: "Platform version older than 1.4.0 only supports metadata endpoint v3" + resolution: "Upgrade to platform version 1.4.0 or LATEST for ECS_CONTAINER_METADATA_URI_V4 support." diff --git a/mcp/ecs-instance-log-mcp/sops/runbooks/Z1-general-troubleshooting.md b/mcp/ecs-instance-log-mcp/sops/runbooks/Z1-general-troubleshooting.md new file mode 100644 index 0000000..dc89c53 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/sops/runbooks/Z1-general-troubleshooting.md @@ -0,0 +1,55 @@ +--- +title: "Z1 — General ECS Troubleshooting" +description: "General troubleshooting guide for ECS issues that don't match specific SOPs" +status: active +severity: INFO +triggers: + - "general" + - "unknown" +owner: devops-agent +objective: "Provide a systematic approach to diagnosing unclassified ECS issues" +context: "Use this SOP when the issue doesn't match any specific category. Follow the systematic approach to narrow down the problem." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `validate` tool with instanceId to verify log bundle completeness +- Use `errors` tool with instanceId and severity=all to get full error summary +- Use `cluster_health` tool with clusterName to check overall cluster state + +SHOULD: +- Use `search` tool with instanceId and query=`error|fail|denied|timeout` for broad error search +- Use `network_diagnostics` tool with instanceId and sections=all for network overview + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build event timeline +- Review findings by component to identify the affected subsystem +- Use `compare_instances` tool if healthy instances are available for comparison + +SHOULD: +- Use `search` tool with targeted queries based on findings from Phase 1 + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the most relevant findings +- Document what was found and what was ruled out +- Recommend next steps for further investigation + +## Systematic Approach + +1. Check ECS agent connectivity (G1, G2 SOPs) +2. Check task startup (A1, A2 SOPs) +3. Check image pulls (B1, B2, B3 SOPs) +4. Check IAM/secrets (C1, C2 SOPs) +5. Check resources (D1, D2, D3 SOPs) +6. Check networking (E1, E2, E3 SOPs) +7. Check health checks (F1, F2 SOPs) +8. Check logging (H1 SOP) +9. Check deployments (I1 SOP) +10. Check container runtime (J1 SOP) diff --git a/mcp/ecs-instance-log-mcp/src/ecs-log-gateway-construct-v2.ts b/mcp/ecs-instance-log-mcp/src/ecs-log-gateway-construct-v2.ts new file mode 100644 index 0000000..dab353a --- /dev/null +++ b/mcp/ecs-instance-log-mcp/src/ecs-log-gateway-construct-v2.ts @@ -0,0 +1,1784 @@ +import * as cdk from 'aws-cdk-lib'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as cognito from 'aws-cdk-lib/aws-cognito'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import * as s3n from 'aws-cdk-lib/aws-s3-notifications'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import * as s3deploy from 'aws-cdk-lib/aws-s3-deployment'; +import * as ssm from 'aws-cdk-lib/aws-ssm'; +import * as sns from 'aws-cdk-lib/aws-sns'; +import * as subscriptions from 'aws-cdk-lib/aws-sns-subscriptions'; +import * as path from 'path'; +import { Construct } from 'constructs'; + +export interface EcsLogGatewayV2Props { + readonly gatewayName?: string; + readonly cognitoUserPoolName?: string; + readonly resourceServerName?: string; + readonly logRetentionDays?: number; + readonly enableKmsEncryption?: boolean; + readonly sopBucketName?: string; + + /** + * Exact ECS cluster names this deployment may inspect or mutate. + * This property is mandatory and must contain at least one valid ECS name. + */ + readonly allowedClusterNames?: string[]; + + /** + * Explicit list of AWS regions permitted for cross-region SSM execution. + * Used for IAM aws:RequestedRegion conditions and Lambda ALLOWED_REGIONS env var. + * @default [Stack region] + */ + readonly allowedRegions?: string[]; + + /** + * IAM role ARNs for ECS container instances that need KMS encrypt and S3 PutObject access. + * At least one explicit role is required; no account-wide compatibility fallback exists. + */ + readonly ecsInstanceRoleArns?: string[]; + + /** + * Presigned URL expiration in seconds for all S3 presigned URLs generated by the Lambda. + * @default 900 + */ + readonly presignedUrlExpirationSeconds?: number; + + /** + * IAM role ARN for the SSM Default Host Management Configuration role. + * This role needs S3 PutObject and KMS Encrypt to upload collected logs. + * @default undefined + */ + readonly ssmDefaultHostRoleArn?: string; + + /** Require native SSM human approval before collection and tcpdump execution. @default true */ + readonly requireCollectionApproval?: boolean; + + /** IAM user/role ARNs allowed to approve. Required when approval is enabled. */ + readonly approvalApproverArns?: string[]; + + /** Email subscriptions created on the approval SNS topic. @default [] */ + readonly approvalNotificationEmails?: string[]; + + /** Approval wait timeout in seconds. @default 900 */ + readonly approvalTtlSeconds?: number; + + /** Restricted MCP tools to expose, for example tcpdump_capture and tcpdump_analyze. @default [] */ + readonly enableRestrictedTools?: string[]; + + /** Presigned URL expiration for packet captures. @default 60 */ + readonly pcapPresignedUrlExpirationSeconds?: number; + + /** Advisory maximum packet capture size in bytes. @default 209715200 */ + readonly maxPcapBytes?: number; +} + +export class EcsLogGatewayConstructV2 extends Construct { + public readonly logsBucket: s3.Bucket; + public readonly kmsKey: kms.Key; + public readonly ssmAutomationFunction: lambda.Function; + public readonly unzipFunction: lambda.Function; + public readonly findingsIndexerFunction: lambda.Function; + public readonly userPool: cognito.UserPool; + public readonly userPoolClient: cognito.UserPoolClient; + public readonly ssmAutomationRole: iam.Role; + public readonly gatewayExecutionRole: iam.Role; + public readonly sopBucket: s3.Bucket; + public readonly collectionApprovalTopic: sns.Topic; + + constructor(scope: Construct, id: string, props: EcsLogGatewayV2Props = {}) { + super(scope, id); + + const gatewayName = props.gatewayName ?? 'EcsInstanceLogMcpGW'; + const cognitoUserPoolName = props.cognitoUserPoolName ?? 'ecs-log-gateway-pool'; + const resourceServerName = props.resourceServerName ?? 'ecs-log-gateway-id'; + const logRetentionDays = props.logRetentionDays ?? 1; + const enableKmsEncryption = props.enableKmsEncryption ?? true; + const requireCollectionApproval = props.requireCollectionApproval ?? true; + const approvalTtlSeconds = props.approvalTtlSeconds ?? 900; + const approverArns = props.approvalApproverArns ?? []; + const enabledRestrictedTools = props.enableRestrictedTools ?? []; + const allowedClusterNames = Array.from(new Set( + (props.allowedClusterNames ?? []).map(value => value.trim()).filter(Boolean), + )); + const ecsInstanceRoleArns = Array.from(new Set( + (props.ecsInstanceRoleArns ?? []).map(value => value.trim()).filter(Boolean), + )); + + if (!enableKmsEncryption) { + throw new Error('EcsLogGatewayV2: KMS encryption is mandatory and cannot be disabled.'); + } + if (allowedClusterNames.length === 0 || allowedClusterNames.some( + name => !/^[A-Za-z0-9_-]{1,255}$/.test(name), + )) { + throw new Error( + 'EcsLogGatewayV2: `allowedClusterNames` must contain at least one valid ECS cluster name.', + ); + } + const stack = cdk.Stack.of(this); + const stackAccount = stack.account; + if (ecsInstanceRoleArns.length === 0 || ecsInstanceRoleArns.some(arn => { + const match = /^arn:(aws|aws-us-gov|aws-cn):iam::(\d{12}):role\/[A-Za-z0-9+=,.@_\/-]{1,512}$/.exec(arn); + return !match || (!cdk.Token.isUnresolved(stackAccount) && match[2] !== stackAccount); + })) { + throw new Error( + 'EcsLogGatewayV2: `ecsInstanceRoleArns` must contain at least one explicit IAM role ARN from the stack account.', + ); + } + + if (requireCollectionApproval && ( + approverArns.length === 0 || approverArns.some(arn => { + const match = /^arn:(aws|aws-us-gov|aws-cn):iam::(\d{12}):(user|role)\/[A-Za-z0-9+=,.@_\/-]{1,512}$/.exec(arn); + return !match || (!cdk.Token.isUnresolved(stackAccount) && match[2] !== stackAccount); + }) + )) { + throw new Error( + 'EcsLogGatewayV2: approval is enabled but `approvalApproverArns` does not contain ' + + 'a valid IAM user or role ARN from the stack account. Set APPROVAL_APPROVER_ARNS, ' + + 'or explicitly set REQUIRE_COLLECTION_APPROVAL=false for a supervised/test deployment.', + ); + } + + const configuredRegions = Array.from(new Set( + (props.allowedRegions ?? []).map(value => value.trim()).filter(Boolean), + )); + const allowedRegions = configuredRegions.length > 0 ? configuredRegions : [stack.region]; + if (allowedRegions.some(region => ( + !cdk.Token.isUnresolved(region) && !/^[a-z]{2}(?:-[a-z0-9]+)+-\d+$/.test(region) + ))) { + throw new Error('EcsLogGatewayV2: `allowedRegions` contains an invalid AWS region.'); + } + if ( + requireCollectionApproval + && !cdk.Token.isUnresolved(stack.region) + && !allowedRegions.includes(stack.region) + ) { + throw new Error( + 'EcsLogGatewayV2: `allowedRegions` must include the stack region when approval is enabled.', + ); + } + + const partition = stack.partition; + const collectApprovalDocName = `${stack.stackName}-collect-with-approval`; + const batchApprovalDocName = `${stack.stackName}-batch-collect-with-approval`; + const tcpdumpApprovalDocName = `${stack.stackName}-tcpdump-with-approval`; + + // ======================================================================== + // KMS KEY (Mandatory) + // ======================================================================== + this.kmsKey = new kms.Key(this, 'LogsEncryptionKey', { + alias: `${cdk.Stack.of(this).stackName}-logs-key`, + description: 'KMS key for ECS log encryption', + enableKeyRotation: true, + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + + this.kmsKey.addToResourcePolicy(new iam.PolicyStatement({ + sid: 'AllowECSInstanceRolesEncrypt', + effect: iam.Effect.ALLOW, + principals: ecsInstanceRoleArns.map(arn => new iam.ArnPrincipal(arn)), + actions: ['kms:GenerateDataKey', 'kms:Encrypt'], + resources: ['*'], + })); + + if (props.ssmDefaultHostRoleArn) { + this.kmsKey.addToResourcePolicy(new iam.PolicyStatement({ + sid: 'AllowSSMDefaultHostRoleEncrypt', + effect: iam.Effect.ALLOW, + principals: [new iam.ArnPrincipal(props.ssmDefaultHostRoleArn)], + actions: ['kms:GenerateDataKey', 'kms:Encrypt'], + resources: ['*'], + })); + } + + // ======================================================================== + // IAM ROLES + // ======================================================================== + this.ssmAutomationRole = new iam.Role(this, 'SSMAutomationRole', { + roleName: `${cdk.Stack.of(this).stackName}-ssm-automation-role`, + assumedBy: new iam.ServicePrincipal('ssm.amazonaws.com'), + }); + + const supportAutomationResources = allowedRegions.flatMap(region => [ + `arn:${partition}:ssm:${region}::automation-definition/AWSSupport-CollectECSInstanceLogs:*`, + `arn:${partition}:ssm:${region}::document/AWSSupport-CollectECSInstanceLogs`, + `arn:${partition}:ssm:${region}:${stack.account}:automation-execution/*`, + ]); + const runShellDocumentResources = allowedRegions.map( + region => `arn:${partition}:ssm:${region}::document/AWS-RunShellScript`, + ); + const allowedInstanceResources = allowedRegions.map( + region => `arn:${partition}:ec2:${region}:${stack.account}:instance/*`, + ); + const requestedRegionCondition = { + StringEquals: { 'aws:RequestedRegion': allowedRegions }, + }; + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + sid: 'StartSupportCollectionAutomation', + effect: iam.Effect.ALLOW, + actions: ['ssm:StartAutomationExecution'], + resources: supportAutomationResources, + })); + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + sid: 'SendRunShellScriptDocument', + effect: iam.Effect.ALLOW, + actions: ['ssm:SendCommand'], + resources: runShellDocumentResources, + })); + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + sid: 'SendCommandToAllowedRegionInstances', + effect: iam.Effect.ALLOW, + actions: ['ssm:SendCommand'], + resources: allowedInstanceResources, + })); + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + sid: 'ReadAutomationExecutions', + effect: iam.Effect.ALLOW, + actions: ['ssm:GetAutomationExecution', 'ssm:StopAutomationExecution'], + resources: allowedRegions.map( + region => `arn:${partition}:ssm:${region}:${stack.account}:automation-execution/*`, + ), + })); + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + sid: 'RegionalSsmReadAndCommandControl', + effect: iam.Effect.ALLOW, + actions: [ + 'ssm:DescribeAutomationExecutions', 'ssm:GetCommandInvocation', + 'ssm:ListCommandInvocations', 'ssm:ListCommands', 'ssm:CancelCommand', + 'ssm:DescribeInstanceInformation', + ], + resources: ['*'], + conditions: requestedRegionCondition, + })); + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + sid: 'ReadRequiredSsmDocuments', + effect: iam.Effect.ALLOW, + actions: ['ssm:GetDocument', 'ssm:DescribeDocument'], + resources: [ + ...allowedRegions.map( + region => `arn:${partition}:ssm:${region}::document/AWSSupport-CollectECSInstanceLogs`, + ), + ...runShellDocumentResources, + ], + })); + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + sid: 'RegionalEcsInstanceRead', + effect: iam.Effect.ALLOW, + actions: [ + 'ec2:DescribeInstances', 'ec2:DescribeInstanceStatus', + 'ecs:DescribeClusters', + 'ecs:DescribeContainerInstances', 'ecs:ListContainerInstances', + ], + resources: ['*'], + conditions: requestedRegionCondition, + })); + + // ======================================================================== + // S3 BUCKET + // ======================================================================== + const bucketProps: s3.BucketProps = { + bucketName: `${cdk.Stack.of(this).stackName.toLowerCase()}-logs-${cdk.Stack.of(this).account}`, + versioned: true, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + enforceSSL: true, + lifecycleRules: [ + { + id: 'DeleteOldLogsAndVersions', + enabled: true, + expiration: cdk.Duration.days(logRetentionDays), + noncurrentVersionExpiration: cdk.Duration.days(Math.max(logRetentionDays + 1, 2)), + abortIncompleteMultipartUploadAfter: cdk.Duration.days(1), + }, + { + id: 'DeleteExpiredObjectMarkers', + enabled: true, + expiredObjectDeleteMarker: true, + }, + { + id: 'DeleteOldIdempotencyMappings', + enabled: true, + prefix: 'idempotency/', + expiration: cdk.Duration.days(7), + noncurrentVersionExpiration: cdk.Duration.days(8), + }, + { + id: 'DeleteExecutionAndBatchMetadata', + enabled: true, + prefix: '_metadata/', + expiration: cdk.Duration.days(7), + noncurrentVersionExpiration: cdk.Duration.days(8), + }, + { id: 'ExpireOldBaselines', enabled: true, prefix: 'baselines/', expiration: cdk.Duration.days(90) }, + { id: 'ExpireTcpdumpArtifacts', enabled: true, prefix: 'tcpdump/', expiration: cdk.Duration.days(1), noncurrentVersionExpiration: cdk.Duration.days(2) }, + ], + encryption: s3.BucketEncryption.KMS, + encryptionKey: this.kmsKey, + bucketKeyEnabled: true, + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }; + + this.logsBucket = new s3.Bucket(this, 'LogsBucket', bucketProps); + this.logsBucket.grantReadWrite(this.ssmAutomationRole); + this.kmsKey.grantEncryptDecrypt(this.ssmAutomationRole); + + const uploadPrincipals: iam.IPrincipal[] = ecsInstanceRoleArns.map( + arn => new iam.ArnPrincipal(arn), + ); + + // Add SSM Default Host Management role if provided + if (props.ssmDefaultHostRoleArn) { + uploadPrincipals.push(new iam.ArnPrincipal(props.ssmDefaultHostRoleArn)); + } + + this.logsBucket.addToResourcePolicy(new iam.PolicyStatement({ + sid: 'AllowEC2InstancesBucketPreflight', + effect: iam.Effect.ALLOW, + principals: uploadPrincipals, + actions: ['s3:GetBucketPolicyStatus', 's3:GetBucketAcl', 's3:ListBucket'], + resources: [this.logsBucket.bucketArn], + conditions: { StringEquals: { 'aws:PrincipalAccount': cdk.Stack.of(this).account } }, + })); + + this.logsBucket.addToResourcePolicy(new iam.PolicyStatement({ + sid: 'AllowEC2InstancesUpload', + effect: iam.Effect.ALLOW, + principals: uploadPrincipals, + actions: ['s3:PutObject'], + resources: [`${this.logsBucket.bucketArn}/*`], + conditions: { StringEquals: { 'aws:PrincipalAccount': cdk.Stack.of(this).account } }, + })); + + // ======================================================================== + // FINDINGS INDEXER LAMBDA (triggered by Unzip after extraction) + // ======================================================================== + const findingsIndexerRole = new iam.Role(this, 'FindingsIndexerRole', { + roleName: `${cdk.Stack.of(this).stackName}-findings-indexer-role`, + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')], + }); + this.logsBucket.grantReadWrite(findingsIndexerRole); + if (this.kmsKey) { this.kmsKey.grantEncryptDecrypt(findingsIndexerRole); } + + this.findingsIndexerFunction = new lambda.Function(this, 'FindingsIndexerFunction', { + functionName: `${cdk.Stack.of(this).stackName}-findings-indexer`, + runtime: lambda.Runtime.PYTHON_3_11, + handler: 'index.lambda_handler', + role: findingsIndexerRole, + timeout: cdk.Duration.minutes(5), + memorySize: 1024, + environment: { LOGS_BUCKET_NAME: this.logsBucket.bucketName }, + code: lambda.Code.fromInline(this.getFindingsIndexerCode()), + logRetention: logs.RetentionDays.TWO_WEEKS, + }); + + // ======================================================================== + // UNZIP LAMBDA (triggers Findings Indexer after extraction) + // ======================================================================== + const unzipLambdaRole = new iam.Role(this, 'UnzipLambdaRole', { + roleName: `${cdk.Stack.of(this).stackName}-unzip-lambda-role`, + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')], + }); + this.logsBucket.grantReadWrite(unzipLambdaRole); + if (this.kmsKey) { this.kmsKey.grantEncryptDecrypt(unzipLambdaRole); } + this.findingsIndexerFunction.grantInvoke(unzipLambdaRole); + + this.unzipFunction = new lambda.Function(this, 'UnzipFunction', { + functionName: `${cdk.Stack.of(this).stackName}-unzip-function`, + runtime: lambda.Runtime.PYTHON_3_11, + handler: 'index.lambda_handler', + role: unzipLambdaRole, + timeout: cdk.Duration.minutes(5), + memorySize: 1024, + environment: { FINDINGS_INDEXER_FUNCTION: this.findingsIndexerFunction.functionName }, + code: lambda.Code.fromInline(this.getUnzipLambdaCode()), + logRetention: logs.RetentionDays.TWO_WEEKS, + }); + + this.logsBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.LambdaDestination(this.unzipFunction), { suffix: '.zip' }); + this.logsBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.LambdaDestination(this.unzipFunction), { suffix: '.tar.gz' }); + this.logsBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.LambdaDestination(this.unzipFunction), { suffix: '.tgz' }); + + // ======================================================================== + // SSM AUTOMATION LAMBDA + // ======================================================================== + const lambdaExecutionRole = new iam.Role(this, 'LambdaExecutionRole', { + roleName: `${cdk.Stack.of(this).stackName}-lambda-execution-role`, + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')], + }); + + const lambdaStartDocumentNames = requireCollectionApproval + ? [collectApprovalDocName, batchApprovalDocName, tcpdumpApprovalDocName] + : ['AWSSupport-CollectECSInstanceLogs']; + const lambdaStartAutomationResources = allowedRegions.flatMap(region => [ + ...lambdaStartDocumentNames.flatMap(documentName => requireCollectionApproval + ? [ + `arn:${partition}:ssm:${region}:${stack.account}:automation-definition/${documentName}:*`, + `arn:${partition}:ssm:${region}:${stack.account}:document/${documentName}`, + ] + : [ + `arn:${partition}:ssm:${region}::automation-definition/${documentName}:*`, + `arn:${partition}:ssm:${region}::document/${documentName}`, + ]), + `arn:${partition}:ssm:${region}:${stack.account}:automation-execution/*`, + ]); + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + sid: 'StartAllowedAutomationDocuments', + effect: iam.Effect.ALLOW, + actions: ['ssm:StartAutomationExecution'], + resources: lambdaStartAutomationResources, + })); + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + sid: 'ReadAutomationExecutionStatus', + effect: iam.Effect.ALLOW, + actions: ['ssm:GetAutomationExecution'], + resources: allowedRegions.map( + region => `arn:${partition}:ssm:${region}:${stack.account}:automation-execution/*`, + ), + })); + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + sid: 'RegionalSsmRead', + effect: iam.Effect.ALLOW, + actions: ['ssm:DescribeAutomationExecutions', 'ssm:DescribeInstanceInformation', + 'ssm:GetCommandInvocation'], + resources: ['*'], + conditions: requestedRegionCondition, + })); + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + sid: 'ReadAllowedAutomationDocuments', + effect: iam.Effect.ALLOW, + actions: ['ssm:GetDocument', 'ssm:DescribeDocument'], + resources: allowedRegions.flatMap(region => lambdaStartDocumentNames.map(documentName => + requireCollectionApproval + ? `arn:${partition}:ssm:${region}:${stack.account}:document/${documentName}` + : `arn:${partition}:ssm:${region}::document/${documentName}`, + )), + })); + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + sid: 'RegionalInfrastructureRead', + effect: iam.Effect.ALLOW, + actions: [ + 'ec2:DescribeInstances', 'ec2:DescribeRegions', + 'ec2:DescribeInstanceStatus', 'ec2:DescribeNetworkInterfaces', + 'ec2:DescribeSubnets', 'ec2:DescribeSecurityGroups', 'ec2:DescribeRouteTables', + 'ecs:DescribeClusters', + 'ecs:DescribeContainerInstances', 'ecs:ListContainerInstances', + 'ecs:DescribeServices', 'ecs:ListServices', + 'ecs:DescribeTasks', 'ecs:ListTasks', + 'autoscaling:DescribeAutoScalingGroups', + ], + resources: ['*'], + conditions: requestedRegionCondition, + })); + + this.logsBucket.grantReadWrite(lambdaExecutionRole); + if (this.kmsKey) { this.kmsKey.grantEncryptDecrypt(lambdaExecutionRole); } + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['iam:PassRole'], + resources: [this.ssmAutomationRole.roleArn], + conditions: { StringEquals: { 'iam:PassedToService': 'ssm.amazonaws.com' } }, + })); + + // Lambda cannot approve its own requests. Direct Run Command is available + // only for the explicitly non-approval tcpdump path. + if (!requireCollectionApproval && enabledRestrictedTools.includes('tcpdump_capture')) { + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + sid: 'DirectTcpdumpRunShellScriptWithoutApproval', + effect: iam.Effect.ALLOW, + actions: ['ssm:SendCommand'], + resources: runShellDocumentResources, + })); + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + sid: 'DirectTcpdumpInstancesWithoutApproval', + effect: iam.Effect.ALLOW, + actions: ['ssm:SendCommand'], + resources: allowedInstanceResources, + })); + } + + this.collectionApprovalTopic = new sns.Topic(this, 'CollectionApprovalTopic', { + topicName: `Automation-${cdk.Stack.of(this).stackName}-approvals`, + displayName: 'ECS diagnostics collection approvals', + masterKey: this.kmsKey, + }); + for (const email of props.approvalNotificationEmails ?? []) { + this.collectionApprovalTopic.addSubscription(new subscriptions.EmailSubscription(email)); + } + this.collectionApprovalTopic.grantPublish(this.ssmAutomationRole); + this.collectionApprovalTopic.grantPublish(lambdaExecutionRole); + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + sid: 'PassSelfToChildAutomations', + effect: iam.Effect.ALLOW, + actions: ['iam:PassRole'], + resources: [this.ssmAutomationRole.roleArn], + conditions: { StringEquals: { 'iam:PassedToService': 'ssm.amazonaws.com' } }, + })); + + if (requireCollectionApproval) { + new ssm.CfnDocument(this, 'CollectApprovalDocument', { + name: collectApprovalDocName, + documentType: 'Automation', + updateMethod: 'NewVersion', + content: { + schemaVersion: '0.3', + description: 'Approve, then collect diagnostics from one ECS EC2 container instance.', + assumeRole: this.ssmAutomationRole.roleArn, + parameters: { + ECSInstanceId: { type: 'String', allowedPattern: '^i-[0-9a-f]{8,17}$' }, + LogDestination: { type: 'String' }, + }, + mainSteps: [ + { + name: 'waitForHumanApproval', + action: 'aws:approve', + timeoutSeconds: approvalTtlSeconds, + onFailure: 'Abort', + inputs: { + NotificationArn: this.collectionApprovalTopic.topicArn, + Message: 'Approve ECS instance log collection on {{ ECSInstanceId }}. Approval runs AWSSupport-CollectECSInstanceLogs and uploads the bundle to {{ LogDestination }}.', + MinRequiredApprovals: 1, + Approvers: approverArns, + }, + }, + { + name: 'collectLogs', + action: 'aws:executeAutomation', + inputs: { + DocumentName: 'AWSSupport-CollectECSInstanceLogs', + RuntimeParameters: { + ECSInstanceId: '{{ ECSInstanceId }}', + LogDestination: '{{ LogDestination }}', + AutomationAssumeRole: this.ssmAutomationRole.roleArn, + }, + }, + }, + ], + }, + }); + + new ssm.CfnDocument(this, 'BatchCollectApprovalDocument', { + name: batchApprovalDocName, + documentType: 'Automation', + updateMethod: 'NewVersion', + content: { + schemaVersion: '0.3', + description: 'Approve once, then collect diagnostics from up to 15 ECS EC2 container instances.', + assumeRole: this.ssmAutomationRole.roleArn, + parameters: { + InstanceIds: { type: 'StringList', description: 'ECS EC2 container instance IDs (max 15)' }, + LogDestination: { type: 'String' }, + }, + mainSteps: [ + { + name: 'waitForHumanApproval', + action: 'aws:approve', + timeoutSeconds: approvalTtlSeconds, + onFailure: 'Abort', + inputs: { + NotificationArn: this.collectionApprovalTopic.topicArn, + Message: 'Approve this batch ECS log collection. The execution Parameters list the ECS EC2 container instances; at most 15 child collections will start.', + MinRequiredApprovals: 1, + Approvers: approverArns, + }, + }, + { + name: 'fanOutCollections', + action: 'aws:executeScript', + timeoutSeconds: 600, + inputs: { + Runtime: 'python3.11', + Handler: 'handler', + InputPayload: { + InstanceIds: '{{ InstanceIds }}', + LogDestination: '{{ LogDestination }}', + }, + Script: [ + 'import boto3', + '', + 'def handler(events, context):', + ' ssm = boto3.client("ssm")', + ' executions, errors = [], []', + ' for instance_id in events["InstanceIds"][:15]:', + ' try:', + ' response = ssm.start_automation_execution(', + ' DocumentName="AWSSupport-CollectECSInstanceLogs",', + ' Parameters={', + ' "ECSInstanceId": [instance_id],', + ' "LogDestination": [events["LogDestination"]],', + ` "AutomationAssumeRole": ["${this.ssmAutomationRole.roleArn}"],`, + ' },', + ' )', + ' executions.append(f"{instance_id}|{response[\'AutomationExecutionId\']}")', + ' except Exception as error:', + ' errors.append(f"{instance_id}|{error}")', + ' return {"executions": executions, "errors": errors}', + ].join('\n'), + }, + outputs: [ + { Name: 'Executions', Selector: '$.Payload.executions', Type: 'StringList' }, + { Name: 'Errors', Selector: '$.Payload.errors', Type: 'StringList' }, + ], + }, + ], + }, + }); + + new ssm.CfnDocument(this, 'TcpdumpApprovalDocument', { + name: tcpdumpApprovalDocName, + documentType: 'Automation', + updateMethod: 'NewVersion', + content: { + schemaVersion: '0.3', + description: 'Approve, then run a task-scoped tcpdump capture on an ECS EC2 container instance.', + assumeRole: this.ssmAutomationRole.roleArn, + parameters: { + InstanceId: { type: 'String', allowedPattern: '^i-[0-9a-f]{8,17}$' }, + Commands: { type: 'String', description: 'Validated task-scoped capture script; review before approval' }, + ExecutionTimeoutSeconds: { type: 'String', allowedPattern: '^\\d{2,4}$', default: '240' }, + DurationSeconds: { type: 'String', allowedPattern: '^\\d{1,3}$' }, + Interface: { type: 'String', allowedPattern: '^[a-zA-Z0-9\\-\\.]+$' }, + BpfFilter: { type: 'String', allowedPattern: '^[^\\r\\n]{1,256}$' }, + CaptureScope: { + type: 'String', + allowedPattern: '^task/[0-9a-f]{32}/container/[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$', + }, + }, + mainSteps: [ + { + name: 'waitForHumanApproval', + action: 'aws:approve', + timeoutSeconds: approvalTtlSeconds, + onFailure: 'Abort', + inputs: { + NotificationArn: this.collectionApprovalTopic.topicArn, + Message: 'Approve task-scoped tcpdump on ECS EC2 instance {{ InstanceId }} ({{ CaptureScope }}), interface {{ Interface }}, duration {{ DurationSeconds }}s, filter {{ BpfFilter }}. Review Commands before approving; packet contents may be sensitive.', + MinRequiredApprovals: 1, + Approvers: approverArns, + }, + }, + { + name: 'runTcpdump', + action: 'aws:runCommand', + inputs: { + DocumentName: 'AWS-RunShellScript', + InstanceIds: ['{{ InstanceId }}'], + Parameters: { + commands: ['{{ Commands }}'], + executionTimeout: ['{{ ExecutionTimeoutSeconds }}'], + }, + Comment: 'Human-approved task-scoped ECS tcpdump on {{ InstanceId }}', + }, + }, + ], + }, + }); + } + + const lambdaEnv: { [key: string]: string } = { + LOGS_BUCKET_NAME: this.logsBucket.bucketName, + SSM_AUTOMATION_ROLE_ARN: this.ssmAutomationRole.roleArn, + ALLOWED_CLUSTER_NAMES: allowedClusterNames.join(','), + ALLOWED_REGIONS: allowedRegions.join(','), + PRESIGNED_URL_EXPIRATION_SECONDS: String(props.presignedUrlExpirationSeconds ?? 900), + REQUIRE_COLLECTION_APPROVAL: String(requireCollectionApproval), + APPROVAL_TOPIC_ARN: this.collectionApprovalTopic.topicArn, + COLLECT_APPROVAL_DOCUMENT: requireCollectionApproval ? collectApprovalDocName : '', + BATCH_APPROVAL_DOCUMENT: requireCollectionApproval ? batchApprovalDocName : '', + TCPDUMP_APPROVAL_DOCUMENT: requireCollectionApproval ? tcpdumpApprovalDocName : '', + APPROVAL_APPROVERS: approverArns.join(','), + ENABLED_RESTRICTED_TOOLS: enabledRestrictedTools.join(','), + PCAP_PRESIGNED_URL_EXPIRATION_SECONDS: String(props.pcapPresignedUrlExpirationSeconds ?? 60), + MAX_PCAP_BYTES: String(props.maxPcapBytes ?? 209715200), + APPROVAL_EMAILS_CONFIGURED: String((props.approvalNotificationEmails ?? []).length > 0), + }; + if (this.kmsKey) { lambdaEnv['KMS_KEY_ARN'] = this.kmsKey.keyArn; } + + // SOP Bucket + this.sopBucket = new s3.Bucket(this, 'SOPBucket', { + bucketName: props.sopBucketName ?? `${cdk.Stack.of(this).stackName.toLowerCase()}-sops-${cdk.Stack.of(this).account}`, + versioned: true, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + enforceSSL: true, + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + lambdaEnv['SOP_BUCKET_NAME'] = this.sopBucket.bucketName; + this.sopBucket.grantRead(lambdaExecutionRole); + + // Auto-deploy SOP runbooks from local sops/ directory to the SOP bucket + new s3deploy.BucketDeployment(this, 'SOPDeployment', { + sources: [s3deploy.Source.asset(path.join(__dirname, '../', 'sops'))], + destinationBucket: this.sopBucket, + prune: true, + }); + + this.ssmAutomationFunction = new lambda.Function(this, 'SSMAutomationFunction', { + functionName: `${cdk.Stack.of(this).stackName}-ssm-automation`, + runtime: lambda.Runtime.PYTHON_3_11, + handler: 'ecs-log-automation.lambda_handler', + role: lambdaExecutionRole, + timeout: cdk.Duration.minutes(5), + memorySize: 1024, + environment: lambdaEnv, + code: lambda.Code.fromAsset(path.join(__dirname, 'lambda')), + logRetention: logs.RetentionDays.TWO_WEEKS, + }); + + // ======================================================================== + // GATEWAY EXECUTION ROLE + // ======================================================================== + this.gatewayExecutionRole = new iam.Role(this, 'GatewayExecutionRole', { + roleName: `${cdk.Stack.of(this).stackName}-gateway-execution-role`, + assumedBy: new iam.ServicePrincipal('bedrock-agentcore.amazonaws.com'), + }); + + this.gatewayExecutionRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['lambda:InvokeFunction'], + resources: [this.ssmAutomationFunction.functionArn], + })); + + this.gatewayExecutionRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], + resources: ['*'], + })); + + this.ssmAutomationFunction.addPermission('AllowAgentCoreInvoke', { + principal: new iam.ServicePrincipal('bedrock-agentcore.amazonaws.com'), + action: 'lambda:InvokeFunction', + sourceAccount: cdk.Stack.of(this).account, + }); + + // ======================================================================== + // COGNITO - Authentication + // ======================================================================== + this.userPool = new cognito.UserPool(this, 'UserPool', { + userPoolName: cognitoUserPoolName, + passwordPolicy: { minLength: 8, requireUppercase: false, requireLowercase: false, requireDigits: false, requireSymbols: false }, + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + + const resourceServer = new cognito.UserPoolResourceServer(this, 'ResourceServer', { + userPool: this.userPool, + identifier: resourceServerName, + scopes: [ + { scopeName: 'gateway:read', scopeDescription: 'Read access to gateway' }, + { scopeName: 'gateway:write', scopeDescription: 'Write access to gateway' }, + ], + }); + + this.userPoolClient = new cognito.UserPoolClient(this, 'UserPoolClient', { + userPool: this.userPool, + userPoolClientName: 'ecs-log-gateway-client', + generateSecret: true, + oAuth: { + flows: { clientCredentials: true }, + scopes: [ + cognito.OAuthScope.custom(`${resourceServerName}/gateway:read`), + cognito.OAuthScope.custom(`${resourceServerName}/gateway:write`), + ], + }, + supportedIdentityProviders: [cognito.UserPoolClientIdentityProvider.COGNITO], + }); + this.userPoolClient.node.addDependency(resourceServer); + + const userPoolDomain = new cognito.UserPoolDomain(this, 'UserPoolDomain', { + userPool: this.userPool, + cognitoDomain: { domainPrefix: `${cdk.Stack.of(this).stackName.toLowerCase()}-${cdk.Stack.of(this).account}` }, + }); + + // ======================================================================== + // AGENTCORE GATEWAY + // ======================================================================== + const gateway = new cdk.CfnResource(this, 'AgentCoreGateway', { + type: 'AWS::BedrockAgentCore::Gateway', + properties: { + Name: gatewayName, + Description: 'ECS Instance Log Collection MCP Server with pre-indexed findings, byte-range streaming, and incident analysis', + ProtocolType: 'MCP', + AuthorizerType: 'CUSTOM_JWT', + AuthorizerConfiguration: { + CustomJWTAuthorizer: { + AllowedClients: [this.userPoolClient.userPoolClientId], + DiscoveryUrl: `https://cognito-idp.${cdk.Stack.of(this).region}.amazonaws.com/${this.userPool.userPoolId}/.well-known/openid-configuration`, + }, + }, + RoleArn: this.gatewayExecutionRole.roleArn, + }, + }); + gateway.node.addDependency(this.userPoolClient); + gateway.node.addDependency(userPoolDomain); + + const gatewayTarget = new cdk.CfnResource(this, 'LambdaGatewayTarget', { + type: 'AWS::BedrockAgentCore::GatewayTarget', + properties: { + GatewayIdentifier: gateway.ref, + Name: 'ECSInstanceLogTarget', + Description: 'ECS Instance Log Collection Target', + TargetConfiguration: { + Mcp: { + Lambda: { + LambdaArn: this.ssmAutomationFunction.functionArn, + ToolSchema: { InlinePayload: this.getToolSchemaDefinitions(enabledRestrictedTools) }, + }, + }, + }, + CredentialProviderConfigurations: [{ CredentialProviderType: 'GATEWAY_IAM_ROLE' }], + }, + }); + gatewayTarget.node.addDependency(this.ssmAutomationFunction); + gatewayTarget.node.addDependency(this.gatewayExecutionRole); + + // ======================================================================== + // OUTPUTS + // ======================================================================== + new cdk.CfnOutput(this, 'GatewayId', { description: 'ID of the AgentCore Gateway', value: gateway.ref, exportName: `${cdk.Stack.of(this).stackName}-GatewayId` }); + new cdk.CfnOutput(this, 'GatewayUrl', { description: 'MCP Server URL', value: gateway.getAtt('GatewayUrl').toString(), exportName: `${cdk.Stack.of(this).stackName}-GatewayUrl` }); + new cdk.CfnOutput(this, 'CognitoUserPoolId', { description: 'Cognito User Pool ID', value: this.userPool.userPoolId, exportName: `${cdk.Stack.of(this).stackName}-CognitoUserPoolId` }); + new cdk.CfnOutput(this, 'CognitoClientId', { description: 'OAuth Client ID', value: this.userPoolClient.userPoolClientId, exportName: `${cdk.Stack.of(this).stackName}-CognitoClientId` }); + new cdk.CfnOutput(this, 'OAuthExchangeUrl', { description: 'OAuth Token URL', value: `https://${cdk.Stack.of(this).stackName.toLowerCase()}-${cdk.Stack.of(this).account}.auth.${cdk.Stack.of(this).region}.amazoncognito.com/oauth2/token`, exportName: `${cdk.Stack.of(this).stackName}-OAuthExchangeUrl` }); + new cdk.CfnOutput(this, 'OAuthScope', { description: 'OAuth Scope', value: `${resourceServerName}/gateway:read`, exportName: `${cdk.Stack.of(this).stackName}-OAuthScope` }); + new cdk.CfnOutput(this, 'LogsBucketName', { description: 'S3 bucket for collected logs', value: this.logsBucket.bucketName, exportName: `${cdk.Stack.of(this).stackName}-LogsBucketName` }); + new cdk.CfnOutput(this, 'SSMAutomationRoleArn', { description: 'SSM Automation Role ARN', value: this.ssmAutomationRole.roleArn, exportName: `${cdk.Stack.of(this).stackName}-SSMAutomationRoleArn` }); + new cdk.CfnOutput(this, 'CollectionApprovalTopicArn', { description: 'SNS topic for native SSM approval notifications', value: this.collectionApprovalTopic.topicArn }); + new cdk.CfnOutput(this, 'CollectionApprovalDocuments', { description: 'Native SSM approval wrapper documents', value: `${collectApprovalDocName},${batchApprovalDocName},${tcpdumpApprovalDocName}` }); + if (this.kmsKey) { new cdk.CfnOutput(this, 'EncryptionKeyArn', { description: 'KMS Encryption Key ARN', value: this.kmsKey.keyArn, exportName: `${cdk.Stack.of(this).stackName}-EncryptionKeyArn` }); } + new cdk.CfnOutput(this, 'SOPBucketName', { description: 'S3 bucket for Standard Operating Procedures', value: this.sopBucket.bucketName, exportName: `${cdk.Stack.of(this).stackName}-SOPBucketName` }); + } + + private getToolSchemaDefinitions(enabledRestrictedTools: string[]): object[] { + const tools: Array<{ Name: string; [key: string]: unknown }> = [ + // ===================================================================== + // TIER 1: CORE OPERATIONS + // ===================================================================== + { + Name: 'collect', + Description: 'Start ECS log collection from an EC2-backed container instance. Collection may be approval-gated: status="pending_approval" means the SSM Automation is paused at native human approval. Show approvalConsoleUrl and poll status(executionId); do not call collect again because collection starts automatically after approval. Supports idempotency and cross-region execution. CITATION: Cite executionId and region.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID of the ECS container instance (e.g., i-0123456789abcdef0)' }, + idempotencyToken: { Type: 'string', Description: 'Optional token to prevent duplicate executions. If provided and a matching execution exists, returns the existing executionId.' }, + region: { Type: 'string', Description: 'AWS region where the instance runs (e.g., us-west-2). Optional: auto-detected from instance if omitted.' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + executionId: { Type: 'string', Description: 'SSM Automation execution ID for polling' }, + instanceId: { Type: 'string' }, + region: { Type: 'string' }, + status: { Type: 'string', Description: 'pending_approval | in_progress | completed | failed' }, + approvalConsoleUrl: { Type: 'string', Description: 'Systems Manager console link when status is pending_approval' }, + humanApproval: { Type: 'object', Description: 'Human approval state and console URL' }, + estimatedCompletionTime: { Type: 'string' }, + idempotent: { Type: 'boolean', Description: 'True if returning existing execution' }, + task: { Type: 'object', Description: 'Async task envelope for polling', Properties: { taskId: { Type: 'string' }, state: { Type: 'string', Description: 'running|completed|failed|cancelled' }, message: { Type: 'string' }, progress: { Type: 'integer', Description: '0-100 percent' } } }, + }, + }, + }, + { + Name: 'status', + Description: 'Get detailed status of a log collection execution including progress percentage, step details, and failure reasons. Automatically resolves the region where the execution was started. CITATION: Always cite executionId and status in your response.', + InputSchema: { + Type: 'object', + Properties: { + executionId: { Type: 'string', Description: 'The SSM Automation execution ID returned from collect' }, + includeStepDetails: { Type: 'boolean', Description: 'Include individual step status (default: true)' }, + region: { Type: 'string', Description: 'AWS region of the execution. Optional: auto-resolved from stored execution metadata if omitted.' }, + }, + Required: ['executionId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + automation: { Type: 'object', Properties: { executionId: { Type: 'string' }, status: { Type: 'string', Description: 'InProgress|Success|Failed|Cancelled' }, progress: { Type: 'integer' }, failureReason: { Type: 'string' }, stepDetails: { Type: 'array' } } }, + task: { Type: 'object', Properties: { taskId: { Type: 'string' }, state: { Type: 'string' }, message: { Type: 'string' }, progress: { Type: 'integer' } } }, + }, + }, + }, + { + Name: 'validate', + Description: 'Verify all expected files were extracted from the log bundle. Returns manifest with file counts, sizes, and missing patterns. Uses manifest.json when available for authoritative file inventory. CITATION: Cite fileCount, totalSizeHuman, and any missingPatterns.', + InputSchema: { + Type: 'object', + Properties: { + executionId: { Type: 'string', Description: 'SSM execution ID to validate' }, + instanceId: { Type: 'string', Description: 'Alternative: Instance ID to locate bundle' }, + }, + }, + OutputSchema: { + Type: 'object', + Properties: { + complete: { Type: 'boolean' }, fileCount: { Type: 'integer' }, totalSize: { Type: 'integer' }, totalSizeHuman: { Type: 'string' }, + missingPatterns: { Type: 'array' }, foundPatterns: { Type: 'array' }, hasFindingsIndex: { Type: 'boolean' }, + manifest: { Type: 'array', Description: 'File list with key, fullKey, size, sizeHuman' }, + }, + }, + }, + { + Name: 'errors', + Description: 'Get pre-indexed error findings (fast path). Returns categorized errors by severity with finding_ids for citation. Each finding has a stable finding_id (F-001 format). Automatically recommends relevant SOPs based on detected error patterns (recommendedSOPs field). CITATION: Always cite finding_id and severity when referencing findings. Use finding_ids with summarize tool.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to get error summary for' }, + severity: { Type: 'string', Description: 'Filter by severity: critical, high, medium, low, info, or all (default: all). Legacy "warning" maps to "high".' }, + response_format: { Type: 'string', Description: '"concise" (default) returns finding_id/severity/pattern/file/count only. "detailed" includes full sample text and line numbers.' }, + pageSize: { Type: 'integer', Description: 'Number of findings per page (default: 50, max: 200)' }, + pageToken: { Type: 'string', Description: 'Opaque token for next page (from previous response nextPageToken)' }, + clusterContext: { Type: 'string', Description: 'ECS cluster name for baseline subtraction. When provided, findings seen 10+ times are annotated as baseline (normal operation).' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, + findings: { Type: 'array', Description: 'Array of findings with finding_id, severity, pattern, file, count' }, + totalFindings: { Type: 'integer' }, + summary: { Type: 'object', Description: 'Counts by severity: critical, high, medium, low, info' }, + hasMore: { Type: 'boolean' }, nextPageToken: { Type: 'string' }, + coverage_report: { Type: 'object', Description: 'files_scanned, files_skipped, scan_complete' }, + recommendedSOPs: { Type: 'array', Description: 'Auto-matched SOP runbooks based on detected error patterns. Each entry has sopName, relevanceScore, matchedKeywords, reason. Use get_sop to retrieve full SOP content.' }, + }, + }, + }, + { + Name: 'read', + Description: 'Read a chunk of a log file using byte-range streaming. NO TRUNCATION. Supports both byte-range and line-based reading for multi-GB files. Line-aligned: byte reads snap to newline boundaries. CITATION: Cite logKey, startByte, endByte, and totalSize.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The ECS EC2 instance that owns this log object' }, + logKey: { Type: 'string', Description: 'The canonical S3 key below ecs_{instanceId}/ (from validate manifest)' }, + startByte: { Type: 'integer', Description: 'Starting byte offset (default: 0). Snaps forward to next newline.' }, + endByte: { Type: 'integer', Description: 'Ending byte offset (default: startByte + 1MB). Snaps forward to next newline.' }, + startLine: { Type: 'integer', Description: 'Alternative: Starting line number (1-based)' }, + lineCount: { Type: 'integer', Description: 'Number of lines to return when using startLine (default: 1000)' }, + }, + Required: ['instanceId', 'logKey'], + }, + OutputSchema: { + Type: 'object', + Properties: { + logKey: { Type: 'string' }, content: { Type: 'string' }, + startByte: { Type: 'integer' }, endByte: { Type: 'integer' }, totalSize: { Type: 'integer' }, + hasMore: { Type: 'boolean' }, nextChunkToken: { Type: 'string' }, + truncated: { Type: 'boolean', Description: 'Always false — never truncates' }, + lineAligned: { Type: 'boolean' }, + }, + }, + }, + // ===================================================================== + // TIER 2: ADVANCED ANALYSIS + // ===================================================================== + { + Name: 'search', + Description: 'Full-text regex search across all logs without truncation. Use for detailed investigation after reviewing error summary. CITATION: Cite finding_id (S-NNN format), file name, and match count for each result group.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to search logs for' }, + query: { Type: 'string', Description: 'Regex pattern to search for (e.g., "CannotPullContainerError|ResourceInitializationError")' }, + logTypes: { Type: 'string', Description: 'Comma-separated log types to search. Available types: ecs-agent, docker, containers, system, dmesg, networking, cgroups, metadata, gpu (default: all)' }, + maxResults: { Type: 'integer', Description: 'Maximum results per file (default: 100, max: 500)' }, + response_format: { Type: 'string', Description: '"concise" (default) or "detailed"' }, + }, + Required: ['instanceId', 'query'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, query: { Type: 'string' }, + filesSearched: { Type: 'integer' }, filesWithMatches: { Type: 'integer' }, totalMatches: { Type: 'integer' }, + results: { Type: 'array', Description: 'Array of {finding_id, file, fullKey, matchCount, matches[]}' }, + coverage_report: { Type: 'object' }, + }, + }, + }, + { + Name: 'correlate', + Description: 'Cross-file timeline correlation for incident analysis. Groups events by component, builds temporal clusters, and identifies potential root cause chains. Automatically recommends relevant SOPs based on correlated findings (recommendedSOPs field). CITATION: Cite finding_ids, confidence level, and any gaps reported.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to correlate events for' }, + timeWindow: { Type: 'integer', Description: 'Seconds around pivot event (default: 60)' }, + pivotEvent: { Type: 'string', Description: 'Event to correlate around (optional)' }, + components: { Type: 'array', Description: 'Components to include (optional)' }, + response_format: { Type: 'string', Description: '"concise" (default) or "detailed"' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, timeline: { Type: 'array' }, byComponent: { Type: 'object' }, + correlations: { Type: 'array' }, temporal_clusters: { Type: 'array' }, + potential_root_cause_chain: { Type: 'array' }, + confidence: { Type: 'string', Description: 'high|medium|low|none' }, + gaps: { Type: 'array' }, coverage_report: { Type: 'object' }, + recommendedSOPs: { Type: 'array', Description: 'Auto-matched SOP runbooks based on correlated findings. Each entry has sopName, relevanceScore, matchedKeywords, reason.' }, + }, + }, + }, + { + Name: 'artifact', + Description: 'Get secure presigned URL for large artifacts. Use for files too large to return directly. CITATION: Cite the logKey and expiresAt timestamp.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The ECS EC2 instance that owns this artifact' }, + logKey: { Type: 'string', Description: 'The canonical S3 key below ecs_{instanceId}/; tcpdump and metadata keys are rejected' }, + expirationMinutes: { Type: 'integer', Description: 'Positive URL lifetime request, capped by deployment PRESIGNED_URL_EXPIRATION' }, + }, + Required: ['instanceId', 'logKey'], + }, + OutputSchema: { + Type: 'object', + Properties: { + logKey: { Type: 'string' }, presignedUrl: { Type: 'string' }, expiresAt: { Type: 'string' }, sizeHuman: { Type: 'string' }, + }, + }, + }, + { + Name: 'summarize', + Description: 'Generate structured incident summary grounded in indexed findings. Pass finding_ids from errors tool to constrain summary to specific findings. Output includes grounded flag (true = all claims backed by finding_ids). Automatically recommends relevant SOPs based on findings and triage category (recommendedSOPs field). CITATION: Always cite the finding_ids that support each claim.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to summarize' }, + includeRecommendations: { Type: 'boolean', Description: 'Include remediation suggestions (default: true)' }, + finding_ids: { Type: 'array', Description: 'Optional list of finding_ids (F-001 format) from errors tool. When provided, summary is constrained to only these findings.' }, + }, + Required: ['instanceId', 'finding_ids'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, grounded: { Type: 'boolean' }, + unresolvedFindingIds: { Type: 'array' }, criticalFindings: { Type: 'array' }, + highFindings: { Type: 'array' }, affectedComponents: { Type: 'array' }, + recommendations: { Type: 'array' }, confidence: { Type: 'string' }, gaps: { Type: 'array' }, + recommendedSOPs: { Type: 'array', Description: 'Auto-matched SOP runbooks based on findings and triage category. Each entry has sopName, relevanceScore, matchedKeywords, reason. Use get_sop to retrieve full SOP content.' }, + }, + }, + }, + { + Name: 'history', + Description: 'List historical log collections for audit and comparison. Supports cross-region listing. CITATION: Cite executionId and status for each entry.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'Filter by instance ID (optional)' }, + maxResults: { Type: 'integer', Description: 'Maximum results (default: 20, max: 50)' }, + status: { Type: 'string', Description: 'Filter by status: Success, Failed, InProgress (optional)' }, + region: { Type: 'string', Description: 'AWS region to list executions from (default: Lambda region).' }, + }, + }, + OutputSchema: { + Type: 'object', + Properties: { executions: { Type: 'array' }, totalCount: { Type: 'integer' } }, + }, + }, + // ===================================================================== + // TIER 3: CLUSTER-LEVEL INTELLIGENCE + // ===================================================================== + { + Name: 'cluster_health', + Description: 'Get a comprehensive health overview of an ECS cluster. Enumerates all container instances, checks SSM agent status, instance metadata (type, AZ, AMI, launch time), ECS agent connectivity, and flags unhealthy instances. The entry point before diving into individual instance investigation. CITATION: Cite clusterName, instance counts, and any unhealthy instanceIds.', + InputSchema: { + Type: 'object', + Properties: { + clusterName: { Type: 'string', Description: 'Name of the ECS cluster to inspect' }, + region: { Type: 'string', Description: 'AWS region of the cluster (auto-detected if omitted)' }, + includeSSMStatus: { Type: 'boolean', Description: 'Check SSM agent connectivity for each instance (default: true)' }, + }, + Required: ['clusterName'], + }, + OutputSchema: { + Type: 'object', + Properties: { + clusterName: { Type: 'string' }, totalInstances: { Type: 'integer' }, + healthyInstances: { Type: 'integer' }, unhealthyInstances: { Type: 'integer' }, + instances: { Type: 'array' }, + }, + }, + }, + { + Name: 'compare_instances', + Description: 'Diff error findings and health status between two or more container instances. Surfaces what is unique to a failing instance vs. common across all instances. CITATION: Cite instanceIds compared and unique findings per instance.', + InputSchema: { + Type: 'object', + Properties: { + instanceIds: { Type: 'array', Description: 'List of 2+ EC2 instance IDs to compare (e.g., ["i-aaa", "i-bbb"])' }, + compareFields: { Type: 'string', Description: 'What to compare: "errors", "config", "all" (default: "all")' }, + }, + Required: ['instanceIds'], + }, + OutputSchema: { + Type: 'object', + Properties: { + commonFindings: { Type: 'array' }, uniqueFindings: { Type: 'object' }, insight: { Type: 'string' }, + }, + }, + }, + { + Name: 'batch_collect', + Description: 'Smart batch ECS log collection with statistical sampling. Defaults to a DRY RUN; pass dryRun=false to collect from at most 15 instances. Real collection may return status="pending_approval" and pause once for native human approval before automatically fanning out child collections. Show approvalConsoleUrl and poll batch_status(batchId); do not call batch_collect again. CITATION: Cite batchId, instance count, and sampling strategy.', + InputSchema: { + Type: 'object', + Properties: { + clusterName: { Type: 'string', Description: 'Name of the ECS cluster' }, + region: { Type: 'string', Description: 'AWS region of the cluster (auto-detected if omitted)' }, + filter: { Type: 'string', Description: 'Instance filter: "all", "unhealthy", "disconnected" (default: "unhealthy")' }, + strategy: { Type: 'string', Description: '"sample" for smart sampling or "all" to collect from every filtered instance (default: "sample")' }, + samplesPerBucket: { Type: 'integer', Description: 'Instances to sample per failure bucket (default: 3, max: 5)' }, + maxTotalCollections: { Type: 'integer', Description: 'Hard cap on total collections (default: 15, max: 15)' }, + groupBy: { Type: 'string', Description: 'Grouping strategy: "auto", "az", "instance-type", "ami" (default: "auto")' }, + dryRun: { Type: 'boolean', Description: 'Preview which instances would be collected without starting (default: true). Set false to collect.' }, + }, + Required: ['clusterName'], + }, + OutputSchema: { + Type: 'object', + Properties: { + batchId: { Type: 'string' }, executions: { Type: 'array' }, + status: { Type: 'string', Description: 'dry_run | pending_approval | in_progress | completed | failed' }, + approvalConsoleUrl: { Type: 'string', Description: 'Systems Manager console link when status is pending_approval' }, + humanApproval: { Type: 'object', Description: 'Human approval state and console URL' }, + totalInstances: { Type: 'integer' }, sampledInstances: { Type: 'integer' }, strategy: { Type: 'string' }, + task: { Type: 'object', Properties: { taskId: { Type: 'string' }, state: { Type: 'string' }, message: { Type: 'string' }, progress: { Type: 'integer' } } }, + }, + }, + }, + { + Name: 'batch_status', + Description: 'Poll status of multiple log collections at once. Returns consolidated view with allComplete boolean. Use after batch_collect to wait for all collections to finish. CITATION: Cite allComplete status and any failed executionIds.', + InputSchema: { + Type: 'object', + Properties: { + batchId: { Type: 'string', Description: 'Batch ID returned by batch_collect' }, + }, + Required: ['batchId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + allComplete: { Type: 'boolean' }, executions: { Type: 'array' }, + successCount: { Type: 'integer' }, failedCount: { Type: 'integer' }, inProgressCount: { Type: 'integer' }, + }, + }, + }, + { + Name: 'network_diagnostics', + Description: 'Extract and structure networking info from collected log bundles. Parses iptables rules, Docker networking, route tables, DNS resolution, ENI attachment status, and security group config. Returns structured data instead of raw text. Automatically recommends relevant SOPs based on detected networking issues (recommendedSOPs field). CITATION: Cite instanceId and each section analyzed.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to analyze networking for' }, + region: { Type: 'string', Description: 'Allowed AWS region used for live ENI and security-group sections' }, + sections: { Type: 'string', Description: 'Comma-separated sections: "iptables,docker,routes,dns,eni,security-groups" or "all" (default: "all")' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, sections: { Type: 'object' }, + assessment: { Type: 'string' }, issues: { Type: 'array' }, + confidence: { Type: 'string' }, gaps: { Type: 'array' }, + recommendedSOPs: { Type: 'array', Description: 'Auto-matched SOP runbooks based on detected networking issues. Each entry has sopName, relevanceScore, matchedKeywords, reason.' }, + }, + }, + }, + // ===================================================================== + // TIER 4: LIVE PACKET CAPTURE + // ===================================================================== + { + Name: 'tcpdump_capture', + Description: 'Start or poll a human-approved, task-scoped tcpdump capture on an ECS EC2 container instance. taskId and confirmCapture=true are required. Fargate, host-network, and host-wide captures are rejected, and the tool never installs tcpdump. When approval is enabled, show approvalConsoleUrl and poll with executionId; do not submit another capture. CITATION: Cite executionId or commandId, instanceId, and taskId.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'EC2 instance ID of the ECS container instance hosting the target task' }, + taskId: { Type: 'string', Description: 'Exact ECS task ID or full task ARN (required; ECS EC2 launch type only)' }, + containerName: { Type: 'string', Description: 'Exact RUNNING application container name. May be omitted only when exactly one eligible container exists; ambiguous tasks fail closed.' }, + durationSeconds: { Type: 'integer', Description: 'Capture duration in seconds (default: 120, max: 300)' }, + interface: { Type: 'string', Description: 'Network interface inside the task namespace (default: "any")' }, + filter: { Type: 'string', Description: 'Restricted BPF filter expression (for example "port 443" or "host 10.0.0.1")' }, + confirmCapture: { Type: 'boolean', Description: 'Must be true to request a new task-scoped packet capture' }, + commandId: { Type: 'string', Description: 'SSM Command UUID from an approved running capture; provide it to poll Run Command status' }, + executionId: { Type: 'string', Description: 'SSM Automation execution ID from a pending_approval response; provide it to poll approval status' }, + region: { Type: 'string', Description: 'AWS region where the ECS EC2 container instance runs (optional; auto-detected)' }, + }, + Required: ['instanceId', 'taskId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + commandId: { Type: 'string', Description: 'SSM Command ID once the approved capture starts' }, + executionId: { Type: 'string', Description: 'Approval-gated SSM Automation execution ID' }, + approvalConsoleUrl: { Type: 'string', Description: 'Systems Manager console link for human approval' }, + humanApproval: { Type: 'object', Description: 'Human approval state and console URL' }, + instanceId: { Type: 'string' }, + taskId: { Type: 'string' }, + status: { Type: 'string', Description: 'pending_approval | in_progress | completed | failed' }, + s3Key: { Type: 'string' }, + s3KeyTxt: { Type: 'string' }, + s3KeyStats: { Type: 'string' }, + s3Bucket: { Type: 'string' }, + presignedUrl: { Type: 'string', Description: 'Short-lived pcap download URL' }, + fileSizeBytes: { Type: 'integer' }, + task: { Type: 'object', Properties: { taskId: { Type: 'string' }, state: { Type: 'string' }, message: { Type: 'string' }, progress: { Type: 'integer' } } }, + }, + }, + }, + { + Name: 'tcpdump_analyze', + Description: 'Analyze a completed task-scoped tcpdump capture from an ECS EC2 task using its exact commandId. Returns decoded packets, protocol statistics, top talkers, and anomaly detection. This tool does not initiate captures and is unavailable unless explicitly enabled. CITATION: Cite commandId, instanceId, task scope, and anomalies.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'EC2 instance ID (required)' }, + commandId: { Type: 'string', Description: 'SSM Command UUID returned by tcpdump_capture (required; no latest-capture fallback)' }, + section: { Type: 'string', Description: '"summary" (decoded packets), "stats" (protocol breakdown), "all" (default: "all")' }, + maxPackets: { Type: 'integer', Description: 'Max decoded packet lines to return (default: 500, max: 3000)' }, + filter: { Type: 'string', Description: 'Text filter on decoded lines (e.g., "SYN", "RST", "10.0.0.5")' }, + }, + Required: ['instanceId', 'commandId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, + commandId: { Type: 'string' }, + captureInfo: { Type: 'object' }, + statistics: { Type: 'object', Description: 'Protocol breakdown, port distribution, TCP flags, top talkers' }, + anomalies: { Type: 'array', Description: 'Detected anomalies with type, severity, and message' }, + decodedPackets: { Type: 'object', Description: 'Decoded packet lines with totalPackets, returnedPackets, truncated flag' }, + pcapDownloadUrl: { Type: 'string', Description: 'Short-lived presigned URL to download the raw pcap file' }, + }, + }, + }, + // ===================================================================== + // SOP MANAGEMENT TOOLS + // ===================================================================== + { + Name: 'list_sops', + Description: 'List all available Standard Operating Procedures (SOPs) in the S3 bucket. Returns name, size, and last modified date for each SOP.', + InputSchema: { + Type: 'object', + Properties: {}, + }, + OutputSchema: { + Type: 'object', + Properties: { + sops: { Type: 'array', Description: 'Array of {name, size, lastModified}' }, + count: { Type: 'integer' }, + bucket: { Type: 'string' }, + }, + }, + }, + { + Name: 'get_sop', + Description: 'Get a specific Standard Operating Procedure (SOP) by name. Returns the full content of the SOP file. Use list_sops first to discover available SOPs.', + InputSchema: { + Type: 'object', + Properties: { + sopName: { + Type: 'string', + Description: 'The name/key of the SOP file to retrieve (e.g., "runbooks/pod-crashloop.md")', + }, + }, + Required: ['sopName'], + }, + OutputSchema: { + Type: 'object', + Properties: { + sop: { Type: 'object', Description: '{name, content, size, lastModified, contentType}' }, + }, + }, + }, + ]; + + return tools.filter(tool => + !['tcpdump_capture', 'tcpdump_analyze'].includes(tool.Name) + || enabledRestrictedTools.includes(tool.Name), + ); + } + + private getUnzipLambdaCode(): string { + return ` +import json +import boto3 +import zipfile +import tarfile +import io +import os +import re +from urllib.parse import unquote_plus +from datetime import datetime + +s3_client = boto3.client('s3') +lambda_client = boto3.client('lambda') + +FINDINGS_INDEXER_FUNCTION = os.environ.get('FINDINGS_INDEXER_FUNCTION', '') + +def get_content_type(file_name): + if file_name.endswith('.log') or file_name.endswith('.txt'): + return 'text/plain' + elif file_name.endswith('.json'): + return 'application/json' + elif file_name.endswith('.yaml') or file_name.endswith('.yml'): + return 'text/yaml' + return 'application/octet-stream' + +def sanitize_archive_path(file_name): + normalized = os.path.normpath(file_name) + if normalized.startswith('..') or '/../' in normalized or normalized.startswith('/'): + print(f"SECURITY: Skipping suspicious archive path: {file_name}") + return None + return normalized.lstrip('./') + +MANAGED_BUNDLE_RE = re.compile( + r'^ecs_(i-[0-9a-f]{8,17})_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\\.(zip|tar\\.gz|tgz)$', + re.IGNORECASE, +) +CANONICAL_BUNDLE_RE = re.compile( + r'^(ecs_(i-[0-9a-f]{8,17})/[A-Za-z0-9_.-]+)\\.(zip|tar\\.gz|tgz)$', + re.IGNORECASE, +) + +def resolve_extract_prefix(key): + managed_match = MANAGED_BUNDLE_RE.fullmatch(key) + if managed_match: + instance_id, execution_id = managed_match.group(1), managed_match.group(2) + return f"ecs_{instance_id}/{execution_id}/extracted/" + canonical_match = CANONICAL_BUNDLE_RE.fullmatch(key) + if canonical_match: + return f"{canonical_match.group(1)}/extracted/" + return None + +def extract_zip(bucket, key, content, extract_prefix): + extracted_files = [] + with zipfile.ZipFile(io.BytesIO(content), 'r') as zip_ref: + for file_info in zip_ref.infolist(): + if file_info.is_dir(): + continue + file_name = sanitize_archive_path(file_info.filename) + if file_name is None: + continue + file_content = zip_ref.read(file_info.filename) + extract_key = f"{extract_prefix}{file_name}" + s3_client.put_object(Bucket=bucket, Key=extract_key, Body=file_content, ContentType=get_content_type(file_name)) + extracted_files.append(extract_key) + print(f"Extracted: {extract_key}") + return extracted_files, extract_prefix + +def extract_targz(bucket, key, content, extract_prefix): + extracted_files = [] + with tarfile.open(fileobj=io.BytesIO(content), mode='r:gz') as tar_ref: + for member in tar_ref.getmembers(): + if not member.isfile(): + continue + file_name = sanitize_archive_path(member.name) + if file_name is None: + continue + file_obj = tar_ref.extractfile(member) + if file_obj is None: + continue + file_content = file_obj.read() + extract_key = f"{extract_prefix}{file_name}" + s3_client.put_object(Bucket=bucket, Key=extract_key, Body=file_content, ContentType=get_content_type(file_name)) + extracted_files.append(extract_key) + print(f"Extracted: {extract_key}") + return extracted_files, extract_prefix + +def trigger_findings_indexer(bucket, prefix, file_count): + if not FINDINGS_INDEXER_FUNCTION: + print("No findings indexer function configured, skipping") + return + try: + lambda_client.invoke(FunctionName=FINDINGS_INDEXER_FUNCTION, InvocationType='Event', Payload=json.dumps({'bucket': bucket, 'prefix': prefix, 'fileCount': file_count})) + print(f"Triggered findings indexer for prefix: {prefix}") + except Exception as e: + print(f"Failed to trigger findings indexer: {str(e)}") + +def generate_manifest(bucket, prefix, extracted_files, archive_key, archive_size): + try: + file_details = [] + for file_key in extracted_files: + try: + head = s3_client.head_object(Bucket=bucket, Key=file_key) + relative_path = file_key.split('/extracted/')[-1] if '/extracted/' in file_key else file_key + file_details.append({'key': relative_path, 'fullKey': file_key, 'size': head['ContentLength'], 'contentType': head.get('ContentType', 'application/octet-stream')}) + except Exception: + file_details.append({'key': file_key.split('/extracted/')[-1] if '/extracted/' in file_key else file_key, 'fullKey': file_key, 'size': 0, 'contentType': 'unknown'}) + manifest = {'version': 2, 'generatedAt': datetime.utcnow().isoformat() + 'Z', 'archiveKey': archive_key, 'archiveSize': archive_size, 'totalFiles': len(extracted_files), 'totalSize': sum(f['size'] for f in file_details), 'files': file_details} + manifest_key = f"{prefix}manifest.json" + s3_client.put_object(Bucket=bucket, Key=manifest_key, Body=json.dumps(manifest, default=str), ContentType='application/json') + print(f"Wrote manifest.json to {manifest_key} ({len(extracted_files)} files)") + except Exception as e: + print(f"Warning: Failed to generate manifest.json: {str(e)}") + +def cleanup_old_bundles(bucket, current_archive_key, max_bundles_to_keep=2): + """Delete old managed-runbook bundles for the same instance.""" + try: + current_match = MANAGED_BUNDLE_RE.fullmatch(current_archive_key) + if not current_match: + print(f"Archive is not a managed-runbook root bundle: {current_archive_key}; skipping cleanup") + return + instance_id = current_match.group(1) + bundle_prefix = f"ecs_{instance_id}_" + paginator = s3_client.get_paginator('list_objects_v2') + archives = [] + for page in paginator.paginate(Bucket=bucket, Prefix=bundle_prefix): + for obj in page.get('Contents', []): + match = MANAGED_BUNDLE_RE.fullmatch(obj['Key']) + if match and match.group(1).lower() == instance_id.lower(): + archives.append({ + 'key': obj['Key'], + 'execution_id': match.group(2), + 'last_modified': obj['LastModified'], + }) + if len(archives) <= max_bundles_to_keep: + print(f"Only {len(archives)} bundles for {instance_id}, nothing to clean up") + return + archives.sort(key=lambda archive: archive['last_modified'], reverse=True) + to_delete = archives[max_bundles_to_keep:] + for old in to_delete: + canonical_prefix = f"ecs_{instance_id}/{old['execution_id']}/" + delete_keys = [] + for page in paginator.paginate(Bucket=bucket, Prefix=canonical_prefix): + delete_keys.extend({'Key': obj['Key']} for obj in page.get('Contents', [])) + delete_keys.append({'Key': old['key']}) + if delete_keys: + for index in range(0, len(delete_keys), 1000): + s3_client.delete_objects( + Bucket=bucket, + Delete={'Objects': delete_keys[index:index + 1000], 'Quiet': True}, + ) + print(f"Cleaned up old bundle: {old['key']} ({len(delete_keys)} objects deleted)") + print(f"Bundle cleanup complete for {instance_id}: kept {max_bundles_to_keep}, deleted {len(to_delete)} old bundles") + except Exception as e: + print(f"Warning: Bundle cleanup failed (non-fatal): {str(e)}") + +def lambda_handler(event, context): + print(f"Received event: {json.dumps(event)}") + for record in event.get('Records', []): + bucket = record['s3']['bucket']['name'] + key = unquote_plus(record['s3']['object']['key']) + if '/extracted/' in key: + print(f"Skipping already extracted file: {key}") + continue + is_zip = key.lower().endswith('.zip') + is_targz = key.lower().endswith('.tar.gz') or key.lower().endswith('.tgz') + if not is_zip and not is_targz: + print(f"Skipping unsupported file type: {key}") + continue + extract_prefix = resolve_extract_prefix(key) + if not extract_prefix: + print(f"SECURITY: Unsupported archive key layout: {key}") + continue + print(f"Processing archive: s3://{bucket}/{key} into {extract_prefix}") + try: + response = s3_client.get_object(Bucket=bucket, Key=key) + content = response['Body'].read() + if is_zip: + extracted_files, prefix = extract_zip(bucket, key, content, extract_prefix) + else: + extracted_files, prefix = extract_targz(bucket, key, content, extract_prefix) + print(f"Successfully extracted {len(extracted_files)} files from {key}") + generate_manifest(bucket, prefix, extracted_files, key, len(content)) + trigger_findings_indexer(bucket, prefix, len(extracted_files)) + cleanup_old_bundles(bucket, key, max_bundles_to_keep=2) + except (zipfile.BadZipFile, tarfile.TarError) as e: + print(f"Error: {key} is not a valid archive: {str(e)}") + except Exception as e: + print(f"Error processing {key}: {str(e)}") + raise e + return {'statusCode': 200, 'body': json.dumps({'message': 'Archive extraction complete'})} +`; + } + + private getFindingsIndexerCode(): string { + return ` +import json +import boto3 +import os +import re +from datetime import datetime + +s3_client = boto3.client('s3') +LOGS_BUCKET = os.environ['LOGS_BUCKET_NAME'] + +# ECS-specific scannable file patterns +SCANNABLE_FILE_PATTERNS = [ + r'ecs[-_]agent', + r'ecs[-_]init', + r'docker[/.]', + r'containerd[/-]log', + r'dmesg', + r'messages', + r'secure', + r'syslog', + r'journal', + r'containers/', + r'container-logs', + r'var_log/.*\\.log', + r'networking', +] +SCANNABLE_REGEXES = [re.compile(p, re.IGNORECASE) for p in SCANNABLE_FILE_PATTERNS] + +SKIP_FILE_PATTERNS = [ + r'sysctl', r'ethtool', r'ifconfig', r'conntrack\\.txt', + r'pkglist', r'ps\\.txt', r'ps-threads', r'top\\.txt', + r'allprocstat', r'mounts\\.txt', r'modinfo', r'iptables', + r'metrics\\.json', r'\\.json$', r'\\.yaml$', r'\\.yml$', + r'\\.conf$', r'\\.toml$', r'\\.service$', + r'docker-info', r'docker-ps', r'docker-images', + r'instance-', r'metadata', +] +SKIP_REGEXES = [re.compile(p, re.IGNORECASE) for p in SKIP_FILE_PATTERNS] + +# ECS-specific error patterns for findings indexer +ERROR_PATTERNS = { + 'critical': [ + (r'(?i)agent.*connected.*false', 'ECS Agent disconnected'), + (r'(?i)AGENT_DISCONNECTED', 'ECS Agent disconnected from cluster'), + (r'(?i)failed.*register.*container.*instance', 'Container instance registration failed'), + (r'(?i)CannotPullContainerError', 'Cannot pull container image'), + (r'(?i)CannotPullECRContainerError', 'Cannot pull ECR container image'), + (r'(?i)ResourceInitializationError', 'Resource initialization failed'), + (r'(?i)TaskFailedToStart', 'Task failed to start'), + (r'(?i)CannotStartContainerError', 'Cannot start container'), + (r'(?i)CannotCreateContainerError', 'Cannot create container'), + (r'(?i)ContainerRuntimeError', 'Container runtime error'), + (r'(?i)pull.*access.*denied', 'Image pull access denied'), + (r'(?i)repository.*does.*not.*exist', 'Repository does not exist'), + (r'(?i)unauthorized.*authentication.*required', 'Authentication required'), + (r'(?i)toomanyrequests.*Too Many Requests', 'Docker Hub rate limit'), + (r'(?i)unable to pull secrets or registry auth', 'Secrets/registry auth failed'), + (r'(?i)docker.*daemon.*not.*running', 'Docker daemon not running'), + (r'(?i)OCI runtime create failed', 'OCI runtime create failed'), + (r'(?i)no.*space.*left.*device', 'No space left on device'), + (r'(?i)invoked oom-killer', 'OOM killer invoked'), + (r'(?i)Out of memory: Kill', 'OOM kill'), + (r'(?i)Memory cgroup out of memory', 'cgroup OOM'), + (r'(?i)kernel.*panic', 'kernel panic'), + (r'(?i)segfault', 'segfault'), + (r'(?i)ENI.*allocation.*failed', 'ENI allocation failed'), + (r'(?i)InsufficientFreeAddressesInSubnet', 'IP exhaustion'), + (r'(?i)deployment circuit breaker.*triggered', 'Circuit breaker triggered'), + (r'(?i)AccessDeniedException', 'Access denied'), + (r'(?i)exit code 137', 'Container killed (OOM)'), + (r'(?i)exit code 139', 'Container segfault'), + (r'(?i)SpotInterruptionError', 'Spot interruption'), + ], + 'warning': [ + (r'(?i)health.*check.*failed', 'Health check failed'), + (r'(?i)UNHEALTHY', 'Container unhealthy'), + (r'(?i)task.*stopped', 'Task stopped'), + (r'(?i)Essential container.*exited', 'Essential container exited'), + (r'(?i)service.*was unable to place a task', 'Task placement failure'), + (r'(?i)connection.*refused', 'Connection refused'), + (r'(?i)connection.*timeout', 'Connection timeout'), + (r'(?i)TLS.*handshake.*timeout', 'TLS timeout'), + (r'(?i)DNS.*failed', 'DNS resolution failed'), + (r'(?i)memory.*pressure', 'Memory pressure'), + (r'(?i)cpu.*throttl', 'CPU throttling'), + (r'(?i)log.*driver.*error', 'Log driver error'), + (r'(?i)awslogs.*error', 'awslogs driver error'), + (r'(?i)docker.*restart', 'Docker restarted'), + (r'(?i)is not authorized to perform', 'IAM permission denied'), + (r'(?i)OOMKilled', 'Container OOMKilled'), + (r'(?i)conntrack.*table.*full', 'Conntrack table full'), + ], + 'info': [ + (r'(?i)level=(?:warn|warning)', 'log-level warning'), + (r'(?i)DEPRECATION:', 'deprecation notice'), + (r'(?i)context deadline exceeded', 'context deadline'), + (r'(?i)retrying', 'retrying operation'), + ], +} + +COMPILED_PATTERNS = {} +for _sev, _pats in ERROR_PATTERNS.items(): + COMPILED_PATTERNS[_sev] = [] + for _pat, _desc in _pats: + try: + COMPILED_PATTERNS[_sev].append((re.compile(_pat), _desc)) + except re.error: + pass + +def is_scannable(filename): + for skip_re in SKIP_REGEXES: + if skip_re.search(filename): + return False + for scan_re in SCANNABLE_REGEXES: + if scan_re.search(filename): + return True + if filename.endswith('.log') or filename.endswith('.txt'): + log_dirs = ['ecs-agent', 'docker', 'var_log', 'kernel', 'system/messages'] + return any(d in filename for d in log_dirs) + return False + +def lambda_handler(event, context): + print(f"Received event: {json.dumps(event)}") + bucket = event.get('bucket', LOGS_BUCKET) + prefix = event.get('prefix', '') + if not prefix: + return {'statusCode': 400, 'body': 'No prefix provided'} + try: + files_to_scan = [] + files_skipped = 0 + paginator = s3_client.get_paginator('list_objects_v2') + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get('Contents', []): + key = obj['Key'] + if any(key.endswith(ext) for ext in ['.tar.gz', '.zip', '.gz', '.bin', '.so']): + continue + if obj['Size'] > 5242880: + continue + filename = key.split('/extracted/')[-1] if '/extracted/' in key else key + if not is_scannable(filename): + files_skipped += 1 + continue + files_to_scan.append({'key': key, 'size': obj['Size']}) + print(f"Scanning {len(files_to_scan)} log files ({files_skipped} non-log files skipped)") + all_findings = [] + for file_info in files_to_scan[:200]: + findings = scan_file(bucket, file_info['key']) + all_findings.extend(findings) + deduplicated = deduplicate_findings(all_findings) + + # False positive suppression + FALSE_POSITIVE_SUPPRESSIONS = [ + ('DNS.*failed', r'health[-.]?check|readiness|liveness', 'Health check DNS'), + ('OOMKilled', r'stress[-.]?test|load[-.]?test|chaos', 'Stress test'), + ('connection refused', r'127\\.0\\.0\\.1.*healthz', 'Local healthz'), + ('TLS handshake', r'health[-.]?check', 'Probe TLS'), + ] + suppressed_count = 0 + filtered = [] + for f in deduplicated: + suppressed = False + pat = f.get('pattern', '') + ctx = f.get('sample', '') + for err_pat, fp_regex, reason in FALSE_POSITIVE_SUPPRESSIONS: + if re.search(err_pat, pat, re.IGNORECASE): + if re.search(fp_regex, ctx, re.IGNORECASE): + suppressed = True + suppressed_count += 1 + break + if not suppressed: + filtered.append(f) + deduplicated = filtered + + for idx, finding in enumerate(deduplicated): + finding['finding_id'] = f"F-{idx + 1:03d}" + finding['evidence'] = {'source_file': finding.get('file', ''), 'full_key': finding.get('fullKey', ''), 'excerpt': finding.get('sample', '')[:500], 'line_range': {'start': finding.get('line', 0), 'end': finding.get('line', 0)}} + + # Multi-signal confirmation for critical findings + pattern_files = {} + for f in deduplicated: + p = f.get('pattern', '') + if p not in pattern_files: + pattern_files[p] = set() + pattern_files[p].add(f.get('file', '')) + for f in deduplicated: + if f.get('severity') == 'critical': + sources = list(pattern_files.get(f.get('pattern', ''), set())) + f['confirmation'] = {'signals': len(sources), 'confirmed': len(sources) >= 2, 'sources': sources[:5]} + if len(sources) < 2: + f['severity_note'] = 'Single-source critical finding. Verify with additional log sources.' + + summary = { + 'critical': len([f for f in deduplicated if f.get('severity') == 'critical']), + 'high': len([f for f in deduplicated if f.get('severity') in ('warning', 'high')]), + 'medium': len([f for f in deduplicated if f.get('severity') == 'medium']), + 'low': len([f for f in deduplicated if f.get('severity') == 'low']), + 'info': len([f for f in deduplicated if f.get('severity') == 'info']), + } + total_files_in_bundle = len(files_to_scan) + files_skipped + coverage = {'files_scanned': len(files_to_scan), 'total_files': total_files_in_bundle, 'coverage_pct': round(len(files_to_scan) / max(total_files_in_bundle, 1) * 100, 1), 'files_skipped': files_skipped, 'suppressed_false_positives': suppressed_count} + index_data = {'index_version': 'v2', 'indexedAt': datetime.utcnow().isoformat(), 'prefix': prefix, 'coverage': coverage, 'filesScanned': len(files_to_scan), 'filesSkipped': files_skipped, 'findings': deduplicated[:500], 'summary': summary} + index_key = f"{prefix}findings_index.json" + s3_client.put_object(Bucket=bucket, Key=index_key, Body=json.dumps(index_data, default=str), ContentType='application/json') + print(f"Wrote findings index to {index_key}") + print(f"Summary: {summary}") + return {'statusCode': 200, 'body': json.dumps({'message': 'Findings indexed', 'indexKey': index_key, 'summary': summary})} + except Exception as e: + print(f"Error indexing findings: {str(e)}") + return {'statusCode': 500, 'body': str(e)} + +def scan_file(bucket, key): + findings = [] + try: + response = s3_client.get_object(Bucket=bucket, Key=key, Range='bytes=0-1048575') + content = response['Body'].read() + try: + content_str = content.decode('utf-8') + except: + content_str = content.decode('latin-1', errors='ignore') + filename = key.split('/extracted/')[-1] if '/extracted/' in key else key + lines = content_str.split('\\n')[:5000] + for severity, compiled_list in COMPILED_PATTERNS.items(): + for regex, description in compiled_list: + match_count = 0 + for i, line in enumerate(lines): + if regex.search(line): + findings.append({'file': filename, 'fullKey': key, 'severity': severity, 'pattern': description, 'line': i + 1, 'sample': line.strip()[:300]}) + match_count += 1 + if match_count > 50 or len(findings) > 200: + break + if len(findings) > 200: + return findings + except Exception as e: + print(f"Error scanning {key}: {str(e)}") + return findings + +def deduplicate_findings(findings): + seen = {} + for finding in findings: + dedup_key = f"{finding.get('file')}:{finding.get('pattern')}" + if dedup_key not in seen: + seen[dedup_key] = {**finding, 'count': 1, 'lines': [finding.get('line')], 'first_seen': finding.get('sample', '')[:50], 'last_seen': finding.get('sample', '')[:50]} + else: + seen[dedup_key]['count'] += 1 + if len(seen[dedup_key]['lines']) < 10: + seen[dedup_key]['lines'].append(finding.get('line')) + seen[dedup_key]['last_seen'] = finding.get('sample', '')[:50] + severity_map = {'warning': 'high'} + for entry in seen.values(): + old_sev = entry.get('severity', 'info') + if old_sev in severity_map: + entry['severity'] = severity_map[old_sev] + severity_order = {'critical': 0, 'high': 1, 'medium': 2, 'low': 3, 'info': 4} + return sorted(seen.values(), key=lambda x: (severity_order.get(x.get('severity'), 4), -x.get('count', 0))) +`; + } +} diff --git a/mcp/ecs-instance-log-mcp/src/ecs-log-gateway-stack-v2.ts b/mcp/ecs-instance-log-mcp/src/ecs-log-gateway-stack-v2.ts new file mode 100644 index 0000000..ebb85ae --- /dev/null +++ b/mcp/ecs-instance-log-mcp/src/ecs-log-gateway-stack-v2.ts @@ -0,0 +1,31 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import { EcsLogGatewayConstructV2, EcsLogGatewayV2Props } from './ecs-log-gateway-construct-v2'; + +export interface EcsLogGatewayStackV2Props extends cdk.StackProps { + /** + * Properties for the ECS Log Gateway construct + */ + readonly gatewayProps?: EcsLogGatewayV2Props; +} + +/** + * Production-grade ECS Instance Log Collection MCP Server Stack + * + * Features: + * - Async task pattern with SSM Run Command + * - Byte-range streaming for large files + * - Manifest validation and completeness verification + * - AI-ready incident summary generation + * - KMS encryption at rest + * - Comprehensive ECS/Docker log collection + */ +export class EcsLogGatewayStackV2 extends cdk.Stack { + public readonly gateway: EcsLogGatewayConstructV2; + + constructor(scope: Construct, id: string, props?: EcsLogGatewayStackV2Props) { + super(scope, id, props); + + this.gateway = new EcsLogGatewayConstructV2(this, 'EcsLogGateway', props?.gatewayProps); + } +} diff --git a/mcp/ecs-instance-log-mcp/src/index.ts b/mcp/ecs-instance-log-mcp/src/index.ts new file mode 100644 index 0000000..8959fab --- /dev/null +++ b/mcp/ecs-instance-log-mcp/src/index.ts @@ -0,0 +1,2 @@ +export * from './ecs-log-gateway-construct-v2'; +export * from './ecs-log-gateway-stack-v2'; diff --git a/mcp/ecs-instance-log-mcp/src/lambda/ecs-log-automation.py b/mcp/ecs-instance-log-mcp/src/lambda/ecs-log-automation.py new file mode 100644 index 0000000..84abf2c --- /dev/null +++ b/mcp/ecs-instance-log-mcp/src/lambda/ecs-log-automation.py @@ -0,0 +1,6612 @@ +""" +ECS Instance Log Collection MCP Server - Enhanced Lambda Handler +Provides comprehensive ECS container instance log collection and analysis via SSM. +Similar to EKS Node Log MCP but tailored for ECS EC2 instances. +""" + +import json +import boto3 +import os +import re +import signal +import threading +import time +import hashlib +from contextlib import contextmanager +from datetime import datetime, timezone, timedelta +from typing import Dict, Any, List, Optional, Tuple +from concurrent.futures import ThreadPoolExecutor, as_completed +from enum import Enum +from botocore.exceptions import ClientError + +# AWS Clients - default region (where Lambda runs) +# S3 client uses SigV4 explicitly — required for presigned URLs on KMS-encrypted buckets +from botocore.config import Config as BotoConfig +ssm_client = boto3.client('ssm') +s3_client = boto3.client('s3', config=BotoConfig(signature_version='s3v4')) +ec2_client = boto3.client('ec2') +ecs_client = boto3.client('ecs') +sns_client = boto3.client('sns') + +# Regional client cache +_regional_clients: Dict[str, Any] = {} + +# Environment variables +LOGS_BUCKET = os.environ.get('LOGS_BUCKET_NAME', '') +SSM_AUTOMATION_ROLE_ARN = os.environ.get('SSM_AUTOMATION_ROLE_ARN', '') +KMS_KEY_ARN = os.environ.get('KMS_KEY_ARN', '') +DEFAULT_REGION = os.environ.get('AWS_REGION', 'us-east-1') + +# Human approval is fail-closed by default. Approvers and document names are +# deployment-owned configuration, never caller-controlled values. +APPROVAL_TOPIC_ARN = os.environ.get('APPROVAL_TOPIC_ARN', '') +COLLECT_APPROVAL_DOCUMENT = os.environ.get('COLLECT_APPROVAL_DOCUMENT', '') +BATCH_APPROVAL_DOCUMENT = os.environ.get('BATCH_APPROVAL_DOCUMENT', '') +TCPDUMP_APPROVAL_DOCUMENT = os.environ.get('TCPDUMP_APPROVAL_DOCUMENT', '') +APPROVAL_APPROVERS = [ + value.strip() for value in os.environ.get('APPROVAL_APPROVERS', '').split(',') + if value.strip() +] +APPROVAL_EMAILS_CONFIGURED = os.environ.get( + 'APPROVAL_EMAILS_CONFIGURED', '' +).strip().lower() in ('1', 'true', 'yes') +REQUIRE_COLLECTION_APPROVAL = os.environ.get( + 'REQUIRE_COLLECTION_APPROVAL', 'true' +).strip().lower() in ('1', 'true', 'yes') + +RESTRICTED_TOOLS = {'tcpdump_capture', 'tcpdump_analyze'} +ENABLED_RESTRICTED_TOOLS = { + value.strip() for value in os.environ.get('ENABLED_RESTRICTED_TOOLS', '').split(',') + if value.strip() +} +TOOL_AUTHORIZATION_ACL: Dict[str, set] = {} + + +def validate_tool_authorization(tool_name: str, caller: Optional[Dict] = None) -> Optional[Dict]: + """Fail closed for invasive tools unless explicitly enabled.""" + if tool_name in RESTRICTED_TOOLS and tool_name not in ENABLED_RESTRICTED_TOOLS: + return error_response( + 403, + f"Tool '{tool_name}' is restricted and not enabled. Set " + f"ENABLED_RESTRICTED_TOOLS to include '{tool_name}'.", + {'restrictedTools': sorted(RESTRICTED_TOOLS)}, + ) + if tool_name in TOOL_AUTHORIZATION_ACL: + allowed = TOOL_AUTHORIZATION_ACL[tool_name] + client_id = (caller or {}).get('client_id', '') + if not allowed or client_id not in allowed: + return error_response(403, f"Caller is not permitted to invoke '{tool_name}'.") + return None + +# Constants +MAX_CONCURRENT_READS = 10 +DEFAULT_MAX_BYTES = 100000 +DEFAULT_CHUNK_SIZE = 1048576 # 1MB +MAX_CHUNK_SIZE = 5242880 # 5MB +DEFAULT_LINE_COUNT = 1000 +MAX_LINE_COUNT = 10000 +FINDINGS_INDEX_FILE = 'findings_index.json' + + +# ============================================================================= +# PRESIGNED URL EXPIRATION — configurable via env var (T5 mitigation) +# ============================================================================= + +def _parse_presigned_url_expiration() -> int: + """Parse PRESIGNED_URL_EXPIRATION_SECONDS env var, default to 900.""" + raw = os.environ.get('PRESIGNED_URL_EXPIRATION_SECONDS', '') + try: + val = int(raw) + if val > 0: + return val + except (ValueError, TypeError): + pass + return 900 + +PRESIGNED_URL_EXPIRATION = _parse_presigned_url_expiration() + + +def _parse_pcap_presigned_url_expiration() -> int: + """Parse the sensitive pcap URL lifetime; cap it at five minutes.""" + try: + value = int(os.environ.get('PCAP_PRESIGNED_URL_EXPIRATION_SECONDS', '')) + if value > 0: + return min(value, 300) + except (TypeError, ValueError): + pass + return 60 + + +def _parse_max_pcap_bytes() -> int: + """Parse the size at which packet captures receive an oversized warning.""" + try: + value = int(os.environ.get('MAX_PCAP_BYTES', '')) + if value > 0: + return value + except (TypeError, ValueError): + pass + return 200 * 1024 * 1024 + + +PCAP_PRESIGNED_URL_EXPIRATION = _parse_pcap_presigned_url_expiration() +MAX_PCAP_BYTES = _parse_max_pcap_bytes() + + +# Conservative BPF allowlist. Shell metacharacters and control bytes are +# rejected before token parsing, including valid-but-risky tcpflags syntax. +_BPF_ALLOWED_KEYWORDS = frozenset({ + 'tcp', 'udp', 'icmp', 'arp', 'ip', 'ip6', 'ether', 'vlan', 'stp', + 'src', 'dst', 'host', 'net', 'port', 'portrange', 'proto', + 'and', 'or', 'not', 'greater', 'less', 'len', +}) +_BPF_VALUE = re.compile( + r'^(?:\d{1,5}|\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?|' + r'[0-9a-f:]+(?:/\d{1,3})?|\d+-\d+)$', re.IGNORECASE +) + + +def validate_bpf_filter(bpf_filter: str) -> Optional[str]: + """Return an error for filters outside a deliberately small safe subset.""" + if not bpf_filter: + return None + if len(bpf_filter) > 256: + return 'BPF filter too long (max 256 characters)' + if re.search(r'[\x00-\x1f\x7f`$\\;&|{}<>!~^\[\]]', bpf_filter): + return 'BPF filter contains forbidden characters' + depth = 0 + for char in bpf_filter: + depth += 1 if char == '(' else -1 if char == ')' else 0 + if depth < 0: + return 'BPF filter has unbalanced parentheses' + if depth: + return 'BPF filter has unbalanced parentheses' + for token in bpf_filter.replace('(', ' ').replace(')', ' ').split(): + if token.lower() not in _BPF_ALLOWED_KEYWORDS and not _BPF_VALUE.fullmatch(token): + return f"BPF filter contains disallowed token: '{token}'" + return None + + +# ============================================================================= +# DEPLOYMENT SCOPE — mandatory cluster and region allowlists +# ============================================================================= + +_INSTANCE_ID_RE = re.compile(r'^i-[0-9a-f]{8,17}$') +_CLUSTER_NAME_RE = re.compile(r'^[A-Za-z0-9_-]{1,255}$') +_REGION_RE = re.compile(r'^[a-z]{2}(?:-[a-z0-9]+)+-\d+$') + +ALLOWED_CLUSTER_NAMES = frozenset( + value.strip() for value in os.environ.get('ALLOWED_CLUSTER_NAMES', '').split(',') + if value.strip() +) +ALLOWED_REGIONS = frozenset( + value.strip() for value in os.environ.get('ALLOWED_REGIONS', '').split(',') + if value.strip() +) or frozenset({DEFAULT_REGION}) + + +def validate_instance_id(instance_id: str) -> Optional[Dict]: + """Require an exact EC2 instance identifier before it reaches AWS or S3.""" + if not isinstance(instance_id, str) or not _INSTANCE_ID_RE.fullmatch(instance_id): + return error_response(400, f'Invalid instanceId format: {instance_id}') + return None + + +def validate_cluster_name(cluster_name: str) -> Optional[Dict]: + """Validate ECS syntax exactly and enforce the deployment cluster allowlist.""" + if not isinstance(cluster_name, str) or not _CLUSTER_NAME_RE.fullmatch(cluster_name): + return error_response( + 400, + 'clusterName must be 1-255 characters using only letters, numbers, hyphens, and underscores.', + ) + if not ALLOWED_CLUSTER_NAMES: + return error_response(503, 'ALLOWED_CLUSTER_NAMES is empty; cluster access is disabled.') + if cluster_name not in ALLOWED_CLUSTER_NAMES: + return error_response( + 403, + f"Cluster '{cluster_name}' is not permitted. Allowed clusters: " + f"{', '.join(sorted(ALLOWED_CLUSTER_NAMES))}", + ) + return None + + +def validate_region(region: str) -> Optional[Dict]: + """Require an exact configured region and fail closed on malformed values.""" + if not isinstance(region, str) or not _REGION_RE.fullmatch(region): + return error_response(400, f"Invalid AWS region: '{region}'") + if region not in ALLOWED_REGIONS: + return error_response( + 403, + f"Region '{region}' is not permitted. Allowed regions: {', '.join(sorted(ALLOWED_REGIONS))}", + ) + return None + + +def resolve_and_validate_region(arguments: Dict, instance_id: str = None) -> tuple: + """Resolve a region and reject malformed, disallowed, or tampered values.""" + explicit = arguments.get('region') + if explicit is not None: + error = validate_region(explicit) + if error: + return explicit, error + region = resolve_region(arguments, instance_id) + return region, validate_region(region) + + +# ============================================================================= +# ECS INSTANCE VALIDATION — allowed clusters and exact ACTIVE membership only +# ============================================================================= + +def validate_ecs_instance(instance_id: str, region: str, + cluster_name: Optional[str] = None) -> Optional[Dict]: + """Verify exact ACTIVE ECS membership without account-wide enumeration or tags.""" + instance_error = validate_instance_id(instance_id) + if instance_error: + return instance_error + region_error = validate_region(region) + if region_error: + return region_error + if not ALLOWED_CLUSTER_NAMES: + return error_response(503, 'ALLOWED_CLUSTER_NAMES is empty; instance access is disabled.') + if cluster_name is not None: + cluster_error = validate_cluster_name(cluster_name) + if cluster_error: + return cluster_error + clusters_to_check = [cluster_name] + else: + clusters_to_check = sorted(ALLOWED_CLUSTER_NAMES) + + try: + regional_ec2 = get_regional_client('ec2', region) + response = regional_ec2.describe_instances(InstanceIds=[instance_id]) + instances = [ + instance + for reservation in response.get('Reservations', []) + for instance in reservation.get('Instances', []) + if instance.get('InstanceId') == instance_id + ] + if len(instances) != 1: + return error_response(404, f'Instance {instance_id} not found in region {region}') + + regional_ecs = get_regional_client('ecs', region) + for allowed_cluster in clusters_to_check: + container_token = None + while True: + request = {'cluster': allowed_cluster} + if container_token: + request['nextToken'] = container_token + try: + page = regional_ecs.list_container_instances(**request) + except ClientError as exc: + if exc.response.get('Error', {}).get('Code') == 'ClusterNotFoundException': + break + raise + container_arns = page.get('containerInstanceArns', []) + for offset in range(0, len(container_arns), 100): + described = regional_ecs.describe_container_instances( + cluster=allowed_cluster, + containerInstances=container_arns[offset:offset + 100], + ) + if described.get('failures'): + return error_response( + 500, f'Failed to verify ECS membership for {instance_id}' + ) + for container_instance in described.get('containerInstances', []): + if ( + container_instance.get('ec2InstanceId') == instance_id + and container_instance.get('status') == 'ACTIVE' + ): + return None + container_token = page.get('nextToken') + if not container_token: + break + return error_response( + 403, + f'Instance {instance_id} is not an ACTIVE container instance in an allowed ECS cluster in {region}.', + ) + except ClientError as exc: + if exc.response.get('Error', {}).get('Code') == 'InvalidInstanceID.NotFound': + return error_response(404, f'Instance {instance_id} not found in region {region}') + return error_response(500, f'Failed to validate instance {instance_id}: {exc}') + except Exception as exc: + return error_response(500, f'Failed to validate instance {instance_id}: {exc}') + + +# ============================================================================= +# TIME WINDOW RESOLVER — enforces time-bounded log analysis +# ============================================================================= + +class TimeWindowResolver: + """ + Resolves an analysis time window from user-provided incident time parameters. + + Rules: + 1. If start_time AND end_time provided: use exactly. + 2. If a single incident_time provided: window = [incident_time - 5min, incident_time + 5min]. + 3. If nothing provided: window = [now_utc - 10min, now_utc]. + + All outputs are UTC datetime objects. + """ + + DEFAULT_WINDOW_MINUTES = 10 + INCIDENT_PADDING_MINUTES = 5 + MAX_WINDOW_HOURS = 24 # safety cap + + @staticmethod + def resolve(arguments: Dict) -> Dict: + now_utc = datetime.utcnow() + incident_time_str = arguments.get('incident_time') + start_time_str = arguments.get('start_time') + end_time_str = arguments.get('end_time') + + window_start = None + window_end = None + reason = '' + + if start_time_str and end_time_str: + window_start = TimeWindowResolver._parse_timestamp(start_time_str) + window_end = TimeWindowResolver._parse_timestamp(end_time_str) + if window_start and window_end: + reason = 'explicit incident window provided' + else: + reason = 'failed to parse explicit window; default last 10 minutes' + window_start = None + window_end = None + + if window_start is None and incident_time_str: + incident_dt = TimeWindowResolver._parse_timestamp(incident_time_str) + if incident_dt: + pad = timedelta(minutes=TimeWindowResolver.INCIDENT_PADDING_MINUTES) + window_start = incident_dt - pad + window_end = incident_dt + pad + reason = f'incident time provided; applied +/- {TimeWindowResolver.INCIDENT_PADDING_MINUTES} minute padding' + else: + reason = 'failed to parse incident_time; default last 10 minutes' + + if window_start is None: + window_end = now_utc + window_start = now_utc - timedelta(minutes=TimeWindowResolver.DEFAULT_WINDOW_MINUTES) + if not reason: + reason = f'no incident time; default last {TimeWindowResolver.DEFAULT_WINDOW_MINUTES} minutes' + + # Safety cap + max_delta = timedelta(hours=TimeWindowResolver.MAX_WINDOW_HOURS) + if (window_end - window_start) > max_delta: + window_start = window_end - max_delta + reason += f' (clamped to max {TimeWindowResolver.MAX_WINDOW_HOURS}h window)' + + if window_end < window_start: + window_start, window_end = window_end, window_start + reason += ' (swapped start/end)' + + jctl_fmt = '%Y-%m-%d %H:%M:%S' + return { + 'window_start_utc': window_start, + 'window_end_utc': window_end, + 'window_start_iso': window_start.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'window_end_iso': window_end.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'resolution_reason': reason, + 'journalctl_since': window_start.strftime(jctl_fmt), + 'journalctl_until': window_end.strftime(jctl_fmt), + } + + @staticmethod + def _parse_timestamp(ts_str: str) -> Optional[datetime]: + if not ts_str or not isinstance(ts_str, str): + return None + ts_str = ts_str.strip() + for fmt in [ + '%Y-%m-%dT%H:%M:%SZ', + '%Y-%m-%dT%H:%M:%S', + '%Y-%m-%dT%H:%M:%S.%fZ', + '%Y-%m-%dT%H:%M:%S.%f', + '%Y-%m-%dT%H:%M:%S%z', + '%Y-%m-%d %H:%M:%S UTC', + '%Y-%m-%d %H:%M:%S', + '%Y-%m-%d %H:%M', + ]: + try: + dt = datetime.strptime(ts_str, fmt) + if dt.tzinfo: + dt = dt.astimezone(timezone.utc).replace(tzinfo=None) + return dt + except ValueError: + continue + try: + ts_float = float(ts_str) + if 1_000_000_000 < ts_float < 2_000_000_000: + return datetime.utcfromtimestamp(ts_float) + if 1_000_000_000_000 < ts_float < 2_000_000_000_000: + return datetime.utcfromtimestamp(ts_float / 1000) + except (ValueError, OSError): + pass + return None + + @staticmethod + def is_within_window(timestamp_str: str, window: Dict) -> bool: + dt = TimeWindowResolver._parse_timestamp(timestamp_str) + if dt is None: + return True # Conservative: include if unparseable + return window['window_start_utc'] <= dt <= window['window_end_utc'] + + @staticmethod + def filter_findings_by_window(findings: List[Dict], window: Dict) -> Dict: + included = [] + excluded_count = 0 + unparseable_count = 0 + for f in findings: + sample = f.get('sample', '') or f.get('line', '') + ts_str = extract_timestamp(sample) if sample else None + if ts_str is None: + unparseable_count += 1 + included.append(f) + continue + if TimeWindowResolver.is_within_window(ts_str, window): + included.append(f) + else: + excluded_count += 1 + return { + 'findings': included, + 'excluded_outside_window': excluded_count, + 'unparseable_timestamps': unparseable_count, + 'total_before_filter': len(findings), + } + + @staticmethod + def window_metadata(window: Dict) -> Dict: + return { + 'window_start_utc': window['window_start_iso'], + 'window_end_utc': window['window_end_iso'], + 'resolution_reason': window['resolution_reason'], + } + +# ============================================================================ +# ECS ERROR PATTERNS +# ============================================================================ + +ECS_ERROR_PATTERNS = { + 'critical': [ + (r'(?i)agent.*connected.*false', 'ECS Agent disconnected'), + (r'(?i)agent.*not.*connected', 'ECS Agent not connected'), + (r'(?i)failed.*register.*container.*instance', 'Container instance registration failed'), + (r'(?i)unable.*connect.*ecs', 'Unable to connect to ECS service'), + (r'(?i)No container instances were found', 'No container instances available'), + (r'(?i)AGENT_DISCONNECTED', 'ECS Agent disconnected from cluster'), + (r'(?i)ECS Agent failed to start', 'ECS Agent startup failure'), + (r'(?i)websocket.*unable to dial', 'Agent websocket connection failed'), + (r'(?i)Error getting ECS instance credentials', 'Agent credential retrieval failed'), + (r'(?i)client version.*is too old', 'Docker API version mismatch'), + (r'(?i)CannotPullContainerError', 'Cannot pull container image'), + (r'(?i)CannotPullECRContainerError', 'Cannot pull ECR container image'), + (r'(?i)ResourceInitializationError', 'Resource initialization failed'), + (r'(?i)TaskFailedToStart', 'Task failed to start'), + (r'(?i)CannotStartContainerError', 'Cannot start container'), + (r'(?i)CannotCreateContainerError', 'Cannot create container'), + (r'(?i)ContainerRuntimeError', 'Container runtime error'), + (r'(?i)ContainerRuntimeTimeoutError', 'Container runtime timeout'), + (r'(?i)CannotCreateVolumeError', 'Cannot create volume mount'), + (r'(?i)CannotInspectContainerError', 'Cannot inspect container'), + (r'(?i)CannotStopContainerError', 'Cannot stop container'), + (r'(?i)SpotInterruptionError', 'Spot capacity interruption'), + (r'(?i)InternalError', 'AWS internal error'), + (r'(?i)pull.*access.*denied', 'Image pull access denied'), + (r'(?i)repository.*does.*not.*exist', 'Repository does not exist'), + (r'(?i)manifest.*not.*found', 'Image manifest not found'), + (r'(?i)image.*not.*found', 'Image not found'), + (r'(?i)unauthorized.*authentication.*required', 'Authentication required for image pull'), + (r'(?i)no.*basic.*auth.*credentials', 'Missing authentication credentials'), + (r'(?i)denied.*requested.*access.*resource', 'Access denied to resource'), + (r'(?i)toomanyrequests.*Too Many Requests', 'Docker Hub rate limit exceeded'), + (r'(?i)failed to resolve ref.*not found', 'Image reference not found'), + (r'(?i)net/http.*request canceled while waiting', 'Image pull network timeout'), + (r'(?i)API error \(500\).*Get https://.*ecr', 'ECR API error 500'), + (r'(?i)ecr:BatchGetImage.*not authorized', 'Cross-account ECR access denied'), + (r'(?i)inspect image has been retried', 'Image inspection retry failure'), + (r'(?i)unable to pull secrets or registry auth', 'Secrets/registry auth pull failed'), + (r'(?i)unable to retrieve secret from asm', 'Secrets Manager retrieval failed'), + (r'(?i)unable to retrieve ecr registry auth', 'ECR registry auth retrieval failed'), + (r'(?i)failed to validate logger args', 'Logger args validation failed'), + (r'(?i)execution resource retrieval failed', 'Execution resource retrieval failed'), + (r'(?i)unable to get registry auth from asm', 'Private registry auth from ASM failed'), + (r'(?i)failed to initialize logging driver', 'Logging driver initialization failed'), + (r'(?i)service call has been retried.*times', 'Service call retry exhausted'), + (r'(?i)insufficient.*cpu.*units', 'Insufficient CPU units'), + (r'(?i)insufficient.*memory', 'Insufficient memory'), + (r'(?i)insufficient.*GPU.*units', 'Insufficient GPU units'), + (r'(?i)OutOfMemoryError', 'Out of memory error'), + (r'(?i)oom.*kill', 'OOM kill detected'), + (r'(?i)Memory cgroup out of memory', 'Memory cgroup OOM'), + (r'(?i)invoked oom-killer', 'OOM killer invoked'), + (r'(?i)Killed process.*total-vm', 'Process killed by OOM'), + (r'(?i)exit code 137', 'Container killed (exit 137 - OOM)'), + (r'(?i)exit code 139', 'Container segfault (exit 139 - SIGSEGV)'), + (r'(?i)exit code 255', 'Container ENTRYPOINT/CMD failed (exit 255)'), + (r'(?i)No valid providers in chain', 'IAM credential chain error'), + (r'(?i)unable.*assume.*role', 'Unable to assume IAM role'), + (r'(?i)AccessDeniedException', 'Access denied exception'), + (r'(?i)UnauthorizedOperation', 'Unauthorized operation'), + (r'(?i)is not authorized to perform', 'IAM permission denied'), + (r'(?i)execution role.*does not have', 'Task execution role missing permissions'), + (r'(?i)task role.*does not have', 'Task role missing permissions'), + (r'(?i)AssumeRoleUnauthorizedAccess', 'Cannot assume IAM role'), + (r'(?i)ecr:GetAuthorizationToken.*denied', 'ECR GetAuthorizationToken denied'), + (r'(?i)failed.*retrieve.*secrets', 'Failed to retrieve secrets'), + (r'(?i)SecretNotFound', 'Secret not found'), + (r'(?i)ParameterNotFound', 'SSM parameter not found'), + (r'(?i)AccessDenied.*secretsmanager', 'Secrets Manager access denied'), + (r'(?i)AccessDenied.*ssm', 'SSM access denied'), + (r'(?i)ResourceNotFoundException.*secret', 'Secret resource not found'), + (r'(?i)InvalidRequestException.*secret', 'Invalid secret request'), + (r'(?i)secretsmanager:GetSecretValue.*denied', 'GetSecretValue permission denied'), + (r'(?i)ssm:GetParameters.*denied', 'SSM GetParameters permission denied'), + (r'(?i)docker.*daemon.*not.*running', 'Docker daemon not running'), + (r'(?i)cannot.*connect.*docker', 'Cannot connect to Docker'), + (r'(?i)OCI runtime create failed', 'OCI runtime create failed'), + (r'(?i)containerd.*not.*running', 'containerd not running'), + (r'(?i)no.*space.*left.*device', 'No space left on device'), + (r'(?i)disk.*full', 'Disk full'), + (r'(?i)exec format error', 'Wrong image architecture'), + (r'(?i)container_linux.go.*starting container process', 'Container process start failed'), + (r'(?i)no such file or directory.*entrypoint', 'Entrypoint not found'), + (r'(?i)permission denied.*entrypoint', 'Entrypoint permission denied'), + (r'(?i)network.*unreachable', 'Network unreachable'), + (r'(?i)ENI.*allocation.*failed', 'ENI allocation failed'), + (r'(?i)failed.*create.*network.*interface', 'Failed to create network interface'), + (r'(?i)InsufficientFreeAddressesInSubnet', 'Insufficient IP addresses in subnet'), + (r'(?i)no.*available.*IP.*addresses', 'No available IP addresses'), + (r'(?i)Timeout waiting for network interface', 'ENI provisioning timeout'), + (r'(?i)deployment circuit breaker.*triggered', 'Deployment circuit breaker triggered'), + (r'(?i)ECS Deployment Circuit Breaker was triggered', 'Circuit breaker deployment failed'), + (r'(?i)kernel.*panic', 'Kernel panic'), + (r'(?i)BUG:.*', 'Kernel bug detected'), + (r'(?i)segfault', 'Segmentation fault'), + (r'(?i)watchdog.*soft.*lockup', 'Soft lockup detected'), + ], + 'warning': [ + (r'(?i)health.*check.*failed', 'Health check failed'), + (r'(?i)UNHEALTHY', 'Container unhealthy'), + (r'(?i)health.*status.*unhealthy', 'Health status unhealthy'), + (r'(?i)target.*unhealthy', 'Target unhealthy'), + (r'(?i)failed.*health.*check', 'Failed health check'), + (r'(?i)failed container health checks', 'Container health check failure'), + (r'(?i)failed ELB health checks', 'ELB health check failure'), + (r'(?i)Instance.*port.*is unhealthy', 'Instance port unhealthy'), + (r'(?i)task.*stopped', 'Task stopped'), + (r'(?i)container.*stopped.*unexpectedly', 'Container stopped unexpectedly'), + (r'(?i)container.*exited.*non-zero', 'Container exited with non-zero code'), + (r'(?i)Essential container.*exited', 'Essential container exited'), + (r'(?i)STOPPED', 'Task/container stopped'), + (r'(?i)DEPROVISIONING', 'Task deprovisioning'), + (r'(?i)exit code 143', 'Container graceful shutdown (SIGTERM)'), + (r'(?i)exit code 1', 'Container general error'), + (r'(?i)task.*stuck.*PROVISIONING', 'Task stuck in provisioning'), + (r'(?i)task.*stuck.*PENDING', 'Task stuck in pending'), + (r'(?i)service.*was unable to place a task', 'Service placement failure'), + (r'(?i)service.*has stopped.*running tasks', 'Service stopped tasks'), + (r'(?i)deployment circuit breaker.*rolling back', 'Deployment rolling back'), + (r'(?i)service.*unable to place a task', 'Task placement failure'), + (r'(?i)service.*discovery.*failed', 'Service discovery failed'), + (r'(?i)failed.*register.*service', 'Failed to register service'), + (r'(?i)DNS.*registration.*failed', 'DNS registration failed'), + (r'(?i)target.*draining', 'Target draining'), + (r'(?i)deregistering.*target', 'Deregistering target'), + (r'(?i)failed.*register.*target', 'Failed to register target'), + (r'(?i)target-group.*is unhealthy', 'Target group unhealthy'), + (r'(?i)scaling.*activity.*failed', 'Scaling activity failed'), + (r'(?i)unable.*scale', 'Unable to scale'), + (r'(?i)capacity.*provider.*error', 'Capacity provider error'), + (r'(?i)service.*began draining connections', 'Service draining connections'), + (r'(?i)connection.*refused', 'Connection refused'), + (r'(?i)connection.*timeout', 'Connection timeout'), + (r'(?i)dial.*tcp.*timeout', 'TCP dial timeout'), + (r'(?i)i/o timeout', 'I/O timeout'), + (r'(?i)TLS.*handshake.*timeout', 'TLS handshake timeout'), + (r'(?i)DNS.*failed', 'DNS resolution failed'), + (r'(?i)no.*route.*host', 'No route to host'), + (r'(?i)packet.*dropped', 'Packets dropped'), + (r'(?i)conntrack.*table.*full', 'Conntrack table full'), + (r'(?i)Post.*dial tcp.*timeout', 'HTTP POST timeout'), + (r'(?i)memory.*pressure', 'Memory pressure'), + (r'(?i)cpu.*throttl', 'CPU throttling'), + (r'(?i)disk.*pressure', 'Disk pressure'), + (r'(?i)inode.*exhausted', 'Inodes exhausted'), + (r'(?i)image.*pull.*slow', 'Slow image pull'), + (r'(?i)layer.*already.*exists', 'Layer already exists (potential issue)'), + (r'(?i)docker.*restart', 'Docker restarted'), + (r'(?i)docker.*timeout', 'Docker timeout'), + (r'(?i)log.*driver.*error', 'Log driver error'), + (r'(?i)failed.*send.*logs', 'Failed to send logs'), + (r'(?i)CloudWatch.*error', 'CloudWatch logging error'), + (r'(?i)awslogs.*error', 'awslogs driver error'), + (r'(?i)logs:CreateLogStream.*denied', 'CreateLogStream permission denied'), + (r'(?i)performing maintenance on.*infrastructure', 'AWS infrastructure maintenance'), + (r'(?i)task retirement', 'Task retirement notice'), + (r'(?i)error', 'Error detected'), + (r'(?i)fail', 'Failure detected'), + (r'(?i)denied', 'Access denied'), + (r'(?i)refused', 'Connection refused'), + (r'(?i)timeout', 'Timeout detected'), + (r'(?i)unauthorized', 'Unauthorized'), + (r'(?i)forbidden', 'Forbidden'), + (r'(?i)backoff', 'Backoff detected'), + ], + 'info': [ + (r'(?i)warn', 'Warning'), + (r'(?i)warning', 'Warning'), + (r'(?i)unable', 'Unable to perform operation'), + (r'(?i)cannot', 'Cannot perform operation'), + (r'(?i)invalid', 'Invalid configuration'), + (r'(?i)deprecated', 'Deprecated feature'), + (r'(?i)missing', 'Missing resource'), + (r'(?i)not found', 'Resource not found'), + (r'(?i)retrying', 'Retrying operation'), + (r'(?i)slow', 'Slow operation'), + (r'(?i)delayed', 'Delayed operation'), + (r'(?i)waiting', 'Waiting for resource'), + (r'(?i)pending', 'Pending operation'), + ] +} + +ECS_TRIAGE_CATEGORIES = { + 'A': {'name': 'Task Startup Failures', 'patterns': [(r'CannotPullContainerError', 'high'), (r'CannotPullECRContainerError', 'high'), (r'ResourceInitializationError', 'high'), (r'TaskFailedToStart', 'high'), (r'CannotStartContainerError', 'high'), (r'CannotCreateContainerError', 'high'), (r'ContainerRuntimeError', 'high'), (r'ContainerRuntimeTimeoutError', 'high'), (r'CannotCreateVolumeError', 'high'), (r'CannotInspectContainerError', 'high'), (r'CannotStopContainerError', 'medium'), (r'SpotInterruptionError', 'high'), (r'InternalError', 'high')], 'log_sources': ['ecs-agent', 'docker', 'containerd'], 'description': 'Task fails to start due to image pull, resource init, or container runtime issues', 'runbook': 'AWSSupport-TroubleshootECSTaskFailedToStart'}, + 'B': {'name': 'Image Pull Issues', 'patterns': [(r'pull.*access.*denied', 'high'), (r'repository.*does.*not.*exist', 'high'), (r'manifest.*not.*found', 'high'), (r'unauthorized.*authentication', 'high'), (r'no.*basic.*auth.*credentials', 'high'), (r'ECR.*token.*expired', 'high'), (r'toomanyrequests.*Too Many Requests', 'high'), (r'failed to resolve ref.*not found', 'high'), (r'net/http.*request canceled', 'high'), (r'API error \(500\)', 'high'), (r'ecr:BatchGetImage.*not authorized', 'high'), (r'inspect image has been retried', 'high')], 'log_sources': ['ecs-agent', 'docker'], 'description': 'ECR/Docker Hub authentication, image not found, registry connectivity, rate limits', 'docs': 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_cannot_pull_image.html'}, + 'C': {'name': 'IAM/Secrets Issues', 'patterns': [(r'No valid providers in chain', 'high'), (r'AccessDeniedException', 'high'), (r'is not authorized to perform', 'high'), (r'failed.*retrieve.*secrets', 'high'), (r'SecretNotFound', 'high'), (r'ParameterNotFound', 'high'), (r'AssumeRoleUnauthorizedAccess', 'high'), (r'ecr:GetAuthorizationToken.*denied', 'high'), (r'secretsmanager:GetSecretValue.*denied', 'high'), (r'ssm:GetParameters.*denied', 'high'), (r'unable to pull secrets or registry auth', 'high'), (r'unable to retrieve secret from asm', 'high'), (r'unable to retrieve ecr registry auth', 'high')], 'log_sources': ['ecs-agent', 'messages', 'secure'], 'description': 'Task execution role, task role, secrets manager, SSM parameter store', 'docs': 'https://repost.aws/knowledge-center/ecs-unable-to-pull-secrets'}, + 'D': {'name': 'Resource Exhaustion', 'patterns': [(r'insufficient.*cpu', 'high'), (r'insufficient.*memory', 'high'), (r'OutOfMemoryError', 'high'), (r'oom.*kill', 'high'), (r'exit code 137', 'high'), (r'exit code 139', 'high'), (r'no.*space.*left', 'high'), (r'Memory cgroup out of memory', 'high'), (r'invoked oom-killer', 'high'), (r'Killed process.*total-vm', 'high')], 'log_sources': ['dmesg', 'messages', 'cgroups', 'docker'], 'description': 'CPU/memory limits, OOM kills, disk space', 'docs': 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/out-of-memory.html'}, + 'E': {'name': 'Networking Issues', 'patterns': [(r'ENI.*allocation.*failed', 'high'), (r'InsufficientFreeAddressesInSubnet', 'high'), (r'network.*unreachable', 'high'), (r'connection.*refused', 'medium'), (r'connection.*timeout', 'medium'), (r'DNS.*failed', 'medium'), (r'Timeout waiting for network interface', 'high'), (r'failed.*create.*network.*interface', 'high'), (r'i/o timeout', 'medium'), (r'dial.*tcp.*timeout', 'medium')], 'log_sources': ['networking', 'ecs-agent', 'docker'], 'description': 'ENI allocation, subnet IP exhaustion, security groups, DNS, VPC endpoints'}, + 'F': {'name': 'Health Check Failures', 'patterns': [(r'health.*check.*failed', 'high'), (r'UNHEALTHY', 'high'), (r'target.*unhealthy', 'high'), (r'Essential container.*exited', 'high'), (r'failed container health checks', 'high'), (r'failed ELB health checks', 'high'), (r'Instance.*port.*is unhealthy', 'high')], 'log_sources': ['ecs-agent', 'docker', 'containers'], 'description': 'Container health checks, ALB/NLB target health'}, + 'G': {'name': 'ECS Agent Issues', 'patterns': [(r'agent.*connected.*false', 'high'), (r'AGENT_DISCONNECTED', 'high'), (r'failed.*register.*container.*instance', 'high'), (r'No container instances were found', 'high'), (r'ECS Agent failed to start', 'high'), (r'websocket.*unable to dial', 'high'), (r'Error getting ECS instance credentials', 'high'), (r'client version.*is too old', 'high')], 'log_sources': ['ecs-agent', 'messages'], 'description': 'Agent connectivity, instance registration, cluster communication'}, + 'H': {'name': 'Logging/Monitoring Issues', 'patterns': [(r'log.*driver.*error', 'medium'), (r'failed.*send.*logs', 'medium'), (r'CloudWatch.*error', 'medium'), (r'awslogs.*error', 'medium'), (r'failed to validate logger args', 'high'), (r'failed to initialize logging driver', 'high'), (r'logs:CreateLogStream.*denied', 'high')], 'log_sources': ['docker', 'ecs-agent'], 'description': 'CloudWatch logs, FireLens, log driver configuration'}, + 'I': {'name': 'Deployment/Circuit Breaker', 'patterns': [(r'deployment circuit breaker.*triggered', 'high'), (r'ECS Deployment Circuit Breaker was triggered', 'high'), (r'deployment circuit breaker.*rolling back', 'high'), (r'service.*was unable to place a task', 'high'), (r'service.*has stopped.*running tasks', 'medium')], 'log_sources': ['ecs-agent'], 'description': 'Deployment failures, circuit breaker triggers, rollbacks', 'docs': 'https://repost.aws/knowledge-center/ecs-troubleshoot-deployment-failures'}, + 'J': {'name': 'Container Runtime Issues', 'patterns': [(r'OCI runtime create failed', 'high'), (r'exec format error', 'high'), (r'container_linux.go.*starting container process', 'high'), (r'no such file or directory.*entrypoint', 'high'), (r'permission denied.*entrypoint', 'high'), (r'docker.*daemon.*not.*running', 'high'), (r'containerd.*not.*running', 'high')], 'log_sources': ['docker', 'containerd'], 'description': 'Docker/containerd runtime errors, entrypoint issues, architecture mismatch'}, +} + +ECS_LOG_TYPE_PATTERNS = { + 'ecs-agent': ['ecs-agent', 'ecs/', 'ecs_agent', 'ecs-init', 'amazon-ecs-agent', + 'ecs.config', 'ecs_agent_data', 'agent-running-info'], + 'docker': ['docker', 'daemon.json', 'containerd', 'docker-info', 'docker-ps', + 'docker-images', 'docker-version', 'docker-stats', 'docker-not-running', + 'sysconfig-docker', 'docker-storage', 'docker.service', 'containerd.service'], + 'containers': ['containers/', 'container-logs', 'container-'], + 'system': ['messages', 'syslog', 'secure', 'audit', 'journal', 'system.log', + 'services.txt', 'top.txt', 'ps.txt', 'pkglist', 'os-release', + 'uname', 'dmidecode', 'lsmod', 'open-file', 'mounts', + 'lvdisplay', 'vgdisplay', 'pvdisplay', 'selinux'], + 'dmesg': ['dmesg'], + 'networking': ['networking', 'iptables', 'ip-', 'netstat', 'ss-', + 'brctlshow', 'ipaddrshow', 'veth'], + 'cgroups': ['cgroup', 'memory-events', 'memory-stat', 'cgroupv2', + 'system.slice', 'ecstasks.slice'], + 'metadata': ['metadata', 'instance-'], + 'gpu': ['gpu', 'nvidia', 'gpu-list', 'gpu-info', 'gpu-open-module', 'gpu-installed-kmod'], +} + + +# Pre-compile all ECS_ERROR_PATTERNS at module level +COMPILED_ERROR_PATTERNS = {} +for _sev_key, _patterns in ECS_ERROR_PATTERNS.items(): + COMPILED_ERROR_PATTERNS[_sev_key] = [] + for _pat, _desc in _patterns: + try: + COMPILED_ERROR_PATTERNS[_sev_key].append((re.compile(_pat, re.IGNORECASE), _desc)) + except re.error: + pass + +# False positive suppression patterns +FALSE_POSITIVE_PATTERNS = [ + re.compile(r'(?i)error_count["\s]*[:=]\s*0'), + re.compile(r'(?i)no\s+errors?\s+found'), + re.compile(r'(?i)errors?["\s]*[:=]\s*null'), + re.compile(r'(?i)error_rate["\s]*[:=]\s*0'), + re.compile(r'(?i)--error-'), + re.compile(r'(?i)error\.log'), + re.compile(r'(?i)if.*error'), + re.compile(r'(?i)catch.*error'), + re.compile(r'(?i)handle.*error'), +] + +# ============================================================================ +# SEVERITY +# ============================================================================ + +class Severity(Enum): + CRITICAL = 'critical' + HIGH = 'high' + MEDIUM = 'medium' + LOW = 'low' + INFO = 'info' + +SEVERITY_ORDER = {'critical': 0, 'high': 1, 'medium': 2, 'low': 3, 'info': 4} + +# ============================================================================ +# SOP KEYWORD MAP — maps issue keywords to runbook files +# ============================================================================ + +SOP_KEYWORD_MAP = { + # ── Task Startup Failures ── + 'task_startup': [ + {'sop': 'runbooks/A1-task-startup-resource-init.md', 'keywords': ['ResourceInitializationError', 'TaskFailedToStart', 'CannotStartContainerError', 'CannotCreateContainerError', 'resource.*init'], 'relevance': 'primary'}, + {'sop': 'runbooks/A2-task-startup-container-runtime.md', 'keywords': ['ContainerRuntimeError', 'ContainerRuntimeTimeoutError', 'docker.*daemon.*not.*running', 'containerd.*not.*running'], 'relevance': 'primary'}, + ], + # ── Image Pull Issues ── + 'image_pull': [ + {'sop': 'runbooks/B1-ecr-image-pull-auth.md', 'keywords': ['CannotPullECRContainerError', 'ecr:GetAuthorizationToken.*denied', 'ecr:BatchGetImage.*not authorized', 'pull.*access.*denied', 'CannotPullContainerError'], 'relevance': 'primary'}, + {'sop': 'runbooks/B2-image-not-found.md', 'keywords': ['manifest.*not.*found', 'repository.*does.*not.*exist', 'image.*not.*found', 'failed to resolve ref.*not found'], 'relevance': 'primary'}, + {'sop': 'runbooks/B3-docker-hub-rate-limit.md', 'keywords': ['toomanyrequests', 'Too Many Requests', 'rate limit', 'docker.io.*rate'], 'relevance': 'primary'}, + ], + # ── IAM / Secrets Issues ── + 'iam_secrets': [ + {'sop': 'runbooks/C1-task-execution-role-permissions.md', 'keywords': ['AccessDeniedException', 'is not authorized to perform', 'No valid providers in chain', 'AssumeRoleUnauthorizedAccess', 'execution role.*does not have', 'UnauthorizedOperation'], 'relevance': 'primary'}, + {'sop': 'runbooks/C2-secrets-manager-retrieval.md', 'keywords': ['unable to pull secrets', 'unable to retrieve secret', 'SecretNotFound', 'ParameterNotFound', 'secretsmanager:GetSecretValue.*denied', 'ssm:GetParameters.*denied', 'execution resource retrieval failed'], 'relevance': 'primary'}, + ], + # ── Resource Exhaustion ── + 'resource_exhaustion': [ + {'sop': 'runbooks/D1-oom-kill-memory.md', 'keywords': ['OutOfMemoryError', 'oom.*kill', 'Memory cgroup out of memory', 'invoked oom-killer', 'exit code 137', 'Killed process.*total-vm', 'insufficient.*memory'], 'relevance': 'primary'}, + {'sop': 'runbooks/D2-disk-space-exhaustion.md', 'keywords': ['no.*space.*left.*device', 'disk.*full', 'disk.*pressure', 'inode.*exhausted'], 'relevance': 'primary'}, + {'sop': 'runbooks/D3-cpu-throttling.md', 'keywords': ['cpu.*throttl', 'insufficient.*cpu', 'cpu.*limit'], 'relevance': 'primary'}, + ], + # ── Networking Issues ── + 'networking': [ + {'sop': 'runbooks/E1-eni-allocation-subnet-ip.md', 'keywords': ['ENI.*allocation.*failed', 'InsufficientFreeAddressesInSubnet', 'Timeout waiting for network interface', 'failed.*create.*network.*interface', 'no.*available.*IP'], 'relevance': 'primary'}, + {'sop': 'runbooks/E2-dns-resolution-failures.md', 'keywords': ['DNS.*failed', 'resolve.*fail', 'SERVFAIL', 'NXDOMAIN', 'name.*resolution', 'no.*route.*host'], 'relevance': 'primary'}, + {'sop': 'runbooks/E3-connection-timeout.md', 'keywords': ['connection.*timeout', 'network.*unreachable', 'dial.*tcp.*timeout', 'i/o timeout', 'TLS.*handshake.*timeout', 'connection.*refused'], 'relevance': 'primary'}, + ], + # ── Health Check Failures ── + 'health_checks': [ + {'sop': 'runbooks/F1-container-health-check.md', 'keywords': ['health.*check.*failed', 'UNHEALTHY', 'failed container health checks', 'health.*status.*unhealthy'], 'relevance': 'primary'}, + {'sop': 'runbooks/F2-elb-target-health.md', 'keywords': ['target.*unhealthy', 'failed ELB health checks', 'Instance.*port.*is unhealthy', 'deregistering.*target'], 'relevance': 'primary'}, + ], + # ── ECS Agent Issues ── + 'ecs_agent': [ + {'sop': 'runbooks/G1-agent-disconnected.md', 'keywords': ['agent.*connected.*false', 'AGENT_DISCONNECTED', 'websocket.*unable to dial', 'Error getting ECS instance credentials', 'agent.*not.*connected'], 'relevance': 'primary'}, + {'sop': 'runbooks/G2-instance-registration-failure.md', 'keywords': ['failed.*register.*container.*instance', 'No container instances were found', 'ECS Agent failed to start', 'client version.*is too old'], 'relevance': 'primary'}, + ], + # ── Logging/Monitoring Issues ── + 'logging': [ + {'sop': 'runbooks/H1-cloudwatch-log-driver.md', 'keywords': ['log.*driver.*error', 'failed.*send.*logs', 'awslogs.*error', 'failed to initialize logging driver', 'logs:CreateLogStream.*denied', 'CloudWatch.*error', 'failed to validate logger args'], 'relevance': 'primary'}, + ], + # ── Deployment / Circuit Breaker ── + 'deployment': [ + {'sop': 'runbooks/I1-deployment-circuit-breaker.md', 'keywords': ['deployment circuit breaker.*triggered', 'ECS Deployment Circuit Breaker', 'circuit breaker.*rolling back', 'service.*was unable to place a task', 'service.*has stopped.*running tasks'], 'relevance': 'primary'}, + ], + # ── Container Runtime Issues ── + 'container_runtime': [ + {'sop': 'runbooks/J1-oci-runtime-entrypoint.md', 'keywords': ['OCI runtime create failed', 'exec format error', 'container_linux.go.*starting container process', 'no such file or directory.*entrypoint', 'permission denied.*entrypoint'], 'relevance': 'primary'}, + ], + # ── Spot Interruption / Instance Draining ── + 'spot_interruption': [ + {'sop': 'runbooks/K1-spot-interruption-instance-draining.md', 'keywords': ['spot.*interrupt', 'instance.*drain', 'DRAINING', 'Spot Instance interruption', 'rebalance.*recommendation', 'capacity.*rebalance'], 'relevance': 'primary'}, + ], + # ── Task Placement Failures ── + 'task_placement': [ + {'sop': 'runbooks/K2-task-placement-failures.md', 'keywords': ['no container instance.*met.*requirements', 'placement.*constraint', 'placement.*strategy', 'unable to place', 'distinctInstance', 'memberOf', 'attribute:ecs'], 'relevance': 'primary'}, + ], + # ── Service Steady State Failures ── + 'service_stability': [ + {'sop': 'runbooks/K3-service-steady-state-failures.md', 'keywords': ['unable to reach steady state', 'steady state', 'has stopped.*running tasks', 'rolling back', 'service.*unstable', 'deployment.*failed'], 'relevance': 'primary'}, + ], + # ── Auto Scaling / Capacity Provider ── + 'auto_scaling': [ + {'sop': 'runbooks/K4-service-auto-scaling-issues.md', 'keywords': ['auto.*scal', 'capacity.*provider', 'managed.*scaling', 'target.*tracking', 'scaling.*policy', 'desired.*count', 'CapacityProviderReservation'], 'relevance': 'primary'}, + ], + # ── ECS Exec / SSM Failures ── + 'ecs_exec': [ + {'sop': 'runbooks/K5-ecs-exec-failures.md', 'keywords': ['ECS Exec', 'execute-command', 'ExecuteCommandAgent', 'SSM.*session', 'ssmmessages', 'session.*manager', 'exec.*failed'], 'relevance': 'primary'}, + ], + # ── Service Connect / Cloud Map ── + 'service_connect': [ + {'sop': 'runbooks/K6-service-connect-discovery-failures.md', 'keywords': ['Service Connect', 'Cloud Map', 'service.*discovery', 'namespace.*not.*found', 'servicediscovery', 'cloudmap', 'DNS.*SRV'], 'relevance': 'primary'}, + ], + # ── EBS/EFS Volume Mount Failures ── + 'volume_mount': [ + {'sop': 'runbooks/K7-ebs-efs-volume-mount-failures.md', 'keywords': ['volume.*mount.*fail', 'EBS.*attach', 'EFS.*mount', 'nfs.*timeout', 'mount.*target', 'volume.*not.*found', 'bind.*mount'], 'relevance': 'primary'}, + ], + # ── Task Stuck in PENDING ── + 'task_pending': [ + {'sop': 'runbooks/K8-task-stuck-pending.md', 'keywords': ['PENDING', 'PROVISIONING', 'stuck', 'task not starting', 'waiting for capacity', 'timed out waiting'], 'relevance': 'primary'}, + ], + # ── Fargate Platform / Ephemeral Storage ── + 'fargate_platform': [ + {'sop': 'runbooks/K9-fargate-platform-ephemeral-storage.md', 'keywords': ['platform version', 'ephemeral storage', 'no space left.*fargate', 'platform 1\\.3', 'platform 1\\.4', 'storage exceeded', 'ephemeralStorage'], 'relevance': 'primary'}, + ], + # ── API Throttling / Service Quotas ── + 'api_throttling': [ + {'sop': 'runbooks/K10-api-throttling-service-quotas.md', 'keywords': ['throttl', 'rate.*limit', 'Rate exceeded', 'TooManyRequestsException', 'Limit exceeded', 'service.*quota', 'RequestLimitExceeded', 'Operations are being throttled'], 'relevance': 'primary'}, + ], + # ── Fargate Metadata / Credential Errors ── + 'metadata_credentials': [ + {'sop': 'runbooks/K11-fargate-metadata-credential-errors.md', 'keywords': ['Missing credentials', 'could not load credentials', 'metadata.*error', 'credential.*provider', '169\\.254\\.170', 'ECS_CONTAINER_METADATA', 'IMDS'], 'relevance': 'primary'}, + ], + # ── Essential Container Exited Non-Zero ── + 'container_exit': [ + {'sop': 'runbooks/K12-essential-container-exited-nonzero.md', 'keywords': ['EssentialContainerExited', 'exit code', 'non-zero', 'exit.*137', 'exit.*139', 'exit.*143', 'SIGKILL', 'SIGTERM', 'segfault', 'essential container.*exit'], 'relevance': 'primary'}, + ], + # ── Windows Container Issues ── + 'windows': [ + {'sop': 'runbooks/K13-windows-container-issues.md', 'keywords': ['Windows', 'OS mismatch', 'operating system does not match', 'EnableTaskIAMRole', 'ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE', 'No valid providers in chain.*windows', 'Unable to assume the role.*windows', 'Windows Server'], 'relevance': 'primary'}, + ], + # ── Task Latency / Performance ── + 'performance': [ + {'sop': 'runbooks/K14-task-latency-performance.md', 'keywords': ['latency', 'slow', 'performance', 'response time', 'TargetResponseTime', 'TTFB', 'EBS.*throttl', 'network.*throughput', 'DNS.*slow'], 'relevance': 'primary'}, + ], + # ── Docker Daemon / Agent Errors ── + 'docker_agent': [ + {'sop': 'runbooks/K15-docker-daemon-agent-errors.md', 'keywords': ['Docker.*API.*500', 'devmapper', 'thin pool', 'docker.*daemon', 'containerd.*error', 'docker\\.sock', 'storage.*driver', 'container.*runtime.*error'], 'relevance': 'primary'}, + ], +} + +# Map ECS triage categories (A-K) to SOP keyword groups +TRIAGE_CATEGORY_TO_SOP_GROUP = { + 'A': ['task_startup', 'image_pull', 'container_exit', 'fargate_platform'], + 'B': ['image_pull'], + 'C': ['iam_secrets', 'metadata_credentials'], + 'D': ['resource_exhaustion', 'performance'], + 'E': ['networking', 'service_connect'], + 'F': ['health_checks'], + 'G': ['ecs_agent', 'docker_agent'], + 'H': ['logging'], + 'I': ['deployment', 'service_stability', 'auto_scaling'], + 'J': ['container_runtime', 'docker_agent', 'windows'], + 'K': ['spot_interruption', 'task_placement', 'task_pending', 'ecs_exec', 'volume_mount', 'api_throttling'], +} + + +def match_sops_for_issues(issues: List[Dict], findings: List[Dict] = None, + triage_category: str = None, max_sops: int = 5) -> List[Dict]: + """ + Match detected issues/findings against SOP runbooks. + Returns a list of recommended SOPs with relevance and reason. + + Args: + issues: List of issue dicts from diagnostics (each has 'message' and 'section') + findings: Optional list of error findings (each has 'pattern', 'sample') + triage_category: Optional triage category ID (A-J) from ECS triage root cause + max_sops: Maximum SOPs to return + """ + scored_sops = {} # sop_name -> {score, reasons, keywords} + + # Build a combined text corpus from issues and findings for keyword matching + issue_texts = [] + for issue in (issues or []): + issue_texts.append(issue.get('message', '')) + for finding in (findings or []): + issue_texts.append(finding.get('pattern', '')) + issue_texts.append(finding.get('sample', '')[:200]) + corpus = ' '.join(issue_texts).lower() + + # If triage category is known, prioritize SOPs from that category's groups + priority_groups = set() + if triage_category and triage_category in TRIAGE_CATEGORY_TO_SOP_GROUP: + priority_groups = set(TRIAGE_CATEGORY_TO_SOP_GROUP[triage_category]) + + for group_name, sop_entries in SOP_KEYWORD_MAP.items(): + is_priority = group_name in priority_groups + for entry in sop_entries: + sop_name = entry['sop'] + matched_keywords = [] + for kw in entry['keywords']: + try: + if re.search(kw, corpus, re.IGNORECASE): + matched_keywords.append(kw) + except re.error: + if kw.lower() in corpus: + matched_keywords.append(kw) + + if matched_keywords: + if sop_name not in scored_sops: + scored_sops[sop_name] = {'score': 0, 'reasons': [], 'keywords': []} + scored_sops[sop_name]['score'] += len(matched_keywords) * 3 + if is_priority: + scored_sops[sop_name]['score'] += 5 + scored_sops[sop_name]['keywords'].extend(matched_keywords[:3]) + scored_sops[sop_name]['reasons'].append( + f"Matched {len(matched_keywords)} keyword(s) from {group_name}" + ) + + # Always include Z1 general troubleshooting if any issues exist but no specific SOPs matched + if not scored_sops and (issues or findings): + scored_sops['runbooks/Z1-general-troubleshooting.md'] = { + 'score': 1, + 'reasons': ['General troubleshooting guide for unmatched issues'], + 'keywords': [] + } + + # Sort by score descending, take top N + sorted_sops = sorted(scored_sops.items(), key=lambda x: x[1]['score'], reverse=True) + result = [] + for sop_name, info in sorted_sops[:max_sops]: + result.append({ + 'sopName': sop_name, + 'relevanceScore': info['score'], + 'matchedKeywords': list(set(info['keywords']))[:5], + 'reason': '; '.join(info['reasons'][:2]), + }) + return result + + +def normalize_severity_filter(severity_filter: str) -> list: + if severity_filter == 'all': + return ['critical', 'warning', 'info'] + if severity_filter in ('critical', 'warning', 'info'): + return [severity_filter] + return ['critical', 'warning', 'info'] + + +def assign_finding_id(index: int) -> str: + return f"F-{index:03d}" + + +# ============================================================================ +# RESPONSE HELPERS +# ============================================================================ + +def success_response(data: Dict) -> Dict: + MAX_PAYLOAD_BYTES = 5_500_000 + body = json.dumps({'success': True, **data}, default=str) + if len(body.encode('utf-8')) > MAX_PAYLOAD_BYTES: + truncated_data = {k: v for k, v in data.items() if not isinstance(v, list)} + for k, v in data.items(): + if isinstance(v, list): + trimmed = v + while trimmed: + candidate = json.dumps({ + 'success': True, **truncated_data, k: trimmed, + '_payloadTruncated': True, '_originalCount': len(v), '_returnedCount': len(trimmed), + }, default=str) + if len(candidate.encode('utf-8')) <= MAX_PAYLOAD_BYTES: + return {'statusCode': 200, 'body': candidate} + trimmed = trimmed[:len(trimmed) // 2] + truncated_data[k] = [] + body = json.dumps({'success': True, **truncated_data, '_payloadTruncated': True, '_error': 'Response too large'}, default=str) + return {'statusCode': 200, 'body': body} + + +def error_response(code: int, message: str, details: Dict = None) -> Dict: + body = {'success': False, 'error': message} + if details: + body['details'] = details + return {'statusCode': code, 'body': json.dumps(body, default=str)} + + +# ============================================================================ +# PUBLIC S3 KEY AND REGEX SAFETY +# ============================================================================ + +_PRIVATE_KEY_PREFIXES = ( + '_metadata/', 'batches/', 'execution-regions/', 'idempotency/', + 'tcpdump/', 'tcpdump-commands/', 'tcpdump-executions/', 'baselines/', +) +MAX_PUBLIC_LOG_KEY_LENGTH = 1024 +REGEX_FILE_TIMEOUT_SECONDS = 2.0 + + +class RegexTimeout(Exception): + """Raised when a per-file regex scan exceeds its wall-clock budget.""" + + +class RegexTimeoutUnavailable(Exception): + """Raised when the hard timeout cannot be installed safely.""" + + +def validate_log_key(log_key: str, instance_id: str) -> Optional[Dict]: + """Allow public reads only within the exact instance's canonical log prefix.""" + instance_error = validate_instance_id(instance_id) + if instance_error: + return instance_error + if not isinstance(log_key, str) or not log_key: + return error_response(400, 'logKey is required') + if len(log_key) > MAX_PUBLIC_LOG_KEY_LENGTH: + return error_response(400, f'logKey too long (max {MAX_PUBLIC_LOG_KEY_LENGTH} characters)') + if log_key.startswith('/') or '\\' in log_key: + return error_response(400, 'logKey contains an illegal path prefix or separator') + if any(ord(character) < 0x20 or ord(character) == 0x7f for character in log_key): + return error_response(400, 'logKey contains control characters') + if log_key.startswith(_PRIVATE_KEY_PREFIXES): + return error_response(403, 'logKey references private gateway metadata') + segments = log_key.split('/') + if any(segment in ('', '.', '..') for segment in segments): + return error_response(400, 'logKey contains an empty or traversal path segment') + expected_prefix = f'ecs_{instance_id}/' + if not log_key.startswith(expected_prefix): + return error_response( + 403, + f"logKey must be below the instance prefix '{expected_prefix}' and may not access another instance.", + ) + private_instance_segments = { + '_metadata', 'batches', 'execution-regions', 'idempotency', + 'tcpdump', 'tcpdump-commands', 'tcpdump-executions', 'baselines', + } + if any(segment in private_instance_segments for segment in segments[1:]): + return error_response(403, 'logKey references private gateway metadata') + if log_key.lower().endswith(('.pcap', '.pcapng')) or '/tcpdump/' in log_key.lower(): + return error_response(403, 'Generic artifact access cannot return packet captures') + return None + + +def is_dangerous_regex(pattern: str) -> bool: + """Conservatively reject common catastrophic-backtracking constructions.""" + if not isinstance(pattern, str) or len(pattern) > 500: + return True + # Nested quantifiers: (a+)+, (.*){2,}, (\w*)?. + for match in re.finditer(r'\(([^()]*)\)\s*(?:[*+?]|\{\d*(?:,\d*)?\})', pattern): + body = match.group(1) + if re.search(r'(?:[*+?]|\{\d*(?:,\d*)?\})', body): + return True + # Repeated alternations with overlapping prefixes, such as (a|aa)+. + alternatives = [part for part in body.split('|') if part] + for left in alternatives: + for right in alternatives: + if left != right and (left.startswith(right) or right.startswith(left)): + return True + # Backreferences and recursive-looking lookarounds are unnecessary for log search. + if re.search(r'\\[1-9]|\(\?[ 0: + signal.setitimer(signal.ITIMER_REAL, *previous_timer) + + +# ============================================================================ +# S3 SAFE HELPERS +# ============================================================================ + +def safe_s3_read_raw(bucket: str, key: str, byte_range: str = None, s3c=None) -> Optional[bytes]: + """Legacy S3 read — returns raw bytes or None.""" + try: + c = s3c or s3_client + kwargs = {'Bucket': bucket, 'Key': key} + if byte_range: + kwargs['Range'] = byte_range + return c.get_object(**kwargs)['Body'].read() + except Exception: + return None + + +def safe_s3_head_raw(bucket: str, key: str, s3c=None) -> Optional[Dict]: + """Legacy S3 head — returns raw boto3 response or None.""" + try: + c = s3c or s3_client + return c.head_object(Bucket=bucket, Key=key) + except Exception: + return None + + +def safe_s3_read(key: str, range_bytes: str = None, max_size: int = 1048576) -> Dict: + """ + EKS-compatible S3 read — returns dict with 'success', 'content' or 'error'. + NEVER raises exceptions. + """ + try: + params = {'Bucket': LOGS_BUCKET, 'Key': key} + if range_bytes: + params['Range'] = range_bytes + elif max_size: + params['Range'] = f'bytes=0-{max_size - 1}' + response = s3_client.get_object(**params) + content = response['Body'].read() + try: + content_str = content.decode('utf-8') + except UnicodeDecodeError: + content_str = content.decode('latin-1', errors='replace') + return {'success': True, 'content': content_str, 'size': len(content), + 'content_type': response.get('ContentType', 'unknown')} + except Exception as e: + return {'success': False, 'error': f'Failed to read {key}: {str(e)}', + 'error_type': 'read_error', 'content': ''} + + +def safe_s3_head(key: str) -> Dict: + """ + EKS-compatible S3 head — returns dict with 'success', 'size' or 'error'. + NEVER raises exceptions. + """ + try: + response = s3_client.head_object(Bucket=LOGS_BUCKET, Key=key) + return {'success': True, 'size': response['ContentLength'], + 'content_type': response.get('ContentType', 'unknown'), + 'last_modified': response.get('LastModified')} + except Exception as e: + return {'success': False, 'error': f'Failed to get metadata for {key}: {str(e)}', + 'error_type': 'metadata_error'} + + +def safe_s3_list(prefix: str, max_keys: int = 1000) -> Dict: + """ + Safely list S3 objects with graceful error handling. + NEVER raises exceptions - always returns a result dict. + """ + try: + all_objects = [] + paginator = s3_client.get_paginator('list_objects_v2') + for page in paginator.paginate(Bucket=LOGS_BUCKET, Prefix=prefix, PaginationConfig={'MaxItems': max_keys}): + for obj in page.get('Contents', []): + all_objects.append({ + 'key': obj['Key'], + 'size': obj['Size'], + 'last_modified': obj.get('LastModified') + }) + return { + 'success': True, + 'objects': all_objects, + 'count': len(all_objects) + } + except Exception as e: + return { + 'success': False, + 'error': f'Failed to list objects with prefix {prefix}: {str(e)}', + 'error_type': 'list_error', + 'objects': [], + 'count': 0 + } + + +def safe_s3_list_raw(bucket: str, prefix: str, max_keys: int = 1000, s3c=None) -> List[Dict]: + """Legacy list helper that returns raw S3 Contents list.""" + try: + c = s3c or s3_client + resp = c.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=max_keys) + return resp.get('Contents', []) + except Exception: + return [] + + +def find_latest_bundle_files(instance_id: str, prefix_scheme: str = 'ecs') -> Dict: + """ + Discover the latest extracted bundle in the exact canonical instance namespace. + + Publicly consumable bundle objects must be below ``ecs_{instanceId}/``. The + trailing delimiter prevents one instance ID from prefix-matching another. + """ + instance_error = validate_instance_id(instance_id) + if instance_error: + return { + 'success': False, 'files': [], 'all_objects': [], + 'error': json.loads(instance_error['body']).get('error', 'Invalid instanceId'), + } + if prefix_scheme != 'ecs': + return { + 'success': False, 'files': [], 'all_objects': [], + 'error': 'Only the canonical ecs instance namespace is supported.', + } + canonical_prefix = f'ecs_{instance_id}/' + search_result = safe_s3_list(canonical_prefix, max_keys=5000) + if not search_result.get('success'): + return {'success': False, 'files': [], 'all_objects': [], 'error': search_result.get('error', 'S3 list failed')} + + all_objects = [] + bundle_files = [] + bundle_timestamps = {} + for obj in search_result.get('objects', []): + key = obj.get('key', '') + if validate_log_key(key, instance_id): + continue + all_objects.append(obj) + if '/extracted/' in key: + bundle_files.append(key) + if obj.get('last_modified'): + bundle_timestamps[key] = obj['last_modified'] + + if not bundle_files: + return {'success': False, 'files': [], 'all_objects': all_objects, 'error': f'No extracted log bundle found for {instance_id}. Run collect first.'} + + from collections import defaultdict + bundles_by_prefix = defaultdict(list) + for f in bundle_files: + prefix_part = f.split('/extracted/')[0] if '/extracted/' in f else f + bundles_by_prefix[prefix_part].append(f) + + latest_prefix = max( + bundles_by_prefix.keys(), + key=lambda p: max( + (bundle_timestamps.get(f, datetime.min.replace(tzinfo=None)) for f in bundles_by_prefix[p]), + default=datetime.min + ) + ) + latest_files = bundles_by_prefix[latest_prefix] + + bundle_age_minutes = None + bundle_collected_at = None + if bundle_timestamps: + ts_values = [ts for f in latest_files for ts in [bundle_timestamps.get(f)] if ts is not None] + if ts_values: + newest_ts = max(ts_values) + now_utc = datetime.now(timezone.utc) + if newest_ts.tzinfo is None: + newest_ts = newest_ts.replace(tzinfo=timezone.utc) + bundle_age_minutes = int((now_utc - newest_ts).total_seconds() / 60) + bundle_collected_at = newest_ts.isoformat() + + return { + 'success': True, + 'files': latest_files, + 'bundle_prefix': latest_prefix, + 'bundle_age_minutes': bundle_age_minutes, + 'bundle_collected_at': bundle_collected_at, + 'all_objects': all_objects, + } + + +# ============================================================================ +# REGIONAL CLIENT HELPERS +# ============================================================================ + +def get_regional_client(service: str, region: str) -> Any: + if region == DEFAULT_REGION: + if service == 'ssm': return ssm_client + if service == 's3': return s3_client + if service == 'ec2': return ec2_client + if service == 'ecs': return ecs_client + cache_key = f'{service}:{region}' + if cache_key not in _regional_clients: + _regional_clients[cache_key] = boto3.client(service, region_name=region) + return _regional_clients[cache_key] + + +def detect_instance_region(instance_id: str) -> Optional[str]: + """Probe only configured regions; never expand scope with a built-in region list.""" + if validate_instance_id(instance_id): + return None + start = time.time() + ordered_regions = [DEFAULT_REGION] + sorted(ALLOWED_REGIONS - {DEFAULT_REGION}) + for region in ordered_regions: + if time.time() - start > 20: + return None + try: + regional_ec2 = get_regional_client('ec2', region) + response = regional_ec2.describe_instances(InstanceIds=[instance_id]) + if any(reservation.get('Instances') for reservation in response.get('Reservations', [])): + return region + except Exception: + continue + return None + + +def resolve_region(arguments: Dict, instance_id: str = None) -> str: + explicit = arguments.get('region') + if isinstance(explicit, str) and _REGION_RE.fullmatch(explicit): + return explicit + if instance_id: + detected = detect_instance_region(instance_id) + if detected: + return detected + return DEFAULT_REGION + + +# ============================================================================ +# UTILITY HELPERS +# ============================================================================ + +def format_bytes(size: int) -> str: + for unit in ['B', 'KB', 'MB', 'GB']: + if size < 1024: + return f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} TB" + + +def parse_failure_reason(invocation: Dict) -> str: + """Parse failure reason from SSM RunCommand invocation.""" + status = invocation.get('Status', invocation.get('StatusDetails', '')) + stderr = invocation.get('StandardErrorContent', '') + if stderr: + lines = [l.strip() for l in stderr.strip().split('\n') if l.strip()] + if lines: + return f"{status}: {lines[-1][:200]}" + return status + + +def estimate_progress(invocation: Dict) -> Dict: + """Estimate progress from RunCommand invocation.""" + status = invocation.get('Status', '') + status_map = { + 'Pending': {'percent': 5, 'step': 'Queued'}, + 'InProgress': {'percent': 50, 'step': 'Collecting logs'}, + 'Delayed': {'percent': 10, 'step': 'Delayed'}, + 'Success': {'percent': 100, 'step': 'Complete'}, + 'Cancelled': {'percent': 0, 'step': 'Cancelled'}, + 'TimedOut': {'percent': 0, 'step': 'Timed out'}, + 'Failed': {'percent': 0, 'step': 'Failed'}, + } + progress = status_map.get(status, {'percent': 0, 'step': status}) + # Refine InProgress estimate from stdout + if status == 'InProgress': + stdout = invocation.get('StandardOutputContent', '') + if 'Creating Archive' in stdout: + progress = {'percent': 85, 'step': 'Creating archive'} + elif 'Uploading to S3' in stdout: + progress = {'percent': 95, 'step': 'Uploading to S3'} + elif 'Collecting Network' in stdout: + progress = {'percent': 70, 'step': 'Collecting network diagnostics'} + elif 'Collecting ECS Agent' in stdout: + progress = {'percent': 40, 'step': 'Collecting ECS agent info'} + elif 'Collecting Docker' in stdout: + progress = {'percent': 30, 'step': 'Collecting Docker info'} + elif 'Collecting System' in stdout: + progress = {'percent': 20, 'step': 'Collecting system info'} + return progress + + +# ============================================================================ +# PRIVATE EXECUTION PROVENANCE AND IDEMPOTENCY +# ============================================================================ + +_EXECUTION_REGION_PREFIX = '_metadata/execution-regions/' +_EXECUTION_PROVENANCE_PREFIX = '_metadata/executions/' +_BATCH_METADATA_PREFIX = '_metadata/batches/' + + +def store_execution_region(execution_id: str, region: str) -> bool: + """Persist a private compatibility mapping after validating its region.""" + if validate_region(region): + return False + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, + Key=f'{_EXECUTION_REGION_PREFIX}{execution_id}.txt', + Body=region.encode('utf-8'), + ) + return True + except Exception: + return False + + +def get_execution_region(execution_id: str) -> Optional[str]: + data = safe_s3_read_raw(LOGS_BUCKET, f'{_EXECUTION_REGION_PREFIX}{execution_id}.txt') + return data.decode('utf-8').strip() if data else None + + +def store_execution_provenance(execution_id: str, tool: str, region: str, + expected_document: str, + instance_id: Optional[str] = None, + cluster_name: Optional[str] = None, + instance_ids: Optional[List[str]] = None) -> bool: + """Persist deployment-created execution identity under a private prefix.""" + if validate_region(region) or not expected_document or not tool: + return False + if instance_id and validate_instance_id(instance_id): + return False + if cluster_name and validate_cluster_name(cluster_name): + return False + clean_instance_ids = [] + for value in instance_ids or []: + if validate_instance_id(value): + return False + clean_instance_ids.append(value) + metadata = { + 'executionId': execution_id, + 'tool': tool, + 'instanceId': instance_id, + 'instanceIds': clean_instance_ids, + 'clusterName': cluster_name, + 'region': region, + 'expectedDocument': expected_document, + 'createdAt': datetime.now(timezone.utc).isoformat(), + } + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, + Key=f'{_EXECUTION_PROVENANCE_PREFIX}{execution_id}.json', + Body=json.dumps(metadata).encode('utf-8'), + ContentType='application/json', + ) + store_execution_region(execution_id, region) + return True + except Exception: + return False + + +def get_execution_provenance(execution_id: str) -> Optional[Dict]: + raw = safe_s3_read_raw( + LOGS_BUCKET, f'{_EXECUTION_PROVENANCE_PREFIX}{execution_id}.json' + ) + if not raw: + return None + try: + value = json.loads(raw.decode('utf-8')) + return value if isinstance(value, dict) else None + except (UnicodeDecodeError, json.JSONDecodeError): + return None + + +def require_execution_provenance(execution_id: str, + requested_instance_id: Optional[str] = None, + allowed_tools: Optional[set] = None) -> Tuple[Optional[Dict], Optional[Dict]]: + """Load and validate trusted gateway provenance before any SSM detail is exposed.""" + metadata = get_execution_provenance(execution_id) + if not metadata: + return None, error_response(404, 'Unknown executionId; gateway provenance was not found.') + if metadata.get('executionId') != execution_id: + return None, error_response(403, 'Execution provenance identifier mismatch.') + region_error = validate_region(metadata.get('region')) + if region_error: + return None, region_error + if not isinstance(metadata.get('expectedDocument'), str) or not metadata['expectedDocument']: + return None, error_response(403, 'Execution provenance has no expected document.') + if allowed_tools and metadata.get('tool') not in allowed_tools: + return None, error_response(403, 'Execution provenance belongs to a different tool.') + if requested_instance_id: + instance_error = validate_instance_id(requested_instance_id) + if instance_error: + return None, instance_error + expected_instance = metadata.get('instanceId') + expected_instances = metadata.get('instanceIds') or [] + if requested_instance_id != expected_instance and requested_instance_id not in expected_instances: + return None, error_response(403, 'Execution provenance belongs to a different instance.') + return metadata, None + + +def validate_execution_details(execution: Dict, provenance: Dict) -> Optional[Dict]: + """Reject wrong documents and instance parameters before returning SSM details.""" + if execution.get('DocumentName') != provenance.get('expectedDocument'): + return error_response(403, 'Execution document does not match gateway provenance.') + expected_instance = provenance.get('instanceId') + if expected_instance: + parameters = execution.get('Parameters', {}) or {} + actual_values = parameters.get('ECSInstanceId') or parameters.get('InstanceId') or [] + if actual_values != [expected_instance]: + return error_response(403, 'Execution instance parameters do not match gateway provenance.') + expected_instances = provenance.get('instanceIds') or [] + if expected_instances: + actual_values = (execution.get('Parameters', {}) or {}).get('InstanceIds') or [] + if sorted(actual_values) != sorted(expected_instances): + return error_response(403, 'Batch execution instance parameters do not match gateway provenance.') + return None + + +def find_execution_by_idempotency_token(instance_id: str, token: str) -> Optional[Dict]: + """Find existing execution by idempotency token, scoped to instance.""" + key = f'idempotency/{instance_id}/{token}.json' + data = safe_s3_read_raw(LOGS_BUCKET, key) + if data: + try: + return json.loads(data) + except Exception: + pass + return None + + +def store_idempotency_mapping(instance_id: str, token: str, execution_id: str): + """Store idempotency mapping scoped to instance.""" + mapping = { + 'executionId': execution_id, + 'instanceId': instance_id, + 'token': token, + 'status': 'InProgress', + 'createdAt': datetime.utcnow().isoformat(), + } + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, + Key=f'idempotency/{instance_id}/{token}.json', + Body=json.dumps(mapping, default=str).encode(), + ContentType='application/json', + ) + except Exception as e: + print(f"Warning: Failed to store idempotency mapping: {str(e)}") + + +# ============================================================================ +# BASELINE HELPERS +# ============================================================================ + +def load_baselines(instance_id: str) -> Dict: + data = safe_s3_read_raw(LOGS_BUCKET, f'baselines/{instance_id}/baseline.json') + if data: + try: + return json.loads(data) + except Exception: + pass + return {} + + +def update_baselines(instance_id: str, findings: List[Dict]): + baseline = load_baselines(instance_id) + for f in findings: + sig = f'{f.get("description", "")}__{f.get("file", "")}' + key = hashlib.md5(sig.encode()).hexdigest() + if key not in baseline: + baseline[key] = {'firstSeen': datetime.now(timezone.utc).isoformat(), 'count': 0, 'description': f.get('description', '')} + baseline[key]['count'] = baseline[key].get('count', 0) + 1 + baseline[key]['lastSeen'] = datetime.now(timezone.utc).isoformat() + try: + s3_client.put_object(Bucket=LOGS_BUCKET, Key=f'baselines/{instance_id}/baseline.json', Body=json.dumps(baseline, default=str).encode(), ServerSideEncryption='AES256') + except Exception: + pass + + +def annotate_findings_with_baselines(findings: List[Dict], baselines: Dict) -> List[Dict]: + for f in findings: + sig = f'{f.get("description", "")}__{f.get("file", "")}' + key = hashlib.md5(sig.encode()).hexdigest() + if key in baselines: + f['isBaseline'] = True + f['baselineFirstSeen'] = baselines[key].get('firstSeen') + f['baselineCount'] = baselines[key].get('count', 0) + else: + f['isBaseline'] = False + return findings + + +# ============================================================================ +# SCAN HELPERS +# ============================================================================ + +def find_findings_index(prefix: str, s3c=None) -> Optional[str]: + """ + Find the findings index file for a log collection. + Returns the S3 key string of the findings_index.json in the LATEST bundle, or None. + """ + parts = prefix.split('_', 1) + instance_id = parts[1] if len(parts) > 1 else prefix + scheme = parts[0] if len(parts) > 1 else 'ecs' + + bundle_info = find_latest_bundle_files(instance_id, prefix_scheme=scheme) + if not bundle_info['success']: + return None + + index_files = [f for f in bundle_info['files'] if FINDINGS_INDEX_FILE in f] + return index_files[0] if index_files else None + + +def scan_file_for_errors(content: str, filename: str) -> List[Dict]: + findings = [] + lines = content.split('\n') + for line_num, line in enumerate(lines, 1): + if not line.strip(): + continue + # False positive suppression + if any(fp.search(line) for fp in FALSE_POSITIVE_PATTERNS): + continue + for sev_key, compiled_patterns in COMPILED_ERROR_PATTERNS.items(): + for regex, description in compiled_patterns: + if regex.search(line): + findings.append({ + 'severity': sev_key, + 'description': description, + 'file': filename, + 'lineNumber': line_num, + 'line': line[:500], + 'pattern': regex.pattern, + }) + break # One match per line per severity + return findings + + +def scan_and_index_errors(prefix: str, s3c=None) -> Dict: + """Scan all files in a bundle and build findings index.""" + c = s3c or s3_client + files = safe_s3_list_raw(LOGS_BUCKET, prefix, s3c=c) + all_findings = [] + files_scanned = 0 + for obj in files: + key = obj['Key'] + size = obj.get('Size', 0) + if any(key.endswith(ext) for ext in ['.tar.gz', '.zip', '.gz', '.tar', '.bin', '.so', '.png', '.jpg']): + continue + if key.endswith(FINDINGS_INDEX_FILE) or key.endswith('manifest.json'): + continue + if size > MAX_CHUNK_SIZE or size == 0: + continue + data = safe_s3_read_raw(LOGS_BUCKET, key, s3c=c) + if not data: + continue + try: + content = data.decode('utf-8', errors='ignore') + except Exception: + continue + files_scanned += 1 + filename = key[len(prefix):] if key.startswith(prefix) else key.split('/')[-1] + file_findings = scan_file_for_errors(content, filename) + all_findings.extend(file_findings) + # Assign finding IDs and sort by severity + all_findings.sort(key=lambda f: SEVERITY_ORDER.get(f.get('severity', 'info'), 4)) + for i, f in enumerate(all_findings): + f['finding_id'] = assign_finding_id(i + 1) + index = { + 'generatedAt': datetime.now(timezone.utc).isoformat(), + 'filesScanned': files_scanned, + 'totalFindings': len(all_findings), + 'findings': all_findings, + 'summary': {}, + } + for sev in ['critical', 'warning', 'info']: + index['summary'][sev] = len([f for f in all_findings if f['severity'] == sev]) + # Store index + try: + c.put_object(Bucket=LOGS_BUCKET, Key=f'{prefix}{FINDINGS_INDEX_FILE}', Body=json.dumps(index, default=str).encode(), ServerSideEncryption='AES256') + except Exception: + pass + return index + + +# ============================================================================ +# READ / SEARCH HELPERS +# ============================================================================ + +def read_by_lines(bucket: str, key: str, start_line: int = 1, max_lines: int = DEFAULT_LINE_COUNT, s3c=None) -> Dict: + head = safe_s3_head_raw(bucket, key, s3c=s3c) + if not head: + return {'error': f'File not found: {key}'} + total_size = head['ContentLength'] + data = safe_s3_read_raw(bucket, key, s3c=s3c) + if not data: + return {'error': f'Cannot read: {key}'} + try: + content = data.decode('utf-8', errors='ignore') + except Exception: + content = data.decode('latin-1', errors='ignore') + lines = content.split('\n') + total_lines = len(lines) + end_line = min(start_line + max_lines - 1, total_lines) + selected = lines[start_line - 1:end_line] + return { + 'lines': selected, 'startLine': start_line, 'endLine': end_line, + 'totalLines': total_lines, 'totalSize': total_size, 'hasMore': end_line < total_lines, + } + + +def search_file_for_pattern(bucket: str, key: str, pattern: re.Pattern, + max_matches: int = 50, s3c=None) -> List[Dict]: + """Search one file using a precompiled regex under a hard wall-clock limit.""" + if not hasattr(pattern, 'search'): + raise TypeError('pattern must be a compiled regular expression') + head = safe_s3_head_raw(bucket, key, s3c=s3c) + if not head: + return [] + if head['ContentLength'] > 10 * 1024 * 1024: + return [] + data = safe_s3_read_raw(bucket, key, s3c=s3c) + if not data: + return [] + try: + content = data.decode('utf-8', errors='ignore') + except Exception: + return [] + matches = [] + filename = key.split('/')[-1] + with regex_time_limit(): + for line_number, line in enumerate(content.split('\n'), 1): + if pattern.search(line): + matches.append({ + 'file': filename, 'fullKey': key, 'lineNumber': line_number, + 'line': line[:500], 'pattern': pattern.pattern, + }) + if len(matches) >= max_matches: + break + return matches + + +def get_line_context(content: str, line_num: int, context: int = 3) -> List[str]: + lines = content.split('\n') + start = max(0, line_num - context - 1) + end = min(len(lines), line_num + context) + return lines[start:end] + + +def extract_timestamp(line: str) -> Optional[str]: + patterns = [ + r'(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2})', + r'(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})', + r'(\[\d+\.\d+\])', + ] + for p in patterns: + m = re.search(p, line) + if m: + return m.group(1) + return None + + +def categorize_log_source(filename: str) -> str: + """Categorize a log file into ECS-specific component.""" + fl = filename.lower() + for category, patterns in ECS_LOG_TYPE_PATTERNS.items(): + if any(p in fl for p in patterns): + return category + if 'kernel' in fl or 'dmesg' in fl: + return 'kernel' + return 'system' + + +def find_correlations(findings: List[Dict]) -> List[Dict]: + """Find ECS-specific cross-component correlations.""" + correlations = [] + # Group findings by component + by_component = {} + for f in findings: + comp = categorize_log_source(f.get('file', '')) + by_component.setdefault(comp, []).append(f) + # Kernel <-> ECS Agent correlation + kernel_findings = by_component.get('kernel', []) + by_component.get('dmesg', []) + ecs_findings = by_component.get('ecs-agent', []) + if kernel_findings and ecs_findings: + correlations.append({ + 'type': 'kernel-ecs-agent', + 'description': 'Kernel issues detected alongside ECS agent problems - kernel instability may cause agent disconnection', + 'components': ['kernel', 'ecs-agent'], + 'findingIds': [f.get('finding_id') for f in (kernel_findings[:3] + ecs_findings[:3]) if f.get('finding_id')], + 'confidence': 'high', + }) + # Network <-> Docker correlation + net_findings = by_component.get('networking', []) + docker_findings = by_component.get('docker', []) + if net_findings and docker_findings: + correlations.append({ + 'type': 'network-docker', + 'description': 'Network issues detected alongside Docker problems - network connectivity may affect container operations', + 'components': ['networking', 'docker'], + 'findingIds': [f.get('finding_id') for f in (net_findings[:3] + docker_findings[:3]) if f.get('finding_id')], + 'confidence': 'medium', + }) + # OOM <-> Task failures + oom_findings = [f for f in findings if any(k in f.get('description', '').lower() for k in ['oom', 'memory', 'exit code 137'])] + task_failures = [f for f in findings if any(k in f.get('description', '').lower() for k in ['task', 'container', 'stopped'])] + if oom_findings and task_failures: + correlations.append({ + 'type': 'oom-task-failure', + 'description': 'OOM kills detected alongside task failures - memory exhaustion likely causing task stops', + 'components': ['cgroups', 'ecs-agent'], + 'findingIds': [f.get('finding_id') for f in (oom_findings[:3] + task_failures[:3]) if f.get('finding_id')], + 'confidence': 'high', + }) + return correlations + + +def generate_recommendations(findings: List[Dict]) -> List[Dict]: + """Generate ECS-specific recommendations based on findings.""" + recs = [] + descs = ' '.join(f.get('description', '') for f in findings).lower() + if 'oom' in descs or 'memory' in descs or 'exit code 137' in descs: + recs.append({'priority': 'high', 'category': 'Resource', 'action': 'Increase task memory limits or investigate memory leaks', 'docs': 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/out-of-memory.html'}) + if 'pull' in descs or 'ecr' in descs or 'image' in descs: + recs.append({'priority': 'high', 'category': 'Image', 'action': 'Verify ECR permissions, image existence, and VPC endpoint connectivity', 'docs': 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_cannot_pull_image.html'}) + if 'agent' in descs and ('disconnect' in descs or 'not connected' in descs): + recs.append({'priority': 'high', 'category': 'Agent', 'action': 'Check ECS agent logs, verify IAM instance profile, and network connectivity to ECS endpoints'}) + if 'secret' in descs or 'parameter' in descs: + recs.append({'priority': 'high', 'category': 'Secrets', 'action': 'Verify task execution role has secretsmanager:GetSecretValue and ssm:GetParameters permissions', 'docs': 'https://repost.aws/knowledge-center/ecs-unable-to-pull-secrets'}) + if 'network' in descs or 'eni' in descs or 'subnet' in descs: + recs.append({'priority': 'high', 'category': 'Network', 'action': 'Check subnet IP availability, security group rules, and VPC endpoint configuration'}) + if 'circuit breaker' in descs or 'deployment' in descs: + recs.append({'priority': 'medium', 'category': 'Deployment', 'action': 'Review deployment configuration, health check settings, and task definition', 'docs': 'https://repost.aws/knowledge-center/ecs-troubleshoot-deployment-failures'}) + if 'health check' in descs or 'unhealthy' in descs: + recs.append({'priority': 'medium', 'category': 'Health', 'action': 'Review health check configuration, grace period, and container startup time'}) + if 'disk' in descs or 'space' in descs or 'inode' in descs: + recs.append({'priority': 'medium', 'category': 'Storage', 'action': 'Clean up unused images/containers, increase EBS volume size'}) + return recs + + +# ============================================================================ +# TRIAGE +# ============================================================================ + +def perform_ecs_triage(findings: List[Dict]) -> Dict: + """Perform ECS-specific triage using ECS_TRIAGE_CATEGORIES.""" + matched_categories = {} + for cat_id, cat_info in ECS_TRIAGE_CATEGORIES.items(): + cat_findings = [] + for f in findings: + line = f.get('line', '') + ' ' + f.get('description', '') + for pattern, priority in cat_info['patterns']: + if re.search(pattern, line, re.IGNORECASE): + cat_findings.append({**f, 'triagePriority': priority}) + break + if cat_findings: + matched_categories[cat_id] = { + 'name': cat_info['name'], + 'description': cat_info['description'], + 'logSources': cat_info['log_sources'], + 'findingCount': len(cat_findings), + 'findings': cat_findings[:10], + 'docs': cat_info.get('docs'), + 'runbook': cat_info.get('runbook'), + } + # Detect ECS task states + task_states = {} + for f in findings: + line = f.get('line', '') + for state in ['PENDING', 'PROVISIONING', 'RUNNING', 'STOPPED', 'DEPROVISIONING']: + if state in line: + task_states[state] = task_states.get(state, 0) + 1 + # Detect instance conditions + instance_conditions = {} + for f in findings: + line = f.get('line', '') + for cond in ['ACTIVE', 'DRAINING', 'DISCONNECTED', 'AGENT_DISCONNECTED']: + if cond in line: + instance_conditions[cond] = instance_conditions.get(cond, 0) + 1 + return { + 'categories': matched_categories, + 'taskStates': task_states, + 'instanceConditions': instance_conditions, + 'topCategory': max(matched_categories.keys(), key=lambda k: matched_categories[k]['findingCount']) if matched_categories else None, + } + + +# ============================================================================ +# TEMPORAL CORRELATION +# ============================================================================ + +def _build_temporal_clusters(findings: List[Dict], window_seconds: int = 60) -> List[Dict]: + """Group findings into temporal clusters.""" + timestamped = [] + for f in findings: + ts = extract_timestamp(f.get('line', '')) + if ts: + timestamped.append({**f, '_ts': ts}) + if not timestamped: + return [] + clusters = [] + current_cluster = [timestamped[0]] + for f in timestamped[1:]: + current_cluster.append(f) + if len(current_cluster) >= 20: + clusters.append({'events': current_cluster, 'count': len(current_cluster)}) + current_cluster = [] + if current_cluster: + clusters.append({'events': current_cluster, 'count': len(current_cluster)}) + return clusters + + +def _build_root_cause_chain(findings: List[Dict]) -> List[Dict]: + """Build ECS-specific causal chains.""" + chains = [] + # ECS causal patterns: kernel issue -> agent disconnect -> task failures + kernel_issues = [f for f in findings if categorize_log_source(f.get('file', '')) in ('kernel', 'dmesg') and f.get('severity') == 'critical'] + agent_issues = [f for f in findings if 'agent' in f.get('description', '').lower() or 'disconnect' in f.get('description', '').lower()] + task_issues = [f for f in findings if any(k in f.get('description', '').lower() for k in ['task', 'container', 'stopped', 'failed'])] + if kernel_issues and agent_issues: + chains.append({ + 'type': 'kernel-cascade', + 'description': 'Kernel instability → ECS Agent disconnect → Task failures', + 'rootCause': kernel_issues[0].get('finding_id'), + 'effects': [f.get('finding_id') for f in agent_issues[:3] + task_issues[:3] if f.get('finding_id')], + 'confidence': 'high', + }) + # OOM cascade: memory pressure -> OOM kill -> container exit -> task stop + oom_issues = [f for f in findings if any(k in f.get('description', '').lower() for k in ['oom', 'memory cgroup', 'invoked oom-killer'])] + exit_137 = [f for f in findings if 'exit code 137' in f.get('description', '').lower()] + if oom_issues: + chains.append({ + 'type': 'oom-cascade', + 'description': 'Memory pressure → OOM kill → Container exit (137) → Task stop', + 'rootCause': oom_issues[0].get('finding_id'), + 'effects': [f.get('finding_id') for f in exit_137[:3] + task_issues[:3] if f.get('finding_id')], + 'confidence': 'high', + }) + # Network cascade: ENI/subnet issue -> connection failures -> health check failures + net_root = [f for f in findings if any(k in f.get('description', '').lower() for k in ['eni', 'subnet', 'network unreachable'])] + conn_issues = [f for f in findings if any(k in f.get('description', '').lower() for k in ['connection refused', 'timeout', 'dns'])] + health_issues = [f for f in findings if 'health' in f.get('description', '').lower()] + if net_root and (conn_issues or health_issues): + chains.append({ + 'type': 'network-cascade', + 'description': 'Network/ENI issue → Connection failures → Health check failures', + 'rootCause': net_root[0].get('finding_id'), + 'effects': [f.get('finding_id') for f in conn_issues[:3] + health_issues[:3] if f.get('finding_id')], + 'confidence': 'medium', + }) + return chains + +# ============================================================================= +# LAMBDA HANDLER + TOOL ROUTING +# ============================================================================= + +def list_sops(arguments: Dict) -> Dict: + """List all SOPs in the SOP S3 bucket.""" + sop_bucket = os.environ.get('SOP_BUCKET_NAME', '') + if not sop_bucket: + return error_response(400, 'SOP_BUCKET_NAME not configured') + try: + s3 = boto3.client('s3') + response = s3.list_objects_v2(Bucket=sop_bucket) + if 'Contents' not in response: + return success_response({'sops': [], 'count': 0, 'bucket': sop_bucket}) + sops = [{'name': obj['Key'], 'size': obj['Size'], 'lastModified': obj['LastModified'].isoformat()} for obj in response['Contents']] + return success_response({'sops': sops, 'count': len(sops), 'bucket': sop_bucket}) + except Exception as e: + return error_response(500, f'Failed to list SOPs: {str(e)}') + + +def get_sop(arguments: Dict) -> Dict: + """Get a specific SOP by name from the SOP S3 bucket.""" + sop_bucket = os.environ.get('SOP_BUCKET_NAME', '') + if not sop_bucket: + return error_response(400, 'SOP_BUCKET_NAME not configured') + sop_name = arguments.get('sopName') + if not sop_name: + return error_response(400, 'sopName is required') + try: + s3 = boto3.client('s3') + response = s3.get_object(Bucket=sop_bucket, Key=sop_name) + content = response['Body'].read().decode('utf-8') + return success_response({ + 'sop': {'name': sop_name, 'content': content, 'size': response['ContentLength'], + 'lastModified': response['LastModified'].isoformat(), + 'contentType': response.get('ContentType', 'text/plain')} + }) + except ClientError as e: + if e.response['Error']['Code'] == 'NoSuchKey': + return error_response(404, f'SOP "{sop_name}" not found. Use list_sops to see available SOPs.') + return error_response(500, f'Failed to get SOP: {str(e)}') + except Exception as e: + return error_response(500, f'Failed to get SOP: {str(e)}') + + +def lambda_handler(event, context): + """Main Lambda handler - routes to appropriate tool function.""" + print(f"Received event: {json.dumps(event)}") + + delimiter = "___" + original_tool_name = context.client_context.custom.get('bedrockAgentCoreToolName', '') + + if delimiter in original_tool_name: + tool_name = original_tool_name[original_tool_name.index(delimiter) + len(delimiter):] + else: + tool_name = original_tool_name + + print(f"Executing tool: {tool_name}") + + tools = { + # Tier 1: Core Operations + 'collect': start_log_collection, + 'status': get_collection_status, + 'validate': validate_bundle_completeness, + 'errors': get_error_summary, + 'read': read_log_chunk, + # Tier 2: Advanced Analysis + 'search': search_logs_deep, + 'correlate': correlate_events, + 'artifact': get_artifact_reference, + 'summarize': generate_incident_summary, + 'history': list_collection_history, + # Tier 3: Cluster-Level Intelligence + 'cluster_health': cluster_health_check, + 'compare_instances': compare_instances, + 'batch_collect': batch_collect, + 'batch_status': batch_status, + 'network_diagnostics': network_diagnostics, + 'tcpdump_capture': tcpdump_capture, + 'tcpdump_analyze': tcpdump_analyze, + 'list_sops': list_sops, + 'get_sop': get_sop, + } + + # Restricted tools are absent from the runtime routing surface until an + # operator explicitly opts in. Authorization is checked again to prevent + # direct invocation from bypassing the visibility gate. + for restricted in RESTRICTED_TOOLS - ENABLED_RESTRICTED_TOOLS: + tools.pop(restricted, None) + + authorization_error = validate_tool_authorization(tool_name) + if authorization_error is not None: + return authorization_error + + if tool_name not in tools: + return error_response(400, f'Unknown tool: {tool_name}', { + 'available_tools': list(tools.keys()) + }) + + try: + return tools[tool_name](event) + except Exception as e: + print(f"Error executing {tool_name}: {str(e)}") + import traceback + traceback.print_exc() + return error_response(500, f'Internal error: {str(e)}') + + +# ============================================================================= +# TIER 1: CORE OPERATIONS +# ============================================================================= + +def start_log_collection(arguments: Dict) -> Dict: + """ + Start ECS log collection via AWSSupport-CollectECSInstanceLogs SSM Automation. + + Inputs: + instanceId: EC2 instance ID (required) + idempotencyToken: Optional token to prevent duplicate executions + region: AWS region where the instance runs (optional, auto-detected) + + Returns: + executionId, estimatedCompletionTime, status, region + """ + instance_id = arguments.get('instanceId') + idempotency_token = arguments.get('idempotencyToken') + + if not instance_id: + return error_response(400, 'instanceId is required') + + if not re.match(r'^i-[0-9a-f]{8,17}$', instance_id): + return error_response(400, f'Invalid instanceId format: {instance_id}. Expected: i-xxxxxxxxxxxxxxxxx') + + # Resolve and validate region + target_region, region_error = resolve_and_validate_region(arguments, instance_id) + if region_error: + return region_error + + # Validate instance belongs to an ECS cluster + instance_error = validate_ecs_instance(instance_id, target_region) + if instance_error: + return instance_error + + try: + regional_ssm = get_regional_client('ssm', target_region) + except Exception as e: + return error_response(500, f'Failed to create SSM client for region {target_region}: {str(e)}') + + print(f"Starting ECS log collection for {instance_id} in region {target_region}") + + # Verify instance state + try: + regional_ec2 = get_regional_client('ec2', target_region) + desc_resp = regional_ec2.describe_instances(InstanceIds=[instance_id]) + reservations = desc_resp.get('Reservations', []) + if reservations and reservations[0].get('Instances'): + state = reservations[0]['Instances'][0].get('State', {}).get('Name', 'unknown') + if state in ('terminated', 'shutting-down'): + return error_response(400, f'Instance {instance_id} is {state}. Cannot collect logs from terminated instances.') + if state == 'stopped': + return error_response(400, f'Instance {instance_id} is stopped. Start the instance first, then retry.') + except Exception as e: + print(f"Warning: Could not verify instance state: {str(e)}") + + # Idempotency check + if idempotency_token: + existing = find_execution_by_idempotency_token(instance_id, idempotency_token) + if existing and get_execution_provenance(existing.get('executionId', '')): + return success_response({ + 'message': 'Returning existing execution (idempotent)', + 'executionId': existing['executionId'], + 'status': existing['status'], + 'instanceId': instance_id, + 'region': target_region, + 'idempotent': True, + }) + + try: + params = { + 'ECSInstanceId': [instance_id], + 'LogDestination': [LOGS_BUCKET], + 'AutomationAssumeRole': [SSM_AUTOMATION_ROLE_ARN], + } + + response = regional_ssm.start_automation_execution( + DocumentName='AWSSupport-CollectECSInstanceLogs', + Parameters=params, + ) + + execution_id = response['AutomationExecutionId'] + + if not store_execution_provenance( + execution_id, + tool='collect', + region=target_region, + expected_document='AWSSupport-CollectECSInstanceLogs', + instance_id=instance_id, + ): + return error_response(500, 'Collection started but secure execution provenance could not be persisted.') + + if idempotency_token: + store_idempotency_mapping(instance_id, idempotency_token, execution_id) + + region_stored = True + + response_data = { + 'message': 'ECS log collection started', + 'executionId': execution_id, + 'instanceId': instance_id, + 'region': target_region, + 's3Bucket': LOGS_BUCKET, + 'estimatedCompletionTime': '3-5 minutes', + 'suggestedPollIntervalSeconds': 15, + 'nextStep': f'Poll status with status(executionId="{execution_id}") every 15 seconds', + 'task': { + 'taskId': execution_id, + 'state': 'running', + 'message': 'Log collection started via SSM Automation', + 'progress': 0, + }, + } + + if not region_stored and target_region != DEFAULT_REGION: + response_data['warning'] = ( + f'Region mapping could not be persisted. Pass region="{target_region}" ' + f'explicitly in subsequent status/validate calls.' + ) + + return success_response(response_data) + + except regional_ssm.exceptions.AutomationDefinitionNotFoundException: + return error_response(404, 'AWSSupport-CollectECSInstanceLogs document not found', { + 'suggestion': f'This SSM document may not be available in region {target_region}. ' + f'Try us-east-1 or us-west-2.', + 'region': target_region, + }) + except Exception as e: + return error_response(500, f'Failed to start log collection in {target_region}: {str(e)}') + + +def get_collection_status(arguments: Dict) -> Dict: + """ + Get detailed status of ECS log collection with progress tracking. + + Inputs: + executionId: SSM Automation execution ID (required) + includeStepDetails: Include individual step status (default: true) + + Returns: + status, progress, stepDetails, failureReason (if failed) + """ + execution_id = arguments.get('executionId') + include_steps = arguments.get('includeStepDetails', True) + + if not execution_id: + return error_response(400, 'executionId is required') + + stored_region = get_execution_region(execution_id) + target_region = stored_region or arguments.get('region', DEFAULT_REGION) + region_error = validate_region(target_region) + if region_error: + return region_error + try: + regional_ssm = get_regional_client('ssm', target_region) + except Exception as e: + return error_response(500, f'Failed to create SSM client for region {target_region}: {str(e)}') + + try: + response = regional_ssm.get_automation_execution(AutomationExecutionId=execution_id) + execution = response['AutomationExecution'] + status = execution['AutomationExecutionStatus'] + + result = { + 'executionId': execution_id, + 'status': status, + 'documentName': execution.get('DocumentName', ''), + 'startTime': execution.get('ExecutionStartTime'), + 'endTime': execution.get('ExecutionEndTime'), + } + + if status == 'Success': + result['progress'] = 100 + elif status == 'Failed': + result['progress'] = 0 + result['failureReason'] = parse_failure_reason(execution) + elif status == 'InProgress': + result['progress'] = estimate_progress(execution) + else: + result['progress'] = 0 + + if include_steps and 'StepExecutions' in execution: + result['stepDetails'] = [ + { + 'stepName': step.get('StepName'), + 'status': step.get('StepStatus'), + 'startTime': step.get('ExecutionStartTime'), + 'endTime': step.get('ExecutionEndTime'), + } + for step in execution.get('StepExecutions', []) + ] + + if 'Outputs' in execution: + result['outputs'] = execution['Outputs'] + + if status == 'Success': + result['nextStep'] = f'Validate bundle with validate(executionId="{execution_id}")' + elif status == 'InProgress': + result['suggestedPollIntervalSeconds'] = 15 + result['nextStep'] = 'Wait 15 seconds then poll again until status is Success or Failed' + elif status == 'Failed': + result['nextStep'] = 'Review failureReason and retry if appropriate' + + SSM_TO_TASK_STATE = { + 'Pending': 'running', 'InProgress': 'running', 'Waiting': 'running', + 'Success': 'completed', 'TimedOut': 'failed', 'Cancelling': 'cancelling', + 'Cancelled': 'cancelled', 'Failed': 'failed', + } + result['task'] = { + 'taskId': execution_id, + 'state': SSM_TO_TASK_STATE.get(status, 'running'), + 'message': result.get('failureReason', f'SSM status: {status}'), + 'progress': result.get('progress', 0), + } + + return success_response({'automation': result}) + + except regional_ssm.exceptions.AutomationExecutionNotFoundException: + return error_response(404, f'Execution {execution_id} not found') + except Exception as e: + return error_response(500, f'Failed to get status: {str(e)}') + + +def validate_bundle_completeness(arguments: Dict) -> Dict: + """ + Verify all expected files were extracted from ECS log bundle. + + Inputs: + executionId: SSM execution ID OR + instanceId: Instance ID to locate bundle + + Returns: + complete, fileCount, totalSize, missingPatterns, manifest + """ + execution_id = arguments.get('executionId') + instance_id = arguments.get('instanceId') + + if not execution_id and not instance_id: + return error_response(400, 'Either executionId or instanceId is required') + + try: + if instance_id: + prefix = f'ecs_{instance_id}' + else: + try: + stored_region = get_execution_region(execution_id) + target_region = stored_region or arguments.get('region', DEFAULT_REGION) + region_error = validate_region(target_region) + if region_error: + return region_error + regional_ssm = get_regional_client('ssm', target_region) + except Exception as e: + return error_response(500, f'Failed to create SSM client for region: {str(e)}') + try: + exec_response = regional_ssm.get_automation_execution(AutomationExecutionId=execution_id) + params = exec_response['AutomationExecution'].get('Parameters', {}) + instance_id = params.get('ECSInstanceId', [''])[0] + prefix = f'ecs_{instance_id}' + except regional_ssm.exceptions.AutomationExecutionNotFoundException: + return error_response(404, f'Execution {execution_id} not found') + except Exception as e: + return error_response(500, f'Failed to get execution details: {str(e)}') + + # Use shared latest-bundle discovery + bundle_info = find_latest_bundle_files(instance_id) + + if not bundle_info['success']: + return success_response({ + 'complete': False, 'fileCount': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'missingPatterns': ['all'], 'foundPatterns': [], 'hasFindingsIndex': False, + 'instanceId': instance_id, 'manifest': [], + 'warning': bundle_info.get('error', 'Failed to list files'), + 'nextStep': 'Check if log collection completed successfully', + }) + + size_map = {obj['key']: obj for obj in bundle_info['all_objects']} + all_files = [size_map[k] for k in bundle_info['files'] if k in size_map] + + # Check for manifest.json in latest bundle + manifest_data = None + manifest_files = [obj for obj in bundle_info['all_objects'] + if obj['key'].endswith('manifest.json') and obj['key'].startswith(bundle_info['bundle_prefix'])] + if manifest_files: + manifest_files.sort(key=lambda x: x.get('last_modified', ''), reverse=True) + manifest_read = safe_s3_read(manifest_files[0]['key']) + if manifest_read['success']: + try: + manifest_data = json.loads(manifest_read['content']) + except json.JSONDecodeError: + manifest_data = None + + if not all_files: + return success_response({ + 'complete': False, 'fileCount': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'missingPatterns': ['all - no extracted logs found'], 'foundPatterns': [], + 'hasFindingsIndex': False, 'instanceId': instance_id, 'manifest': [], + 'info': 'No extracted log files found. Log collection may still be in progress.', + 'nextStep': 'Check log collection status with status', + }) + + total_size = sum(f['size'] for f in all_files) + + # ECS-specific expected patterns (aligned with amazon-ecs-logs-collector.sh output) + expected_patterns = [ + 'ecs', 'docker', 'messages', 'dmesg', 'networking', 'containers', + 'metadata', 'iptables', 'mounts', 'services', 'os-release', 'uname', + 'pkglist', 'ps.txt', 'top.txt', 'open-file', 'cgroup', + ] + found_patterns = set() + for f in all_files: + key_lower = f['key'].lower() + for pattern in expected_patterns: + if pattern in key_lower: + found_patterns.add(pattern) + + missing_patterns = list(set(expected_patterns) - found_patterns) + has_findings_index = any(FINDINGS_INDEX_FILE in f['key'] for f in all_files) + is_complete = len(all_files) >= 5 and len(found_patterns) >= 3 + + result = { + 'complete': is_complete, + 'fileCount': len(all_files), + 'totalSize': total_size, + 'totalSizeHuman': format_bytes(total_size), + 'missingPatterns': missing_patterns, + 'foundPatterns': list(found_patterns), + 'hasFindingsIndex': has_findings_index, + 'instanceId': instance_id, + } + + if manifest_data and manifest_data.get('version', 1) >= 2: + result['manifestVersion'] = manifest_data.get('version') + result['archiveSize'] = manifest_data.get('archiveSize', 0) + result['archiveSizeHuman'] = format_bytes(manifest_data.get('archiveSize', 0)) + manifest_file_count = manifest_data.get('totalFiles', 0) + if manifest_file_count > 0 and len(all_files) < manifest_file_count: + result['warning'] = ( + f'Manifest reports {manifest_file_count} files but only {len(all_files)} found in S3.' + ) + result['complete'] = False + + if missing_patterns: + result['info'] = f'Some log types not found: {", ".join(missing_patterns)}. May be normal depending on instance config.' + + result['manifest'] = [ + { + 'key': f['key'].split('/extracted/')[-1] if '/extracted/' in f['key'] else f['key'], + 'fullKey': f['key'], + 'size': f['size'], + 'sizeHuman': format_bytes(f['size']), + } + for f in sorted(all_files, key=lambda x: x['size'], reverse=True)[:50] + ] + + if is_complete: + result['nextStep'] = f'Get error summary with errors(instanceId="{instance_id}")' + else: + result['nextStep'] = 'Bundle may be incomplete. Check SSM Automation status or proceed with available logs.' + + return success_response(result) + + except Exception as e: + return success_response({ + 'complete': False, 'fileCount': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'missingPatterns': ['unknown'], 'foundPatterns': [], 'hasFindingsIndex': False, + 'instanceId': instance_id or 'unknown', 'manifest': [], + 'error': f'Unexpected error during validation: {str(e)}', + 'nextStep': 'Retry or check AWS console for log collection status', + }) + + +def get_error_summary(arguments: Dict) -> Dict: + """ + Get pre-indexed error findings with pagination and baseline support. + + Inputs: + instanceId: EC2 instance ID (required) + severity: Filter (critical|high|medium|low|info|all) + response_format: 'concise' (default) or 'detailed' + pageSize: Findings per page (default: 50, max: 200) + pageToken: Opaque token for next page + + Returns: + findings[], summary counts, coverage_report + """ + instance_id = arguments.get('instanceId') + severity_filter = arguments.get('severity', 'all') + response_format = arguments.get('response_format', 'concise') + page_size = min(arguments.get('pageSize', 50), 200) + page_token = arguments.get('pageToken') + cluster_context = arguments.get('clusterContext') + + if not instance_id: + return error_response(400, 'instanceId is required') + instance_error = validate_instance_id(instance_id) + if instance_error: + return instance_error + if cluster_context: + cluster_error = validate_cluster_name(cluster_context) + if cluster_error: + return cluster_error + + page_offset = 0 + if page_token: + try: + import base64 + page_offset = int(base64.b64decode(page_token).decode('utf-8')) + except Exception: + page_offset = 0 + + try: + prefix = f'ecs_{instance_id}' + index_key = find_findings_index(prefix) + + if index_key: + read_result = safe_s3_read(index_key) + if read_result['success']: + try: + index_data = json.loads(read_result['content']) + findings = index_data.get('findings', []) + + for idx, f in enumerate(findings): + if 'finding_id' not in f: + f['finding_id'] = assign_finding_id(idx + 1) + + if cluster_context: + findings = annotate_findings_with_baselines( + findings, load_baselines(cluster_context) + ) + + allowed_severities = normalize_severity_filter(severity_filter) + if severity_filter != 'all': + findings = [f for f in findings if f.get('severity') in allowed_severities] + + total_findings = len(findings) + page_findings = findings[page_offset:page_offset + page_size] + has_more = (page_offset + page_size) < total_findings + + next_token = None + if has_more: + import base64 + next_token = base64.b64encode(str(page_offset + page_size).encode('utf-8')).decode('utf-8') + + summary = index_data.get('summary', {}) + + if response_format == 'concise': + page_findings = [ + { + 'finding_id': f.get('finding_id'), + 'severity': f.get('severity'), + 'pattern': f.get('pattern'), + 'file': f.get('file'), + 'count': f.get('count'), + **( + {'is_baseline': f.get('is_baseline', False), 'baseline_note': f.get('baseline_note')} + if f.get('is_baseline') else {} + ), + } + for f in page_findings + ] + + coverage_report = { + 'files_scanned': index_data.get('filesScanned', 0), + 'files_skipped': index_data.get('filesSkipped', 0), + 'scan_complete': True, + 'index_version': index_data.get('index_version', 'v1'), + } + + if cluster_context: + update_baselines(cluster_context, findings) + + return success_response({ + 'instanceId': instance_id, + 'indexedAt': index_data.get('indexedAt'), + 'findings': page_findings, + 'totalFindings': total_findings, + 'pageSize': page_size, + 'pageOffset': page_offset, + 'hasMore': has_more, + 'nextPageToken': next_token, + 'summary': summary, + 'cached': True, + 'coverage_report': coverage_report, + 'interpretationGuide': { + 'CannotPullContainerError': 'ECR auth or image not found. Check IAM role and image URI.', + 'OOMKilled': 'Container exceeded memory limit. Check task definition memory settings.', + 'STOPPED (Essential container exited)': 'Essential container crashed. Check container logs.', + 'AGENT_DISCONNECTED': 'ECS agent lost connection. Check instance networking and agent logs.', + 'ResourceNotFoundException': 'ECS resource not found. Service/task may have been deleted.', + }, + 'nextStep': 'Use search for detailed investigation, or summarize with finding_ids', + 'recommendedSOPs': match_sops_for_issues([], findings=page_findings), + }) + except json.JSONDecodeError: + print("Warning: Findings index corrupted, will scan on-demand") + + # Slow path: scan and index on-demand + result = scan_and_index_errors(instance_id, severity_filter) + return result + + except Exception as e: + return success_response({ + 'instanceId': instance_id, + 'findings': [], + 'totalFindings': 0, + 'summary': {'critical': 0, 'high': 0, 'medium': 0, 'low': 0, 'info': 0}, + 'cached': False, + 'warning': f'Could not retrieve error summary: {str(e)}', + 'nextStep': 'Check if logs exist with validate', + }) + + +def read_log_chunk(arguments: Dict) -> Dict: + """ + Byte-range streaming for large log files. NO TRUNCATION. Line-aligned. + + Inputs: + logKey: S3 key of log file (required) + startByte: Starting byte offset (default: 0) + endByte: Ending byte offset (optional, defaults to startByte + 1MB) + startLine: Starting line number (alternative to byte range) + lineCount: Number of lines to return (default: 1000) + + Returns: + content, startByte, endByte, totalSize, hasMore, nextChunkToken + """ + log_key = arguments.get('logKey') + instance_id = arguments.get('instanceId') + start_byte = arguments.get('startByte', 0) + end_byte = arguments.get('endByte') + start_line = arguments.get('startLine') + line_count = arguments.get('lineCount', DEFAULT_LINE_COUNT) + + if not instance_id: + return error_response(400, 'instanceId is required') + key_error = validate_log_key(log_key, instance_id) + if key_error: + return key_error + + try: + head_result = safe_s3_head(log_key) + if not head_result['success']: + return success_response({ + 'logKey': log_key, 'content': '', 'startByte': 0, 'endByte': 0, + 'chunkSize': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'hasMore': False, 'nextChunkToken': None, 'truncated': False, + 'fileNotFound': True, + 'warning': head_result.get('error', 'File not found'), + 'suggestion': 'Try listing available logs first.', + }) + + total_size = head_result['size'] + + if total_size > MAX_CHUNK_SIZE * 10: + return get_artifact_reference({ + 'logKey': log_key, + 'instanceId': instance_id, + 'reason': 'File too large for direct read', + }) + + # Line-based reading + if start_line is not None: + return read_by_lines(LOGS_BUCKET, log_key, start_line, min(line_count, MAX_LINE_COUNT)) + + # Byte-range reading + if end_byte is None: + end_byte = min(start_byte + DEFAULT_CHUNK_SIZE, total_size) + + start_byte = max(0, start_byte) + end_byte = min(end_byte, total_size) + chunk_size = end_byte - start_byte + + if chunk_size > MAX_CHUNK_SIZE: + end_byte = start_byte + MAX_CHUNK_SIZE + chunk_size = MAX_CHUNK_SIZE + + if total_size == 0 or chunk_size <= 0: + return success_response({ + 'logKey': log_key, 'content': '', 'startByte': 0, 'endByte': 0, + 'chunkSize': 0, 'totalSize': total_size, 'totalSizeHuman': format_bytes(total_size), + 'hasMore': False, 'nextChunkToken': None, 'truncated': False, + 'info': 'File is empty or requested range is invalid', + }) + + # Line-aligned byte-range reads + BOUNDARY_SCAN = 4096 + actual_start = max(0, start_byte - 1) if start_byte > 0 else 0 + actual_end = min(end_byte + BOUNDARY_SCAN, total_size) + + range_header = f'bytes={actual_start}-{actual_end - 1}' + read_result = safe_s3_read(log_key, range_bytes=range_header) + + if not read_result['success']: + return success_response({ + 'logKey': log_key, 'content': '', 'startByte': start_byte, 'endByte': end_byte, + 'chunkSize': 0, 'totalSize': total_size, 'totalSizeHuman': format_bytes(total_size), + 'hasMore': False, 'nextChunkToken': None, 'truncated': False, + 'warning': read_result.get('error', 'Failed to read file content'), + }) + + raw = read_result['content'] + + aligned_start = start_byte + if start_byte > 0: + first_nl = raw.find('\n') + if first_nl >= 0: + aligned_start = actual_start + first_nl + 1 + raw = raw[first_nl + 1:] + + content_end_offset = end_byte - aligned_start + if content_end_offset < len(raw) and end_byte < total_size: + nl_pos = raw.find('\n', content_end_offset) + if nl_pos >= 0: + raw = raw[:nl_pos + 1] + aligned_end = aligned_start + nl_pos + 1 + else: + raw = raw[:content_end_offset] + aligned_end = end_byte + else: + aligned_end = aligned_start + len(raw) + + has_more = aligned_end < total_size + + return success_response({ + 'logKey': log_key, + 'content': raw, + 'startByte': aligned_start, + 'endByte': aligned_end, + 'chunkSize': len(raw), + 'totalSize': total_size, + 'totalSizeHuman': format_bytes(total_size), + 'hasMore': has_more, + 'nextChunkToken': str(aligned_end) if has_more else None, + 'truncated': False, + 'lineAligned': True, + }) + + except Exception as e: + return success_response({ + 'logKey': log_key, 'content': '', 'startByte': 0, 'endByte': 0, + 'chunkSize': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'hasMore': False, 'nextChunkToken': None, 'truncated': False, + 'error': f'Unexpected error reading log: {str(e)}', + }) + + +# ============================================================================= +# TIER 2: ADVANCED ANALYSIS +# ============================================================================= + +def search_logs_deep(arguments: Dict) -> Dict: + """ + Full-text regex search across all collected logs. + + Inputs: + instanceId: EC2 instance ID (required) + query: Regex pattern to search (required) + logTypes: Comma-separated log types to search (optional) + maxResults: Max results per file (default: 100) + + Returns: + matches[], pagination info, coverage_report + """ + instance_id = arguments.get('instanceId') + query = arguments.get('query') + log_types_str = arguments.get('logTypes', '') + try: + max_results = int(arguments.get('maxResults', 100)) + except (TypeError, ValueError): + return error_response(400, 'maxResults must be an integer between 1 and 500') + if max_results < 1 or max_results > 500: + return error_response(400, 'maxResults must be between 1 and 500') + + if not instance_id: + return error_response(400, 'instanceId is required') + instance_error = validate_instance_id(instance_id) + if instance_error: + return instance_error + if not query: + return error_response(400, 'query is required') + if len(query) > 500: + return error_response(400, 'query too long (max 500 characters)') + if is_dangerous_regex(query): + return error_response(400, 'Regex pattern rejected as potentially unsafe') + + try: + try: + pattern = re.compile(query, re.IGNORECASE) + except re.error as e: + return error_response(400, f'Invalid regex pattern: {str(e)}') + + file_patterns = None + if log_types_str: + file_patterns = [] + for log_type in log_types_str.split(','): + log_type = log_type.strip().lower() + if log_type in ECS_LOG_TYPE_PATTERNS: + file_patterns.extend(ECS_LOG_TYPE_PATTERNS[log_type]) + + # Use shared latest-bundle discovery + bundle_info = find_latest_bundle_files(instance_id) + + if not bundle_info['success']: + return success_response({ + 'instanceId': instance_id, 'query': query, + 'filesSearched': 0, 'filesWithMatches': 0, 'totalMatches': 0, + 'results': [], 'truncated': False, + 'warning': bundle_info.get('error', 'Failed to list log files'), + 'nextStep': 'Check if logs exist with validate', + }) + + files_to_search = [] + large_file_count = 0 + size_map = {obj['key']: obj['size'] for obj in bundle_info['all_objects']} + for key in bundle_info['files']: + if any(key.endswith(ext) for ext in ['.tar.gz', '.zip', '.gz', '.bin', '.so']): + continue + fsize = size_map.get(key, 0) + if fsize > 52428800: + large_file_count += 1 + continue + if file_patterns: + if not any(p in key.lower() for p in file_patterns): + continue + files_to_search.append({'key': key, 'size': fsize}) + + if not files_to_search: + return success_response({ + 'instanceId': instance_id, 'query': query, + 'filesSearched': 0, 'filesWithMatches': 0, 'totalMatches': 0, + 'results': [], 'truncated': False, + 'info': 'No log files found matching criteria.', + 'nextStep': 'Check log collection status or try different log types', + }) + + all_matches = [] + files_searched = 0 + files_with_errors = 0 + files_timed_out = 0 + + for file_info in files_to_search[:50]: + files_searched += 1 + try: + matches = search_file_for_pattern( + LOGS_BUCKET, file_info['key'], pattern, max_results + ) + except RegexTimeout: + files_timed_out += 1 + continue + except RegexTimeoutUnavailable as exc: + return error_response(503, f'Regex search timeout unavailable; refusing search: {exc}') + except Exception: + files_with_errors += 1 + continue + if matches: + filename = file_info['key'].split('/extracted/')[-1] + all_matches.append({ + 'file': filename, 'fullKey': file_info['key'], + 'matchCount': len(matches), 'matches': matches, + }) + if sum(len(m['matches']) for m in all_matches) >= max_results * 3: + break + + all_matches.sort(key=lambda x: x['matchCount'], reverse=True) + + finding_counter = 0 + for match_group in all_matches: + finding_counter += 1 + match_group['finding_id'] = f"S-{finding_counter:03d}" + + total_matches_kept = 0 + for match_group in all_matches: + remaining_budget = max(10, max_results * 3 - total_matches_kept) + if len(match_group['matches']) > remaining_budget: + match_group['matches'] = match_group['matches'][:remaining_budget] + match_group['matchCount'] = len(match_group['matches']) + match_group['matchesTruncated'] = True + total_matches_kept += len(match_group['matches']) + + return success_response({ + 'instanceId': instance_id, + 'query': query, + 'filesSearched': files_searched, + 'filesWithMatches': len(all_matches), + 'totalMatches': sum(m['matchCount'] for m in all_matches), + 'results': all_matches, + 'truncated': files_searched < len(files_to_search), + 'coverage_report': { + 'files_searched': files_searched, + 'files_available': len(files_to_search), + 'files_skipped_size': large_file_count, + 'files_with_errors': files_with_errors, + 'files_timed_out': files_timed_out, + 'scan_complete': files_searched >= len(files_to_search), + }, + 'regexTimeoutCount': files_timed_out, + 'interpretationGuide': { + 'CannotPullContainerError': 'ECR auth failure or image not found. Check task execution role.', + 'OOMKilled': 'Container exceeded memory limit. Check task definition memory.', + 'AGENT_DISCONNECTED': 'ECS agent lost connection. Check instance networking.', + 'connection timed out': 'Network connectivity issue. Check security groups and NACLs.', + }, + 'nextStep': 'Use read to get full context around specific matches', + }) + + except Exception as e: + return success_response({ + 'instanceId': instance_id, 'query': query, + 'filesSearched': 0, 'filesWithMatches': 0, 'totalMatches': 0, + 'results': [], 'truncated': False, + 'error': f'Search encountered an error: {str(e)}', + 'nextStep': 'Check if logs exist with validate', + }) + + +def correlate_events(arguments: Dict) -> Dict: + """ + Cross-file timeline correlation for ECS incident analysis. + + Inputs: + instanceId: EC2 instance ID (required) + timeWindow: Seconds around pivot event (default: 60) + + Returns: + timeline[], correlations, temporal_clusters, potential_root_cause_chain + """ + instance_id = arguments.get('instanceId') + time_window = arguments.get('timeWindow', 60) + + if not instance_id: + return error_response(400, 'instanceId is required') + + try: + prefix = f'ecs_{instance_id}' + index_key = find_findings_index(prefix) + findings = [] + files_scanned = 0 + + if index_key: + read_result = safe_s3_read(index_key) + if read_result['success']: + try: + index_data = json.loads(read_result['content']) + findings = index_data.get('findings', []) + files_scanned = index_data.get('filesScanned', 0) + except json.JSONDecodeError: + pass + + if not findings: + error_summary = scan_and_index_errors(instance_id, 'all') + if error_summary['statusCode'] != 200: + return success_response({ + 'instanceId': instance_id, 'timeWindow': time_window, + 'timeline': [], 'byComponent': {}, 'correlations': [], + 'temporal_clusters': [], 'potential_root_cause_chain': [], + 'coverage_report': {'files_scanned': 0, 'scan_complete': False}, + 'confidence': 'none', 'gaps': ['Could not retrieve error data'], + 'nextStep': 'Check if logs exist with validate', + }) + summary_data = json.loads(error_summary['body']) + findings = summary_data.get('findings', []) + files_scanned = summary_data.get('coverage_report', {}).get('files_scanned', 0) + + if not findings: + return success_response({ + 'instanceId': instance_id, 'timeWindow': time_window, + 'timeline': [], 'byComponent': {}, 'correlations': [], + 'temporal_clusters': [], 'potential_root_cause_chain': [], + 'coverage_report': {'files_scanned': files_scanned, 'scan_complete': True}, + 'confidence': 'none', 'gaps': [], + 'info': 'No error findings to correlate. Instance may be healthy.', + 'nextStep': 'Use search to search for specific patterns', + }) + + # Build timeline + timeline = [] + for idx, finding in enumerate(findings): + timestamp = extract_timestamp(finding.get('sample', '')) + timeline.append({ + 'finding_id': finding.get('finding_id', assign_finding_id(idx + 1)), + 'timestamp': timestamp, + 'source': finding.get('file', 'unknown'), + 'severity': finding.get('severity', 'info'), + 'event': finding.get('pattern', ''), + 'sample': finding.get('sample', '')[:200], + 'count': finding.get('count', 1), + }) + + timeline.sort(key=lambda x: (SEVERITY_ORDER.get(x['severity'], 4), -x['count'])) + + by_component = {} + for event in timeline: + component = categorize_log_source(event['source']) + if component not in by_component: + by_component[component] = [] + by_component[component].append(event) + + temporal_clusters = _build_temporal_clusters(timeline, time_window) + root_cause_chain = _build_root_cause_chain(findings) + + critical_count = len([e for e in timeline if e['severity'] == 'critical']) + if critical_count > 0 and len(timeline) >= 3: + confidence = 'high' + elif len(timeline) >= 2: + confidence = 'medium' + else: + confidence = 'low' + + gaps = [] + if files_scanned < 10: + gaps.append('Few files scanned — some log sources may be missing') + timestamps_present = sum(1 for e in timeline if e.get('timestamp')) + if timestamps_present < len(timeline) * 0.5: + gaps.append('Many events lack timestamps — temporal ordering may be unreliable') + + return success_response({ + 'instanceId': instance_id, + 'timeWindow': time_window, + 'timeline': timeline[:50], + 'byComponent': by_component, + 'correlations': find_correlations(timeline), + 'temporal_clusters': temporal_clusters, + 'potential_root_cause_chain': root_cause_chain, + 'confidence': confidence, + 'gaps': gaps, + 'coverage_report': { + 'files_scanned': files_scanned, + 'components_found': list(by_component.keys()), + 'events_with_timestamps': timestamps_present, + 'events_total': len(timeline), + 'scan_complete': True, + }, + 'caveat': ( + 'Timeline correlation is based on pattern matching across log files. ' + 'Timestamps may not be perfectly synchronized across components. ' + 'Correlation does not imply causation.' + ), + 'nextStep': 'Use search to investigate specific events', + 'recommendedSOPs': match_sops_for_issues([], findings=findings), + }) + + except Exception as e: + return success_response({ + 'instanceId': instance_id, 'timeWindow': time_window, + 'timeline': [], 'byComponent': {}, 'correlations': [], + 'temporal_clusters': [], 'potential_root_cause_chain': [], + 'confidence': 'none', 'gaps': [f'Correlation error: {str(e)}'], + 'coverage_report': {'files_scanned': 0, 'scan_complete': False}, + 'error': f'Correlation encountered an error: {str(e)}', + 'nextStep': 'Check if logs exist with validate', + }) + + +def get_artifact_reference(arguments: Dict) -> Dict: + """ + Get secure presigned URL for large artifacts. + + Inputs: + logKey: S3 key of artifact (required) + expirationMinutes: URL expiration (default: 15, max: 60) + + Returns: + presignedUrl, s3Uri, size + """ + log_key = arguments.get('logKey') + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + key_error = validate_log_key(log_key, instance_id) + if key_error: + return key_error + requested_expiration = arguments.get('expirationMinutes', None) + if requested_expiration is None: + expiration_seconds = PRESIGNED_URL_EXPIRATION + else: + if isinstance(requested_expiration, bool): + return error_response(400, 'expirationMinutes must be a positive integer') + try: + expiration_minutes = int(requested_expiration) + except (TypeError, ValueError): + return error_response(400, 'expirationMinutes must be a positive integer') + if expiration_minutes <= 0: + return error_response(400, 'expirationMinutes must be a positive integer') + expiration_seconds = min(expiration_minutes * 60, PRESIGNED_URL_EXPIRATION) + + try: + head_result = safe_s3_head(log_key) + if not head_result['success']: + return success_response({ + 'logKey': log_key, 'presignedUrl': None, + 's3Uri': f's3://{LOGS_BUCKET}/{log_key}', + 'size': 0, 'sizeHuman': '0 B', 'fileNotFound': True, + 'warning': head_result.get('error', 'File not found'), + }) + + try: + presigned_url = s3_client.generate_presigned_url( + 'get_object', + Params={'Bucket': LOGS_BUCKET, 'Key': log_key}, + ExpiresIn=expiration_seconds, + ) + except Exception as e: + return success_response({ + 'logKey': log_key, 'presignedUrl': None, + 's3Uri': f's3://{LOGS_BUCKET}/{log_key}', + 'size': head_result['size'], 'sizeHuman': format_bytes(head_result['size']), + 'warning': f'Could not generate presigned URL: {str(e)}', + }) + + return success_response({ + 'logKey': log_key, + 'presignedUrl': presigned_url, + 's3Uri': f's3://{LOGS_BUCKET}/{log_key}', + 'size': head_result['size'], + 'sizeHuman': format_bytes(head_result['size']), + 'contentType': head_result.get('content_type', 'application/octet-stream'), + 'lastModified': head_result.get('last_modified'), + 'expiresIn': f'{expiration_seconds} seconds', + 'expiresAt': ( + datetime.now(timezone.utc) + timedelta(seconds=expiration_seconds) + ).isoformat(), + }) + + except Exception as e: + return success_response({ + 'logKey': log_key, 'presignedUrl': None, + 's3Uri': f's3://{LOGS_BUCKET}/{log_key}', + 'size': 0, 'sizeHuman': '0 B', + 'error': f'Unexpected error: {str(e)}', + }) + + +def generate_incident_summary(arguments: Dict) -> Dict: + """ + Generate structured incident summary grounded in finding_ids. + + Inputs: + instanceId: EC2 instance ID (required) + finding_ids: List of finding IDs from errors/search (required) + includeRecommendations: Include remediation suggestions (default: true) + includeTriage: Include ECS triage analysis (default: true) + + Returns: + summary with criticalFindings, timeline, recommendations, ecs_triage + """ + import time as _time + start_time = _time.time() + MAX_EXECUTION_TIME = 25 + + def check_timeout(): + if _time.time() - start_time > MAX_EXECUTION_TIME: + raise TimeoutError(f"Execution time exceeded {MAX_EXECUTION_TIME}s") + + instance_id = arguments.get('instanceId') + finding_ids = arguments.get('finding_ids', []) + include_recommendations = arguments.get('includeRecommendations', True) + include_triage = arguments.get('includeTriage', True) + + if not instance_id: + return error_response(400, 'instanceId is required') + if not finding_ids: + return error_response(400, + 'finding_ids is required. Call errors tool first to get finding_ids (F-001 format), ' + 'then pass them here to ground the summary in verified evidence.') + + try: + bundle_data = {} + try: + check_timeout() + bundle_result = validate_bundle_completeness({'instanceId': instance_id}) + if bundle_result['statusCode'] == 200: + bundle_data = json.loads(bundle_result['body']) + except TimeoutError: + raise + except Exception as e: + print(f"Warning: Could not get bundle completeness: {str(e)}") + + error_data = {} + try: + check_timeout() + error_result = get_error_summary({'instanceId': instance_id, 'severity': 'all', 'pageSize': 200}) + if error_result['statusCode'] == 200: + error_data = json.loads(error_result['body']) + except TimeoutError: + raise + except Exception as e: + print(f"Warning: Could not get error summary: {str(e)}") + + check_timeout() + + all_findings = error_data.get('findings', []) + summary_counts = error_data.get('summary', {'critical': 0, 'high': 0, 'medium': 0, 'low': 0, 'info': 0}) + + finding_id_set = set(finding_ids) + findings = [f for f in all_findings if f.get('finding_id') in finding_id_set] + unresolved_ids = finding_id_set - {f.get('finding_id') for f in findings} + + critical_findings = [f for f in findings if f.get('severity') == 'critical'][:10] + high_findings = [f for f in findings if f.get('severity') == 'high'][:10] + medium_findings = [f for f in findings if f.get('severity') == 'medium'][:5] + + affected_components = set() + for finding in findings: + affected_components.add(categorize_log_source(finding.get('file', ''))) + + if len(findings) >= 3 and critical_findings: + confidence = 'high' + elif len(findings) >= 1: + confidence = 'medium' + else: + confidence = 'low' + + gaps = [] + if unresolved_ids: + gaps.append(f'{len(unresolved_ids)} finding_ids could not be resolved: {list(unresolved_ids)[:5]}') + + summary = { + 'instanceId': instance_id, + 'generatedAt': datetime.utcnow().isoformat(), + 'executionTimeMs': int((_time.time() - start_time) * 1000), + 'grounded': True, + 'confidence': confidence, + 'gaps': gaps, + 'bundleStatus': { + 'complete': bundle_data.get('complete', False), + 'fileCount': bundle_data.get('fileCount', 0), + 'totalSize': bundle_data.get('totalSizeHuman', 'unknown'), + }, + 'errorSummary': { + 'critical': summary_counts.get('critical', 0), + 'high': summary_counts.get('high', 0), + 'medium': summary_counts.get('medium', 0), + 'low': summary_counts.get('low', 0), + 'info': summary_counts.get('info', 0), + 'total': len(all_findings), + }, + 'criticalFindings': [ + {'finding_id': f.get('finding_id'), 'file': f.get('file'), 'fullKey': f.get('fullKey'), + 'pattern': f.get('pattern'), 'count': f.get('count'), 'sample': f.get('sample', '')[:200]} + for f in critical_findings + ], + 'highFindings': [ + {'finding_id': f.get('finding_id'), 'file': f.get('file'), 'fullKey': f.get('fullKey'), + 'pattern': f.get('pattern'), 'count': f.get('count')} + for f in high_findings + ], + 'affectedComponents': list(affected_components), + } + + if not findings: + summary['info'] = 'No error findings detected. Instance may be healthy.' + + if include_recommendations: + summary['recommendations'] = generate_recommendations(critical_findings, high_findings, medium_findings) + + summary['artifactLinks'] = [] + for finding in critical_findings[:5]: + if finding.get('fullKey'): + summary['artifactLinks'].append({ + 'finding_id': finding.get('finding_id'), + 'file': finding.get('file'), + 'key': finding.get('fullKey'), + 'action': f'read(logKey="{finding.get("fullKey")}")', + }) + + if include_triage and findings: + try: + check_timeout() + triage_result = perform_ecs_triage(instance_id, findings, bundle_data) + summary['ecs_triage'] = triage_result + except TimeoutError: + summary['ecs_triage'] = {'triageVersion': '1.0', 'warning': 'Triage skipped due to time constraints.'} + except Exception as e: + summary['ecs_triage'] = {'triageVersion': '1.0', 'error': f'Triage failed: {str(e)}'} + elif include_triage: + summary['ecs_triage'] = { + 'triageVersion': '1.0', 'info': 'No findings to triage.', + 'task_states_detected': [], 'instance_conditions_detected': [], + 'most_likely_root_cause': None, 'evidence': [], + } + + summary['caveat'] = ( + 'Root cause analysis is based on log pattern matching only. ' + 'Verify findings by checking ECS agent logs, Docker daemon config, ' + 'task definitions, and instance networking. ' + 'Log patterns indicate symptoms, not always root causes.' + ) + + if summary.get('ecs_triage', {}).get('most_likely_root_cause'): + root_cause = summary['ecs_triage']['most_likely_root_cause'] + summary['nextStep'] = f"Root cause: {root_cause['category_name']} ({root_cause['confidence']}). Follow remediation steps." + else: + summary['nextStep'] = 'Use search for detailed investigation of specific patterns' + + # Auto-match SOPs based on findings and triage category + triage_cat = None + if summary.get('ecs_triage', {}).get('most_likely_root_cause'): + triage_cat = summary['ecs_triage']['most_likely_root_cause'].get('category') + summary['recommendedSOPs'] = match_sops_for_issues( + [], findings=findings, triage_category=triage_cat + ) + + summary['executionTimeMs'] = int((_time.time() - start_time) * 1000) + return success_response(summary) + + except TimeoutError: + return success_response({ + 'instanceId': instance_id, 'generatedAt': datetime.utcnow().isoformat(), + 'grounded': True, 'confidence': 'low', + 'gaps': ['Execution timed out — partial results only'], + 'warning': 'Execution timed out', + 'nextStep': 'Call errors first, then summarize with includeTriage=false', + }) + except Exception as e: + return success_response({ + 'instanceId': instance_id, 'generatedAt': datetime.utcnow().isoformat(), + 'grounded': True, 'confidence': 'none', + 'gaps': [f'Summary generation failed: {str(e)}'], + 'error': f'Could not generate complete summary: {str(e)}', + 'nextStep': 'Check if logs exist with validate', + }) + + +def list_collection_history(arguments: Dict) -> Dict: + """ + List historical ECS log collections for audit and comparison. + + Inputs: + instanceId: Filter by instance (optional) + maxResults: Max results (default: 20) + status: Filter by status (optional) + + Returns: + collections[], count + """ + instance_id = arguments.get('instanceId') + if instance_id: + instance_error = validate_instance_id(instance_id) + if instance_error: + return instance_error + try: + max_results = int(arguments.get('maxResults', 20)) + except (TypeError, ValueError): + return error_response(400, 'maxResults must be an integer between 1 and 50') + if max_results < 1 or max_results > 50: + return error_response(400, 'maxResults must be between 1 and 50') + status_filter = arguments.get('status') + document_name = 'AWSSupport-CollectECSInstanceLogs' + + try: + filters = [{'Key': 'DocumentNamePrefix', 'Values': [document_name]}] + if status_filter: + filters.append({'Key': 'ExecutionStatus', 'Values': [status_filter]}) + + explicit_region = arguments.get('region') + if explicit_region is not None: + region_error = validate_region(explicit_region) + if region_error: + return region_error + regions_to_try = [explicit_region] + else: + regions_to_try = sorted(ALLOWED_REGIONS) + + collections = [] + searched_regions = [] + + for region in regions_to_try: + try: + regional_ssm = get_regional_client('ssm', region) + response = regional_ssm.describe_automation_executions( + Filters=filters, MaxResults=max_results, + ) + for exec_meta in response.get('AutomationExecutionMetadataList', []): + exec_id = exec_meta.get('AutomationExecutionId', '') + provenance = get_execution_provenance(exec_id) + if ( + not provenance + or provenance.get('tool') != 'collect' + or provenance.get('region') != region + ): + continue + provenance_region_error = validate_region(provenance.get('region')) + if provenance_region_error: + continue + exec_instance = provenance.get('instanceId') + if not exec_instance or validate_instance_id(exec_instance): + continue + if instance_id and instance_id != exec_instance: + continue + if exec_meta.get('DocumentName') != provenance.get('expectedDocument'): + continue + bundle_info = find_latest_bundle_files(exec_instance) + bundle_exists = bool(bundle_info.get('success') and bundle_info.get('files')) + + collections.append({ + 'executionId': exec_id, + 'documentName': exec_meta.get('DocumentName', ''), + 'status': exec_meta['AutomationExecutionStatus'], + 'startTime': exec_meta.get('ExecutionStartTime'), + 'endTime': exec_meta.get('ExecutionEndTime'), + 'instanceId': exec_instance or None, + 'region': region, + 'bundleExists': bundle_exists, + }) + + searched_regions.append(region) + if collections: + break + except Exception: + searched_regions.append(f"{region} (error)") + continue + + return success_response({ + 'collections': collections, + 'count': len(collections), + 'searchedRegions': searched_regions, + 'filters': {'instanceId': instance_id, 'status': status_filter, 'documentName': document_name}, + }) + + except Exception as e: + return error_response(500, f'Failed to list history: {str(e)}') + + +# ============================================================================= +# TIER 3: CLUSTER-LEVEL INTELLIGENCE +# ============================================================================= + +def cluster_health_check(arguments: Dict) -> Dict: + """ + Comprehensive ECS cluster health overview. + Enumerates container instances, checks SSM status, flags unhealthy instances. + + Inputs: + clusterName: ECS cluster name (required) + region: AWS region (optional) + includeSSMStatus: Check SSM agent per instance (default: true) + + Returns: + clusterInfo, instances[], healthSummary + """ + cluster_name = arguments.get('clusterName') + cluster_error = validate_cluster_name(cluster_name) + if cluster_error: + return cluster_error + + include_ssm = arguments.get('includeSSMStatus', True) + target_region, region_error = resolve_and_validate_region(arguments) + if region_error: + return region_error + + try: + regional_ecs = get_regional_client('ecs', target_region) + regional_ec2 = get_regional_client('ec2', target_region) + regional_ssm = get_regional_client('ssm', target_region) + + # 1. Describe the cluster + try: + cluster_resp = regional_ecs.describe_clusters(clusters=[cluster_name], include=['STATISTICS', 'SETTINGS']) + clusters = cluster_resp.get('clusters', []) + if not clusters: + return error_response(404, f'Cluster {cluster_name} not found in {target_region}') + cluster_info = clusters[0] + cluster_meta = { + 'name': cluster_info.get('clusterName'), + 'status': cluster_info.get('status'), + 'registeredContainerInstancesCount': cluster_info.get('registeredContainerInstancesCount', 0), + 'runningTasksCount': cluster_info.get('runningTasksCount', 0), + 'pendingTasksCount': cluster_info.get('pendingTasksCount', 0), + 'activeServicesCount': cluster_info.get('activeServicesCount', 0), + 'region': target_region, + } + except Exception as e: + return error_response(404, f'Cluster {cluster_name} not found in {target_region}: {str(e)}') + + # 2. List container instances + container_instance_arns = [] + try: + paginator = regional_ecs.get_paginator('list_container_instances') + for page in paginator.paginate(cluster=cluster_name): + container_instance_arns.extend(page.get('containerInstanceArns', [])) + except Exception as e: + return error_response(500, f'Failed to list container instances: {str(e)}') + + instances = [] + instance_ids = [] + + if container_instance_arns: + # Describe in batches of 100 + for i in range(0, len(container_instance_arns), 100): + batch = container_instance_arns[i:i + 100] + try: + desc_resp = regional_ecs.describe_container_instances( + cluster=cluster_name, containerInstances=batch, + ) + for ci in desc_resp.get('containerInstances', []): + ec2_id = ci.get('ec2InstanceId', '') + if ec2_id: + instance_ids.append(ec2_id) + instances.append({ + 'containerInstanceArn': ci.get('containerInstanceArn'), + 'ec2InstanceId': ec2_id, + 'status': ci.get('status'), + 'agentConnected': ci.get('agentConnected'), + 'runningTasksCount': ci.get('runningTasksCount', 0), + 'pendingTasksCount': ci.get('pendingTasksCount', 0), + 'registeredAt': ci.get('registeredAt'), + 'agentVersion': ci.get('versionInfo', {}).get('agentVersion'), + 'dockerVersion': ci.get('versionInfo', {}).get('dockerVersion'), + 'ssmStatus': None, + }) + except Exception: + pass + + # 3. Enrich with EC2 metadata + if instance_ids: + try: + ec2_paginator = regional_ec2.get_paginator('describe_instances') + ec2_map = {} + for page in ec2_paginator.paginate(InstanceIds=instance_ids): + for res in page.get('Reservations', []): + for inst in res.get('Instances', []): + ec2_map[inst['InstanceId']] = { + 'instanceType': inst.get('InstanceType'), + 'availabilityZone': inst.get('Placement', {}).get('AvailabilityZone'), + 'state': inst.get('State', {}).get('Name'), + 'privateIp': inst.get('PrivateIpAddress'), + 'imageId': inst.get('ImageId'), + 'launchTime': inst.get('LaunchTime'), + } + for inst in instances: + ec2_data = ec2_map.get(inst['ec2InstanceId'], {}) + inst.update(ec2_data) + except Exception: + pass + + # 4. Check SSM status + if include_ssm and instance_ids: + ssm_status_map = {} + try: + for i in range(0, len(instance_ids), 50): + chunk = instance_ids[i:i + 50] + ssm_paginator = regional_ssm.get_paginator('describe_instance_information') + for page in ssm_paginator.paginate(Filters=[{'Key': 'InstanceIds', 'Values': chunk}]): + for info in page.get('InstanceInformationList', []): + ssm_status_map[info['InstanceId']] = { + 'pingStatus': info.get('PingStatus'), + 'agentVersion': info.get('AgentVersion'), + 'lastPingTime': info.get('LastPingDateTime'), + } + except Exception as e: + print(f"SSM status check failed: {e}") + + for inst in instances: + ssm_info = ssm_status_map.get(inst.get('ec2InstanceId')) + inst['ssmStatus'] = ssm_info or {'pingStatus': 'NotRegistered', 'agentVersion': None} + + # 5. Build health summary + total = len(instances) + active = sum(1 for i in instances if i.get('status') == 'ACTIVE') + agent_connected = sum(1 for i in instances if i.get('agentConnected')) + ssm_online = sum(1 for i in instances if i.get('ssmStatus', {}).get('pingStatus') == 'Online') + + az_distribution = {} + for i in instances: + az = i.get('availabilityZone', 'unknown') + az_distribution[az] = az_distribution.get(az, 0) + 1 + + unhealthy = [] + for i in instances: + issues = [] + if i.get('status') != 'ACTIVE': + issues.append(f"ecsStatus={i.get('status')}") + if not i.get('agentConnected'): + issues.append('agentDisconnected') + if include_ssm and i.get('ssmStatus', {}).get('pingStatus') != 'Online': + issues.append(f"ssm={i.get('ssmStatus', {}).get('pingStatus', 'unknown')}") + if issues: + unhealthy.append({'instanceId': i.get('ec2InstanceId'), 'issues': issues}) + + health_summary = { + 'totalInstances': total, + 'active': active, + 'agentConnected': agent_connected, + 'ssmOnline': ssm_online, + 'unhealthyCount': len(unhealthy), + 'azDistribution': az_distribution, + } + + gaps = [] + if not include_ssm: + gaps.append('SSM status not checked') + if total == 0: + gaps.append('No container instances found — cluster may be empty or Fargate-only') + + if total > 0 and include_ssm and ssm_online == total: + confidence = 'high' + elif total > 0: + confidence = 'medium' + else: + confidence = 'none' + + return success_response({ + 'cluster': cluster_meta, + 'instances': instances, + 'unhealthyInstances': unhealthy, + 'healthSummary': health_summary, + 'region': target_region, + 'confidence': confidence, + 'gaps': gaps, + 'nextStep': 'Use compare_instances to diff specific instances, or batch_collect to sample unhealthy ones' if unhealthy else 'Cluster looks healthy.', + }) + + except Exception as e: + return error_response(500, f'cluster_health failed: {str(e)}') + + +def compare_instances(arguments: Dict) -> Dict: + """ + Diff error findings between two or more ECS container instances. + + Inputs: + instanceIds: list of 2+ instance IDs (required) + compareFields: "errors", "config", "all" (default: "all") + + Returns: + commonFindings[], uniqueFindings{}, comparisonMatrix + """ + instance_ids = arguments.get('instanceIds', []) + if not instance_ids or len(instance_ids) < 2: + return error_response(400, 'instanceIds must contain at least 2 instance IDs') + + seen = set() + deduped = [] + for iid in instance_ids: + if iid not in seen: + seen.add(iid) + deduped.append(iid) + instance_ids = deduped + if len(instance_ids) < 2: + return error_response(400, 'instanceIds must contain at least 2 distinct instance IDs') + if len(instance_ids) > 10: + return error_response(400, 'Maximum 10 instances for comparison') + for instance_id in instance_ids: + instance_error = validate_instance_id(instance_id) + if instance_error: + return instance_error + + compare_fields = arguments.get('compareFields', 'all') + + try: + node_findings = {} + node_configs = {} + + def _gather_instance_data(iid): + nf = [] + nc = {} + if compare_fields in ('errors', 'all'): + prefix = f"ecs_{iid}" + try: + idx = find_findings_index(prefix) + if idx: + resp = s3_client.get_object(Bucket=LOGS_BUCKET, Key=idx) + findings_data = json.loads(resp['Body'].read().decode('utf-8')) + nf = findings_data.get('findings', []) + else: + nf = [{'error': f'No findings index for {iid}. Run collect first.', 'needsCollection': True}] + except Exception as e: + nf = [{'error': f'Could not load findings: {str(e)}'}] + + if compare_fields in ('config', 'all'): + bundle_info = find_latest_bundle_files(iid) + extracted_prefix = None + if bundle_info['success']: + extracted_prefix = bundle_info['bundle_prefix'] + '/extracted/' + if extracted_prefix: + config_files = [ + ('ecs_config', f"{extracted_prefix}ecs.config"), + ('docker_daemon', f"{extracted_prefix}docker-daemon.json"), + ] + else: + config_files = [] + for config_name, config_key in config_files: + result = safe_s3_read(config_key, max_size=65536) + nc[config_name] = result['content'][:2000] if result.get('success') else None + return iid, nf, nc + + with ThreadPoolExecutor(max_workers=min(len(instance_ids), 10)) as executor: + futures = {executor.submit(_gather_instance_data, iid): iid for iid in instance_ids} + for future in as_completed(futures): + iid_key = futures[future] + try: + iid, nf, nc = future.result() + node_findings[iid] = nf + node_configs[iid] = nc + except Exception as e: + node_findings[iid_key] = [{'error': f'Failed: {str(e)}'}] + node_configs[iid_key] = {} + + # Build comparison + common_findings = [] + unique_findings = {} + + if compare_fields in ('errors', 'all') and node_findings: + def finding_signature(f): + if 'error' in f and 'severity' not in f: + return f"__error__{f.get('error', 'unknown')[:80]}" + return f"{f.get('severity', '')}__{f.get('category', '')}__{f.get('pattern', f.get('message', ''))[:80]}" + + sig_to_nodes = {} + for iid, findings in node_findings.items(): + unique_findings[iid] = [] + for f in findings: + sig = finding_signature(f) + if sig not in sig_to_nodes: + sig_to_nodes[sig] = {'finding': f, 'nodes': []} + sig_to_nodes[sig]['nodes'].append(iid) + + for sig, data in sig_to_nodes.items(): + if len(data['nodes']) == len(instance_ids): + common_findings.append({**data['finding'], 'presentOnAllInstances': True}) + else: + for iid in data['nodes']: + unique_findings[iid].append({**data['finding'], 'uniqueTo': iid}) + + # Config diff + config_diffs = {} + if compare_fields in ('config', 'all') and node_configs: + ref_id = instance_ids[0] + ref_config = node_configs.get(ref_id, {}) + for iid in instance_ids[1:]: + other_config = node_configs.get(iid, {}) + diffs = [] + all_keys = set(list(ref_config.keys()) + list(other_config.keys())) + for key in all_keys: + if ref_config.get(key) != other_config.get(key): + diffs.append({ + 'configFile': key, 'referenceInstance': ref_id, + 'comparedInstance': iid, 'match': False, + 'note': 'Content differs' if (ref_config.get(key) and other_config.get(key)) else 'Missing on one instance', + }) + config_diffs[f"{ref_id}_vs_{iid}"] = diffs if diffs else [{'match': True, 'note': 'Configs identical'}] + + matrix = [] + for iid in instance_ids: + total_f = len(node_findings.get(iid, [])) + unique_count = len(unique_findings.get(iid, [])) + critical_count = sum(1 for f in node_findings.get(iid, []) if f.get('severity') == 'critical') + matrix.append({ + 'instanceId': iid, 'totalFindings': total_f, + 'criticalFindings': critical_count, 'uniqueFindings': unique_count, + 'commonFindings': total_f - unique_count, + }) + + common_count = len(common_findings) + total_unique = sum(len(v) for v in unique_findings.values()) + if common_count > 0 and total_unique == 0: + insight = f"All {len(instance_ids)} instances share the same {common_count} findings. Likely a cluster-wide issue." + elif common_count == 0 and total_unique > 0: + insight = "No common findings. Each instance has unique issues — investigate individually." + elif common_count > total_unique: + insight = f"{common_count} common vs {total_unique} unique. Mostly a shared problem." + else: + insight = f"{common_count} common, {total_unique} unique. Mixed picture." + + return success_response({ + 'comparedInstances': instance_ids, + 'commonFindings': common_findings, + 'commonFindingsCount': common_count, + 'uniqueFindings': unique_findings, + 'configDiffs': config_diffs, + 'comparisonMatrix': matrix, + 'insight': insight, + 'caveat': 'Comparison is based on pre-indexed error findings. Differences may reflect different workloads.', + 'nextStep': 'Common findings suggest cluster-wide issue. Unique findings point to instance-specific problems.', + }) + + except Exception as e: + return error_response(500, f'compare_instances failed: {str(e)}') + + +def batch_collect(arguments: Dict) -> Dict: + """ + Smart batch log collection with statistical sampling for ECS clusters. + + Inputs: + clusterName: ECS cluster name (required) + region: AWS region (optional) + filter: "all", "unhealthy", "disconnected" (default: "unhealthy") + strategy: "sample" or "all" (default: "sample") + samplesPerBucket: instances per bucket (default: 3, max: 5) + maxTotalCollections: hard cap (default: 15, max: 15) + dryRun: preview only (default: false) + + Returns: + buckets[], plannedCollections, executions[] (if not dryRun) + """ + cluster_name = arguments.get('clusterName') + cluster_error = validate_cluster_name(cluster_name) + if cluster_error: + return cluster_error + + target_region, region_error = resolve_and_validate_region(arguments) + if region_error: + return region_error + node_filter = arguments.get('filter', 'unhealthy') + valid_filters = ('all', 'unhealthy', 'disconnected') + if node_filter not in valid_filters: + return error_response(400, f"Invalid filter '{node_filter}'. Must be one of: {', '.join(valid_filters)}") + strategy = arguments.get('strategy', 'sample') + if strategy not in ('sample', 'all'): + return error_response(400, "strategy must be 'sample' or 'all'") + try: + samples_per_bucket = int(arguments.get('samplesPerBucket', 3)) + max_total = int(arguments.get('maxTotalCollections', 15)) + except (TypeError, ValueError): + return error_response(400, 'samplesPerBucket and maxTotalCollections must be integers') + if samples_per_bucket < 1 or samples_per_bucket > 5: + return error_response(400, 'samplesPerBucket must be between 1 and 5') + if max_total < 1 or max_total > 15: + return error_response(400, 'maxTotalCollections must be between 1 and 15') + dry_run = arguments.get('dryRun', True) + + try: + regional_ecs = get_regional_client('ecs', target_region) + regional_ec2 = get_regional_client('ec2', target_region) + regional_ssm = get_regional_client('ssm', target_region) + + # 1. List container instances + ci_arns = [] + paginator = regional_ecs.get_paginator('list_container_instances') + for page in paginator.paginate(cluster=cluster_name): + ci_arns.extend(page.get('containerInstanceArns', [])) + + if not ci_arns: + return error_response(404, f'No container instances found for cluster {cluster_name}') + + # 2. Describe container instances + all_instances = [] + for i in range(0, len(ci_arns), 100): + batch = ci_arns[i:i + 100] + desc_resp = regional_ecs.describe_container_instances( + cluster=cluster_name, containerInstances=batch + ) + if desc_resp.get('failures'): + return error_response(500, 'Failed to describe every selected container instance') + for ci in desc_resp.get('containerInstances', []): + ec2_id = ci.get('ec2InstanceId', '') + if ci.get('status') != 'ACTIVE' or validate_instance_id(ec2_id): + continue + all_instances.append({ + 'instanceId': ec2_id, + 'status': 'ACTIVE', + 'agentConnected': ci.get('agentConnected'), + 'runningTasks': ci.get('runningTasksCount', 0), + }) + + # 3. Check SSM status + instance_ids = [i['instanceId'] for i in all_instances if i['instanceId']] + ssm_status = {} + if instance_ids: + try: + for i in range(0, len(instance_ids), 50): + chunk = instance_ids[i:i + 50] + ssm_paginator = regional_ssm.get_paginator('describe_instance_information') + for page in ssm_paginator.paginate(Filters=[{'Key': 'InstanceIds', 'Values': chunk}]): + for info in page.get('InstanceInformationList', []): + ssm_status[info['InstanceId']] = info.get('PingStatus', 'Unknown') + except Exception: + pass + + # 4. Apply filter + filtered = [] + for inst in all_instances: + ssm_ping = ssm_status.get(inst['instanceId'], 'NotRegistered') + inst['ssmPingStatus'] = ssm_ping + is_unhealthy = inst['status'] != 'ACTIVE' or not inst['agentConnected'] or ssm_ping != 'Online' + is_disconnected = not inst['agentConnected'] or ssm_ping != 'Online' + + if node_filter == 'all': + filtered.append(inst) + elif node_filter == 'unhealthy' and is_unhealthy: + filtered.append(inst) + elif node_filter == 'disconnected' and is_disconnected: + filtered.append(inst) + + if not filtered and node_filter in ('unhealthy', 'disconnected'): + return success_response({ + 'message': f'No {node_filter} instances found — cluster looks healthy', + 'totalInstances': len(all_instances), 'filteredInstances': 0, + 'filter': node_filter, 'buckets': [], 'plannedCollections': 0, + }) + + # 5. Group into buckets by status + SSM + buckets = {} + for inst in filtered: + key = f"{inst['status']}|{inst['ssmPingStatus']}" + if key not in buckets: + buckets[key] = {'signature': key, 'nodes': [], 'count': 0} + buckets[key]['nodes'].append(inst) + buckets[key]['count'] += 1 + + # 6. Select samples + bucket_list = [] + total_planned = 0 + for sig, bucket in buckets.items(): + sample_count = min(samples_per_bucket, bucket['count']) if strategy == 'sample' else bucket['count'] + if total_planned + sample_count > max_total: + sample_count = max(0, max_total - total_planned) + sample_nodes = bucket['nodes'][:sample_count] + total_planned += len(sample_nodes) + bucket_list.append({ + 'signature': sig, 'totalInstances': bucket['count'], + 'sampleCount': len(sample_nodes), + 'sampleInstances': [n['instanceId'] for n in sample_nodes], + }) + + if dry_run: + return success_response({ + 'dryRun': True, 'clusterName': cluster_name, 'region': target_region, + 'totalInstances': len(all_instances), 'filteredInstances': len(filtered), + 'filter': node_filter, 'strategy': strategy, + 'bucketCount': len(bucket_list), 'buckets': bucket_list, + 'plannedCollections': total_planned, + 'message': f'{len(filtered)} instances grouped into {len(bucket_list)} buckets. Will collect from {total_planned}. Re-run with dryRun=false to proceed.', + }) + + # 7. Execute collections + batch_id = hashlib.md5(f"{cluster_name}-{datetime.utcnow().isoformat()}".encode()).hexdigest()[:12] + executions = [] + + for bucket in bucket_list: + for iid in bucket['sampleInstances']: + try: + membership_error = validate_ecs_instance( + iid, target_region, cluster_name=cluster_name + ) + if membership_error: + executions.append({ + 'instanceId': iid, + 'bucket': bucket['signature'], + 'status': 'Failed', + 'error': json.loads(membership_error['body']).get('error'), + }) + continue + collect_args = { + 'instanceId': iid, 'region': target_region, + 'idempotencyToken': f"batch-{batch_id}-{iid}", + } + result = start_log_collection(collect_args) + result_body = json.loads(result.get('body', '{}')) + executions.append({ + 'instanceId': iid, 'bucket': bucket['signature'], + 'executionId': result_body.get('executionId'), + 'status': 'Started' if result_body.get('success') else 'Failed', + 'error': result_body.get('error'), + }) + except Exception as e: + executions.append({'instanceId': iid, 'bucket': bucket['signature'], 'status': 'Failed', 'error': str(e)}) + + # Store batch metadata + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, Key=f'{_BATCH_METADATA_PREFIX}{batch_id}.json', + Body=json.dumps({'batchId': batch_id, 'clusterName': cluster_name, 'region': target_region, + 'createdAt': datetime.utcnow().isoformat(), 'executions': executions, 'buckets': bucket_list}, default=str), + ContentType='application/json', + ) + except Exception: + pass + + started = sum(1 for e in executions if e['status'] == 'Started') + failed = sum(1 for e in executions if e['status'] == 'Failed') + + return success_response({ + 'batchId': batch_id, 'clusterName': cluster_name, 'region': target_region, + 'totalInstances': len(all_instances), 'filteredInstances': len(filtered), + 'bucketCount': len(bucket_list), 'buckets': bucket_list, + 'executions': executions, 'collectionsStarted': started, 'collectionsFailed': failed, + 'task': { + 'taskId': batch_id, 'state': 'running' if started > 0 else 'failed', + 'message': f'{started} collections started, {failed} failed', 'progress': 0, + }, + 'nextStep': f'Use batch_status(batchId="{batch_id}") to poll all collections.', + }) + + except Exception as e: + return error_response(500, f'batch_collect failed: {str(e)}') + + +def batch_status(arguments: Dict) -> Dict: + """ + Poll status of multiple log collections at once. + Returns consolidated view with allComplete flag. + + Inputs: + executionIds: list of SSM execution IDs (required if no batchId) + batchId: batch ID from batch_collect (alternative to executionIds) + + Returns: + allComplete, summary counts, per-execution status + """ + execution_ids = arguments.get('executionIds', []) + batch_id = arguments.get('batchId') + + # If batchId provided, load execution IDs from stored metadata + if batch_id and not execution_ids: + try: + meta_result = safe_s3_read(f'{_BATCH_METADATA_PREFIX}{batch_id}.json') + if meta_result.get('success'): + meta = json.loads(meta_result['content']) + execution_ids = [ + e['executionId'] for e in meta.get('executions', []) + if e.get('executionId') + ] + except Exception: + pass + + if not execution_ids: + return error_response(400, 'executionIds list or batchId is required') + + # Deduplicate + execution_ids = list(dict.fromkeys(execution_ids)) + + # Poll all executions in parallel + results = [] + + def _poll(eid): + try: + provenance, provenance_error = require_execution_provenance( + eid, allowed_tools={'collect'} + ) + if provenance_error: + return { + 'executionId': eid, 'instanceId': None, 'status': 'Unknown', + 'progress': 0, + 'error': json.loads(provenance_error['body']).get('error'), + } + target_region = provenance['region'] + regional_ssm = get_regional_client('ssm', target_region) + resp = regional_ssm.get_automation_execution(AutomationExecutionId=eid) + execution = resp['AutomationExecution'] + details_error = validate_execution_details(execution, provenance) + if details_error: + return { + 'executionId': eid, 'instanceId': None, 'status': 'Unknown', + 'progress': 0, + 'error': json.loads(details_error['body']).get('error'), + } + status = execution['AutomationExecutionStatus'] + instance_id = provenance.get('instanceId') + return { + 'executionId': eid, + 'instanceId': instance_id, + 'status': status, + 'progress': 100 if status == 'Success' else (0 if status == 'Failed' else estimate_progress(execution)), + 'failureReason': parse_failure_reason(execution) if status == 'Failed' else None, + } + except Exception as e: + return { + 'executionId': eid, + 'instanceId': None, + 'status': 'Unknown', + 'progress': 0, + 'error': str(e), + } + + with ThreadPoolExecutor(max_workers=min(len(execution_ids), 15)) as executor: + results = list(executor.map(_poll, execution_ids)) + + # Compute summary + succeeded = [r for r in results if r['status'] == 'Success'] + failed = [r for r in results if r['status'] == 'Failed'] + in_progress = [r for r in results if r['status'] in ('InProgress', 'Pending', 'Waiting')] + unknown = [r for r in results if r['status'] not in ('Success', 'Failed', 'InProgress', 'Pending', 'Waiting')] + + all_complete = len(in_progress) == 0 and len(unknown) == 0 + + response_data = { + 'allComplete': all_complete, + 'summary': { + 'total': len(results), + 'succeeded': len(succeeded), + 'failed': len(failed), + 'inProgress': len(in_progress), + 'unknown': len(unknown), + }, + 'executions': results, + } + + if all_complete: + ready_instances = [r['instanceId'] for r in succeeded if r['instanceId']] + failed_instances = [r['instanceId'] for r in failed if r['instanceId']] + response_data['nextStep'] = ( + f"All collections complete. {len(succeeded)} succeeded, {len(failed)} failed. " + f"Use errors/search/network_diagnostics on succeeded instances: {ready_instances[:5]}." + ) + if failed_instances: + response_data['failedInstances'] = failed_instances + else: + response_data['nextStep'] = f'{len(in_progress)} still running. Poll again in 15 seconds.' + response_data['suggestedPollIntervalSeconds'] = 15 + + return success_response(response_data) + + +def network_diagnostics(arguments: Dict) -> Dict: + """ + Extract and structure networking info from collected ECS log bundles. + Parses iptables, docker networking, routes, DNS, ENI, and security groups. + + Inputs: + instanceId: EC2 instance ID (required) + sections: comma-separated: "iptables,docker,routes,dns,eni,security-groups" or "all" (default: "all") + + Returns: + Structured networking diagnostics per section + """ + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + + sections_str = arguments.get('sections', 'all') + valid_sections = {'iptables', 'docker', 'routes', 'dns', 'eni', 'security-groups'} + if sections_str == 'all': + sections = ['iptables', 'docker', 'routes', 'dns', 'eni', 'security-groups'] + else: + sections = [s.strip() for s in sections_str.split(',')] + invalid = [s for s in sections if s not in valid_sections] + if invalid: + return error_response(400, f"Invalid section(s): {', '.join(invalid)}. Valid: {', '.join(sorted(valid_sections))}") + if not sections: + return error_response(400, 'At least one section is required') + + instance_error = validate_instance_id(instance_id) + if instance_error: + return instance_error + target_region = None + if 'eni' in sections or 'security-groups' in sections: + target_region, region_error = resolve_and_validate_region(arguments, instance_id) + if region_error: + return region_error + membership_error = validate_ecs_instance(instance_id, target_region) + if membership_error: + return membership_error + + prefix = f"logs/{instance_id}/extracted/" + results = {} + issues_found = [] + + try: + # Use shared latest-bundle discovery + bundle_info = find_latest_bundle_files(instance_id) + if not bundle_info['success']: + return error_response(404, bundle_info.get('error', f'No extracted log bundle found for {instance_id}. Run collect first.')) + + bundle_files = bundle_info['files'] + bundle_age_minutes = bundle_info['bundle_age_minutes'] + bundle_collected_at = bundle_info['bundle_collected_at'] + + # HARD BLOCK: Do NOT analyze stale bundles — force fresh collection + STALE_THRESHOLD_MINUTES = 15 + if bundle_age_minutes is not None and bundle_age_minutes > STALE_THRESHOLD_MINUTES: + return error_response(409, ( + f'STALE BUNDLE: The log bundle for {instance_id} is {bundle_age_minutes} minutes old ' + f'(collected at {bundle_collected_at}). The instance state has likely changed since then. ' + f'You MUST run the collect tool first to gather fresh logs, wait for it to complete ' + f'(poll status until success), then call network_diagnostics again. ' + f'Do NOT draw conclusions from stale data.' + ), { + 'bundleInfo': { + 'collectedAt': bundle_collected_at, + 'ageMinutes': bundle_age_minutes, + 'isStale': True, + 'staleThresholdMinutes': STALE_THRESHOLD_MINUTES, + }, + 'action': 'Run collect tool, poll status until complete, then retry network_diagnostics', + }) + + def find_files(patterns): + """Find bundle files matching any of the given patterns.""" + matched = [] + for f in bundle_files: + fname = f.lower() + for p in patterns: + if p in fname: + matched.append(f) + break + return matched + + # Pre-fetch all needed files in parallel + files_to_fetch = set() + section_file_map = {} + fetch_sizes = {} + + if 'iptables' in sections: + keys = find_files(['iptables', 'ip-tables', 'iptable'])[:6] + section_file_map['iptables'] = keys + for k in keys: files_to_fetch.add(k); fetch_sizes[k] = 262144 + if 'docker' in sections: + keys = find_files(['docker', 'containerd', 'bridge', 'docker-network', 'brctlshow', 'veth'])[:5] + section_file_map['docker'] = keys + for k in keys: files_to_fetch.add(k); fetch_sizes[k] = 262144 + if 'routes' in sections: + r_keys = find_files(['ip-route', 'ip_route', 'route-table', 'routes'])[:3] + i_keys = find_files(['ifconfig', 'ip-addr', 'ip_addr', 'interfaces', 'ipaddrshow'])[:2] + section_file_map['routes'] = r_keys + section_file_map['routes_iface'] = i_keys + for k in r_keys + i_keys: files_to_fetch.add(k); fetch_sizes[k] = 262144 + if 'dns' in sections: + keys = find_files(['resolv', 'dns'])[:5] + section_file_map['dns'] = keys + for k in keys: files_to_fetch.add(k); fetch_sizes[k] = 262144 + if 'eni' in sections: + keys = find_files(['eni', 'network-interface', 'eth'])[:3] + section_file_map['eni'] = keys + for k in keys: files_to_fetch.add(k); fetch_sizes[k] = 32768 + if 'security-groups' in sections: + section_file_map['security-groups'] = [] # Fetched via API + + # Parallel S3 reads + file_contents = {} + def _fetch(key): + r = safe_s3_read(key, max_size=fetch_sizes.get(key, 262144)) + return key, r.get('content', '') if r.get('success') else None + + with ThreadPoolExecutor(max_workers=10) as executor: + fetch_list = list(files_to_fetch) + for key, content in executor.map(_fetch, fetch_list): + file_contents[key] = content + + def read_file_content(key, max_size=262144): + cached = file_contents.get(key) + if cached is not None: + return cached + r = safe_s3_read(key, max_size=max_size) + return r.get('content', '') if r.get('success') else None + + # ================================================================= + # IPTABLES + # ================================================================= + if 'iptables' in sections: + ipt_data = {'chainCount': 0, 'ruleCount': 0, 'natRules': [], 'dockerRules': [], 'issues': []} + + # BUG FIX: Parse ALL iptables files and merge results. + # Docker DNAT/MASQUERADE rules live in the NAT table, not the filter table. + # Previously we broke after the first file (usually iptables-filter.txt), + # missing NAT table rules entirely. + ipt_files = find_files(['iptables', 'ip-tables', 'iptable']) + + all_lines_merged = [] + all_nat_rules = [] + all_docker_rules = [] + total_rule_count = 0 + total_chain_count = 0 + source_files = [] + + # Prefer iptables-save.txt if available + save_file = None + for f in ipt_files: + fl = f.lower() + if 'iptables-save' in fl or 'iptables_save' in fl: + save_file = f + break + + files_to_parse = [save_file] if save_file else ipt_files[:6] + + for f in files_to_parse: + content = read_file_content(f) + if not content: + continue + lines = content.splitlines() + source_files.append(f) + all_lines_merged.extend(lines) + + total_rule_count += sum(1 for l in lines if l.strip() and not l.startswith('#') and not l.startswith('*') and not l.startswith(':')) + total_chain_count += sum(1 for l in lines if l.startswith(':')) + all_nat_rules.extend(l.strip() for l in lines if 'DNAT' in l or 'SNAT' in l or 'MASQUERADE' in l) + all_docker_rules.extend(l.strip() for l in lines if 'DOCKER' in l) + + ipt_data['ruleCount'] = total_rule_count + ipt_data['chainCount'] = total_chain_count + ipt_data['natRules'] = all_nat_rules[:20] + ipt_data['dockerRules'] = all_docker_rules[:20] + ipt_data['sourceFiles'] = source_files + ipt_data['sourceFile'] = source_files[0] if source_files else None + + if all_lines_merged: + if total_rule_count == 0: + ipt_data['issues'].append('No iptables rules found — Docker networking may be broken') + issues_found.append({'section': 'iptables', 'severity': 'critical', 'message': 'No iptables rules found'}) + if not any('DOCKER' in l for l in all_lines_merged): + ipt_data['issues'].append('DOCKER chain missing — Docker bridge networking not configured') + issues_found.append({'section': 'iptables', 'severity': 'warning', 'message': 'DOCKER chain missing'}) + + results['iptables'] = ipt_data + + # ================================================================= + # DOCKER NETWORKING + # ================================================================= + if 'docker' in sections: + docker_data = {'networkMode': None, 'bridgeConfig': {}, 'errors': [], 'issues': []} + docker_files = find_files(['docker', 'containerd', 'bridge', 'docker-network']) + for f in docker_files[:5]: + content = read_file_content(f) + if content: + # Parse docker daemon config + if f.endswith('.json') or 'daemon' in f.lower(): + try: + cfg = json.loads(content) + docker_data['bridgeConfig'] = cfg + if 'bridge' in cfg: + docker_data['networkMode'] = 'bridge' + if cfg.get('iptables') is False: + docker_data['issues'].append('Docker iptables disabled — container networking may fail') + issues_found.append({'section': 'docker', 'severity': 'critical', 'message': 'Docker iptables disabled'}) + except json.JSONDecodeError: + pass + # Parse docker logs for network errors + for line in content.split('\n'): + ll = line.lower() + if ('error' in ll or 'failed' in ll) and ('network' in ll or 'bridge' in ll or 'eni' in ll): + docker_data['errors'].append(line.strip()[:200]) + if docker_data['errors']: + docker_data['issues'].append(f"{len(docker_data['errors'])} Docker networking errors found") + issues_found.append({'section': 'docker', 'severity': 'warning', 'message': f"{len(docker_data['errors'])} Docker network errors"}) + docker_data['errors'] = docker_data['errors'][:20] + results['docker'] = docker_data + + # ================================================================= + # ROUTE TABLES + # ================================================================= + if 'routes' in sections: + route_data = {'routes': [], 'defaultGateway': None, 'interfaces': [], 'issues': []} + route_files = find_files(['ip-route', 'ip_route', 'route-table', 'routes']) + for f in route_files[:3]: + content = read_file_content(f) + if content: + for line in content.split('\n'): + line = line.strip() + if not line: + continue + route_data['routes'].append(line) + if line.startswith('default') or 'default' in line: + route_data['defaultGateway'] = line + # Parse interfaces + iface_files = find_files(['ifconfig', 'ip-addr', 'ip_addr', 'interfaces']) + for f in iface_files[:2]: + content = read_file_content(f) + if content: + current_iface = None + for line in content.split('\n'): + if re.match(r'^\d+:\s+\S+', line) or re.match(r'^\S+:', line): + iface_match = re.search(r'(\S+?)[@:]', line) + if iface_match: + current_iface = iface_match.group(1) + if 'inet ' in line and current_iface: + ip_match = re.search(r'inet\s+(\S+)', line) + if ip_match: + route_data['interfaces'].append({'name': current_iface, 'ip': ip_match.group(1)}) + if not route_data['defaultGateway']: + route_data['issues'].append('No default gateway found') + issues_found.append({'section': 'routes', 'severity': 'critical', 'message': 'No default gateway'}) + route_data['routeCount'] = len(route_data['routes']) + route_data['routes'] = route_data['routes'][:50] + results['routes'] = route_data + + # ================================================================= + # DNS + # ================================================================= + if 'dns' in sections: + dns_data = {'resolv_conf': {}, 'nameservers': [], 'searchDomains': [], 'issues': []} + dns_files = find_files(['resolv', 'dns']) + for f in dns_files[:5]: + content = read_file_content(f) + if content: + if 'resolv' in f.lower(): + for line in content.split('\n'): + line = line.strip() + if line.startswith('nameserver'): + ns = line.split(None, 1)[1] if len(line.split()) > 1 else '' + dns_data['nameservers'].append(ns) + elif line.startswith('search'): + dns_data['searchDomains'] = line.split()[1:] + elif line.startswith('options'): + dns_data['resolv_conf']['options'] = line + dns_data['resolv_conf']['raw'] = content[:500] + if not dns_data['nameservers']: + dns_data['issues'].append('No nameservers in resolv.conf') + issues_found.append({'section': 'dns', 'severity': 'critical', 'message': 'No nameservers configured'}) + dns_data['_note'] = ( + "This is the NODE-LEVEL /etc/resolv.conf. It is expected to show VPC DNS " + "(e.g., 172.31.0.2 = VPC CIDR+2). Container DNS is configured separately " + "by Docker/ECS agent via --dns flags or task-level dnsServers config." + ) + results['dns'] = dns_data + + # ================================================================= + # ENI (Elastic Network Interfaces) + # ================================================================= + if 'eni' in sections: + eni_data = {'attachedENIs': [], 'eniCount': 0, 'issues': []} + try: + regional_ec2 = get_regional_client('ec2', target_region) + eni_resp = regional_ec2.describe_network_interfaces( + Filters=[{'Name': 'attachment.instance-id', 'Values': [instance_id]}] + ) + for eni in eni_resp.get('NetworkInterfaces', []): + eni_data['attachedENIs'].append({ + 'eniId': eni['NetworkInterfaceId'], + 'subnetId': eni.get('SubnetId'), + 'privateIp': eni.get('PrivateIpAddress'), + 'secondaryIps': [addr['PrivateIpAddress'] for addr in eni.get('PrivateIpAddresses', []) if not addr.get('Primary')], + 'status': eni.get('Status'), + 'description': eni.get('Description', '')[:100], + 'securityGroups': [sg['GroupId'] for sg in eni.get('Groups', [])], + }) + eni_data['eniCount'] = len(eni_data['attachedENIs']) + eni_data['totalSecondaryIPs'] = sum(len(e['secondaryIps']) for e in eni_data['attachedENIs']) + if eni_data['eniCount'] == 0: + eni_data['issues'].append('No ENIs attached — instance may be detached from VPC') + issues_found.append({'section': 'eni', 'severity': 'critical', 'message': 'No ENIs attached'}) + except Exception as e: + eni_data['issues'].append(f'Could not query ENI info: {str(e)}') + # Also check from bundle files + eni_files = find_files(['eni', 'network-interface', 'eth']) + for f in eni_files[:3]: + content = read_file_content(f, max_size=32768) + if content: + eni_data['bundleNetworkInfo'] = content[:2000] + break + results['eni'] = eni_data + + # ================================================================= + # SECURITY GROUPS + # ================================================================= + if 'security-groups' in sections: + sg_data = {'securityGroups': [], 'issues': []} + try: + regional_ec2 = get_regional_client('ec2', target_region) + inst_resp = regional_ec2.describe_instances(InstanceIds=[instance_id]) + sgs = [] + for res in inst_resp.get('Reservations', []): + for inst in res.get('Instances', []): + sgs = inst.get('SecurityGroups', []) + sg_ids = [sg['GroupId'] for sg in sgs] + if sg_ids: + sg_resp = regional_ec2.describe_security_groups(GroupIds=sg_ids) + for sg in sg_resp.get('SecurityGroups', []): + ingress_rules = [] + for rule in sg.get('IpPermissions', []): + for cidr in rule.get('IpRanges', []): + ingress_rules.append({ + 'protocol': rule.get('IpProtocol', 'all'), + 'fromPort': rule.get('FromPort'), + 'toPort': rule.get('ToPort'), + 'cidr': cidr.get('CidrIp'), + }) + for sg_ref in rule.get('UserIdGroupPairs', []): + ingress_rules.append({ + 'protocol': rule.get('IpProtocol', 'all'), + 'fromPort': rule.get('FromPort'), + 'toPort': rule.get('ToPort'), + 'sourceGroup': sg_ref.get('GroupId'), + }) + sg_data['securityGroups'].append({ + 'groupId': sg['GroupId'], + 'groupName': sg.get('GroupName', ''), + 'description': sg.get('Description', '')[:100], + 'ingressRuleCount': len(ingress_rules), + 'ingressRules': ingress_rules[:30], + }) + # Check for ECS-specific port requirements + all_ingress_ports = set() + for sg_info in sg_data['securityGroups']: + for rule in sg_info['ingressRules']: + fp = rule.get('fromPort') + tp = rule.get('toPort') + if fp and tp: + all_ingress_ports.update(range(fp, tp + 1)) + # ECS agent needs outbound (usually allowed by default), but check for common task ports + if not sg_ids: + sg_data['issues'].append('No security groups attached to instance') + issues_found.append({'section': 'security-groups', 'severity': 'critical', 'message': 'No security groups'}) + except Exception as e: + sg_data['issues'].append(f'Could not query security groups: {str(e)}') + results['security-groups'] = sg_data + + # ================================================================= + # ECS NETWORKING GUARDRAILS — prevent misinterpretation of findings + # ================================================================= + ecs_context = { + '_purpose': 'ECS-specific networking context to prevent misinterpretation of findings. ' + 'ECS container instances use Docker bridge/awsvpc networking which differs from ' + 'bare-metal Linux hosts. DO NOT diagnose ECS instances like standalone servers.', + 'guardrails': [], + } + + # Docker bridge networking context + docker_cfg = results.get('docker', {}).get('bridgeConfig', {}) + if docker_cfg: + ecs_context['guardrails'].append( + 'ECS EC2 launch type uses Docker bridge networking by default for tasks without ' + 'awsvpc network mode. In bridge mode, containers share the host ENI via port mappings ' + '(DNAT rules). Missing DOCKER iptables chains means Docker networking is broken, ' + 'but missing SNAT/MASQUERADE rules may be normal if tasks use awsvpc mode.' + ) + + # awsvpc mode context + eni_count = results.get('eni', {}).get('eniCount', 0) + if eni_count > 1: + ecs_context['guardrails'].append( + f'Multiple ENIs detected ({eni_count}): ECS tasks using awsvpc network mode get their own ENI. ' + 'Each task ENI has its own security group and private IP. This is NORMAL for awsvpc tasks. ' + 'Do NOT flag extra ENIs as anomalous on ECS instances running awsvpc tasks.' + ) + + # ANTI-HALLUCINATION: Docker iptables=false + if docker_cfg.get('iptables') is False: + ecs_context['guardrails'].append( + 'CRITICAL ANTI-HALLUCINATION: Docker daemon has "iptables": false in daemon.json. ' + 'This means Docker will NOT manage iptables rules for container networking. ' + 'This breaks bridge-mode container networking (no DNAT for port mappings, no MASQUERADE for egress). ' + 'However, awsvpc-mode tasks are UNAFFECTED because they use their own ENI directly. ' + 'Do NOT claim all container networking is broken — check the task network mode first.' + ) + + # ANTI-HALLUCINATION: Missing default gateway on awsvpc instances + if not results.get('routes', {}).get('defaultGateway') and eni_count > 1: + # Downgrade the "no default gateway" issue if awsvpc ENIs are present + for issue in issues_found: + if issue.get('section') == 'routes' and 'default gateway' in issue.get('message', '').lower(): + issue['severity'] = 'info' + issue['message'] += ' [May be EXPECTED: awsvpc tasks use per-task ENI routing]' + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: On ECS instances with awsvpc tasks, the host route table may appear ' + 'minimal. awsvpc tasks have their own network namespace with separate routing. ' + 'Check per-task ENI configuration before flagging host routing as broken.' + ) + + # ANTI-HALLUCINATION: ECS Agent connectivity vs container networking + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: ECS Agent connectivity issues (AGENT_DISCONNECTED) and container ' + 'networking issues are SEPARATE problems. The ECS Agent communicates with the ECS service ' + 'endpoint via HTTPS (port 443). Container networking depends on the task network mode ' + '(bridge/awsvpc/host). An agent disconnect does NOT mean container networking is broken, ' + 'and vice versa. Diagnose them independently.' + ) + + # ANTI-HALLUCINATION: Security group requirements differ by network mode + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: Security group requirements differ by ECS network mode. ' + 'Bridge mode: Only the instance security group matters — containers use host ports via DNAT. ' + 'awsvpc mode: Each task has its own security group — the instance SG does NOT apply to task traffic. ' + 'Host mode: Tasks share the instance security group directly. ' + 'Do NOT apply bridge-mode SG analysis to awsvpc tasks or vice versa.' + ) + + # ANTI-HALLUCINATION: DNS configuration + if results.get('dns', {}).get('nameservers'): + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: The node-level /etc/resolv.conf shows VPC DNS resolver ' + '(typically VPC CIDR + 2, e.g., 172.31.0.2). Container DNS is configured separately: ' + 'bridge mode uses Docker --dns flags (default: host DNS). ' + 'awsvpc mode uses the VPC DNS directly in the task network namespace. ' + 'Custom DNS can be set via task definition dnsServers/dnsSearchDomains. ' + 'Do NOT assume container DNS matches host DNS without checking the task network mode.' + ) + + # ANTI-HALLUCINATION: Conntrack exhaustion is kernel-level + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: "nf_conntrack: table full" is a kernel resource exhaustion issue. ' + 'Do NOT blame Docker or the ECS Agent. Fix: increase nf_conntrack_max via sysctl. ' + 'High-traffic instances (running many tasks with bridge networking) are most susceptible ' + 'because all containers share the host conntrack table in bridge mode.' + ) + + # ANTI-HALLUCINATION: ENI limits and task density + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: Each EC2 instance type has a maximum number of ENIs and IPs per ENI. ' + 'awsvpc tasks each consume one ENI (or one IP with ENI trunking enabled). ' + 'If ENI allocation fails, it may be an instance limit, NOT a subnet IP exhaustion issue. ' + 'Check the instance type ENI limit before blaming the subnet. ' + 'ENI trunking (account-level opt-in) allows more tasks per instance by sharing trunk ENIs.' + ) + + # ANTI-HALLUCINATION: ECS service connect vs service discovery + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: ECS Service Connect and ECS Service Discovery (Cloud Map) are ' + 'DIFFERENT features. Service Connect uses an Envoy sidecar proxy for service mesh. ' + 'Service Discovery uses Route 53 DNS (A/SRV records). They can coexist but have ' + 'different failure modes. DNS failures may be Service Discovery issues, while ' + 'connection proxy errors point to Service Connect. Do NOT conflate them.' + ) + + # ANTI-HALLUCINATION: Container health checks vs ELB health checks + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: ECS container health checks (HEALTHCHECK in Dockerfile or ' + 'healthCheck in task definition) and ELB target health checks are INDEPENDENT. ' + 'A container can be healthy per its own health check but unhealthy per the ALB/NLB. ' + 'Common cause: security group not allowing health check traffic from the load balancer. ' + 'Diagnose each health check type separately.' + ) + + results['ecsContext'] = ecs_context + + # ================================================================= + # OVERALL SUMMARY + # ================================================================= + total_issues = len(issues_found) + critical_issues = sum(1 for i in issues_found if i.get('severity') == 'critical') + warning_issues = sum(1 for i in issues_found if i.get('severity') == 'warning') + + sections_with_data = sum(1 for s in sections if s in results and results[s]) + if sections_with_data >= 4 and critical_issues > 0: + confidence = 'high' + elif sections_with_data >= 2 and total_issues > 0: + confidence = 'medium' + elif sections_with_data >= 1: + confidence = 'low' + else: + confidence = 'none' + + gaps = [] + if not bundle_files: + gaps.append('No extracted bundle found — collect and wait for completion first') + sections_without_files = [s for s in sections if s not in section_file_map or not section_file_map.get(s)] + if sections_without_files: + gaps.append(f'No files found for sections: {", ".join(sections_without_files)}') + + # Build response (stale bundles are already rejected above, so this is always fresh) + response = { + 'instanceId': instance_id, + } + + # Bundle freshness info (always fresh since stale bundles are hard-blocked) + if bundle_age_minutes is not None: + response['bundleInfo'] = { + 'collectedAt': bundle_collected_at, + 'ageMinutes': bundle_age_minutes, + 'isStale': False, + } + + response['sections'] = sections + response['diagnostics'] = results + response['issuesSummary'] = { + 'total': total_issues, + 'critical': critical_issues, + 'warning': warning_issues, + 'issues': issues_found, + } + response['confidence'] = confidence + response['gaps'] = gaps + response['overallAssessment'] = _network_assessment(issues_found) + response['nextStep'] = 'Use search tool to dig deeper into specific networking errors, or correlate to build a timeline.' if issues_found else 'No networking issues detected in the bundle.' + response['recommendedSOPs'] = match_sops_for_issues(issues_found) + + return success_response(response) + + except Exception as e: + return error_response(500, f'network_diagnostics failed: {str(e)}') + + +def _network_assessment(issues: List[Dict]) -> str: + """Generate overall network health assessment.""" + if not issues: + return "HEALTHY — No networking issues detected in the log bundle." + critical = [i for i in issues if i.get('severity') == 'critical'] + if critical: + sections = set(i['section'] for i in critical) + return f"CRITICAL — {len(critical)} critical networking issues in: {', '.join(sections)}. Immediate investigation needed." + return f"WARNING — {len(issues)} non-critical networking issues found. Review recommended." + +def tcpdump_capture(arguments: Dict) -> Dict: + """ + Run tcpdump on an ECS container instance via SSM Run Command for a specified duration, + then upload the pcap file to S3. + + Inputs: + instanceId: EC2 instance ID (required) + durationSeconds: Capture duration in seconds (default: 120, max: 300) + interface: Network interface to capture on (default: "any") + filter: BPF filter expression (e.g., "port 443", "host 10.0.0.1") (optional) + taskId: ECS task ID to capture traffic for a specific container's network namespace (optional) + containerName: Container name within the task (optional, uses first container if omitted) + region: AWS region where the instance runs (optional, auto-detected) + commandId: If provided, polls status of an existing capture instead of starting a new one + + Returns: + commandId for async polling, or capture results if already complete + """ + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + + if not re.match(r'^i-[0-9a-f]{8,17}$', instance_id): + return error_response(400, f'Invalid instanceId format: {instance_id}') + + duration = int(arguments.get('durationSeconds', 120)) + if duration < 10 or duration > 300: + return error_response(400, 'durationSeconds must be between 10 and 300') + + interface = arguments.get('interface', 'any') + if not re.match(r'^[a-zA-Z0-9\-\.]+$', interface): + return error_response(400, f'Invalid interface name: {interface}') + + bpf_filter = arguments.get('filter', '') + if bpf_filter and re.search(r'[;&|`$(){}]', bpf_filter): + return error_response(400, 'filter contains invalid characters') + + ecs_task_id = arguments.get('taskId', '').strip() + container_name = arguments.get('containerName', '').strip() + + # Check if this is a status poll for an existing command + command_id = arguments.get('commandId') + if command_id: + return _poll_tcpdump_status(command_id, instance_id, arguments) + + target_region, region_error = resolve_and_validate_region(arguments, instance_id) + if region_error: + return region_error + instance_error = validate_ecs_instance(instance_id, target_region) + if instance_error: + return instance_error + + try: + regional_ssm = get_regional_client('ssm', target_region) + except Exception as e: + return error_response(500, f'Failed to create SSM client for region {target_region}: {str(e)}') + + # Build the shell script that runs tcpdump and uploads to S3 + timestamp = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') + s3_prefix = f"tcpdump/{instance_id}/{timestamp}" + s3_key = f"{s3_prefix}/capture.pcap" + s3_key_txt = f"{s3_prefix}/capture_summary.txt" + s3_key_stats = f"{s3_prefix}/capture_stats.json" + s3_uri = f"s3://{LOGS_BUCKET}/{s3_key}" + s3_uri_txt = f"s3://{LOGS_BUCKET}/{s3_key_txt}" + s3_uri_stats = f"s3://{LOGS_BUCKET}/{s3_key_stats}" + + filter_clause = f' {bpf_filter}' if bpf_filter else '' + + use_nsenter = bool(ecs_task_id) + ns_label = '' + if ecs_task_id: + ns_label = f' (ECS task {ecs_task_id}' + (f' container {container_name}' if container_name else '') + ' namespace)' + + script = f"""#!/bin/bash +set -euo pipefail + +PCAP_FILE="/tmp/tcpdump_capture_{timestamp}.pcap" +TXT_FILE="/tmp/tcpdump_summary_{timestamp}.txt" +STATS_FILE="/tmp/tcpdump_stats_{timestamp}.json" + +# Check if tcpdump is available +if ! command -v tcpdump &>/dev/null; then + echo "ERROR: tcpdump not found. Installing..." + if command -v yum &>/dev/null; then + yum install -y tcpdump 2>/dev/null || {{ echo "FATAL: Failed to install tcpdump"; exit 1; }} + elif command -v apt-get &>/dev/null; then + apt-get update -qq && apt-get install -y tcpdump 2>/dev/null || {{ echo "FATAL: Failed to install tcpdump"; exit 1; }} + else + echo "FATAL: No package manager found to install tcpdump" + exit 1 + fi +fi + +NSENTER_PREFIX="" +""" + + # Add ECS task container PID resolution when taskId is provided + if ecs_task_id: + container_filter = f'--filter "name={container_name}"' if container_name else '' + script += f""" +# === Resolve ECS task "{ecs_task_id}" to container PID === +echo "Resolving ECS task to container PID..." +TARGET_PID="" + +# Method 1: ECS agent introspection endpoint (works on all ECS-optimized AMIs) +echo "Trying ECS agent introspection endpoint..." +TASK_META=$(curl -s http://localhost:51678/v1/tasks 2>/dev/null || true) +if [ -n "$TASK_META" ] && command -v python3 &>/dev/null; then + TARGET_DOCKER_ID=$(python3 -c " +import sys, json +try: + data = json.loads('''$TASK_META''') + tasks = data if isinstance(data, list) else data.get('Tasks', []) + for task in tasks: + task_arn = task.get('Arn', '') + # Match by full ARN or just the task ID suffix + if '{ecs_task_id}' in task_arn: + containers = task.get('Containers', []) + for c in containers: + container_name_filter = '{container_name}' + if container_name_filter and c.get('Name') != container_name_filter: + continue + docker_id = c.get('DockerId', '') + if docker_id: + print(docker_id) + sys.exit(0) + # If no container_name filter matched, take the first container + if containers: + docker_id = containers[0].get('DockerId', '') + if docker_id: + print(docker_id) + sys.exit(0) +except Exception as e: + print('', file=sys.stderr) +" 2>/dev/null || true) + if [ -n "$TARGET_DOCKER_ID" ]; then + echo "Found Docker container ID from ECS introspection: $TARGET_DOCKER_ID" + fi +fi + +# Method 2: docker ps with ECS task label +if [ -z "$TARGET_DOCKER_ID" ] && command -v docker &>/dev/null; then + echo "Trying docker ps with task label..." + TARGET_DOCKER_ID=$(docker ps --filter "label=com.amazonaws.ecs.task-arn" {container_filter} --format '{{{{.ID}}}} {{{{.Labels}}}}' 2>/dev/null | grep '{ecs_task_id}' | head -1 | awk '{{print $1}}') + if [ -z "$TARGET_DOCKER_ID" ]; then + # Broader search + TARGET_DOCKER_ID=$(docker ps --format '{{{{.ID}}}} {{{{.Labels}}}}' 2>/dev/null | grep '{ecs_task_id}' | head -1 | awk '{{print $1}}') + fi + if [ -n "$TARGET_DOCKER_ID" ]; then + echo "Found Docker container ID from docker ps: $TARGET_DOCKER_ID" + fi +fi + +# Get PID from Docker container ID +if [ -n "$TARGET_DOCKER_ID" ] && command -v docker &>/dev/null; then + TARGET_PID=$(docker inspect --format '{{{{.State.Pid}}}}' "$TARGET_DOCKER_ID" 2>/dev/null || true) + echo "Docker container $TARGET_DOCKER_ID -> PID $TARGET_PID" +fi + +# Method 3: ctr (containerd native CLI — for newer ECS AMIs using containerd without docker) +if [ -z "$TARGET_PID" ] || [ "$TARGET_PID" = "0" ]; then + CTR="" + for p in /usr/local/bin/ctr /usr/bin/ctr $(which ctr 2>/dev/null); do + if [ -x "$p" ]; then CTR="$p"; break; fi + done + if [ -n "$CTR" ]; then + echo "Trying ctr ($CTR) to find ECS task container..." + # ECS uses the 'moby' namespace in containerd (or 'default') + # ctr containers ls does NOT show task IDs — must inspect each container's labels + for NS in moby default; do + for cid in $($CTR -n $NS containers ls -q 2>/dev/null); do + INFO=$($CTR -n $NS containers info "$cid" 2>/dev/null || true) + if echo "$INFO" | grep -q "{ecs_task_id}"; then + echo "ctr: found container $cid in namespace $NS" + CTR_PID=$($CTR -n $NS task ls 2>/dev/null | grep "$cid" | awk '{{print $2}}') + if [ -n "$CTR_PID" ] && [ "$CTR_PID" != "0" ] && [ -e "/proc/$CTR_PID/ns/net" ]; then + TARGET_PID="$CTR_PID" + echo "ctr: resolved pid=$TARGET_PID" + break 2 + fi + fi + done + done + [ -z "$TARGET_PID" ] && echo "ctr: could not resolve ECS task '{ecs_task_id}'" + fi +fi + +if [ -z "$TARGET_PID" ] || [ "$TARGET_PID" = "0" ]; then + echo "FATAL: Could not resolve ECS task {ecs_task_id} to a container PID on this instance." + echo "Ensure the task is running on instance {instance_id}." + echo "" + echo "Debug info:" + echo " ECS agent introspection:" + curl -s http://localhost:51678/v1/tasks 2>/dev/null | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) + tasks = d if isinstance(d, list) else d.get('Tasks', []) + for t in tasks[:10]: + print(f' Task: {{t.get(\"Arn\",\"?\")}} Status: {{t.get(\"DesiredStatus\",\"?\")}}') +except: print(' (could not parse)') +" 2>/dev/null || echo " (endpoint not available)" + echo " Running docker containers:" + docker ps --format 'table {{{{.ID}}}}\\t{{{{.Names}}}}\\t{{{{.Status}}}}' 2>/dev/null | head -10 || echo " (docker not available)" + echo " ctr binary: ${{CTR:-not found}}" + echo " containerd socket: $(ls -la /run/containerd/containerd.sock 2>/dev/null || echo 'not found')" + exit 1 +fi + +echo "Resolved ECS task {ecs_task_id} -> PID $TARGET_PID" +if [ ! -e "/proc/$TARGET_PID/ns/net" ]; then + if [ ! -d "/proc/$TARGET_PID" ]; then + echo "FATAL: PID $TARGET_PID does not exist in /proc (container may have exited)" + else + echo "FATAL: PID $TARGET_PID exists but /proc/$TARGET_PID/ns/net is missing" + fi + exit 1 +fi +NSENTER_PREFIX="nsenter -n -t $TARGET_PID " +""" + + script += rf""" +echo "Starting tcpdump{ns_label} on interface '{interface}' for {duration}s..." +echo "Filter: '{bpf_filter or 'none'}'" +echo "Output: $PCAP_FILE" + +# Run tcpdump with timeout (with optional nsenter) +timeout {duration} ${{NSENTER_PREFIX}}tcpdump -i {interface} -w "$PCAP_FILE" -c 100000{filter_clause} 2>&1 || true + +# Verify capture file exists and has data +if [ ! -f "$PCAP_FILE" ]; then + echo "FATAL: Capture file not created" + exit 1 +fi + +FILE_SIZE=$(stat -c%s "$PCAP_FILE" 2>/dev/null || stat -f%z "$PCAP_FILE" 2>/dev/null || echo "0") +echo "Capture complete. File size: $FILE_SIZE bytes" + +if [ "$FILE_SIZE" -eq 0 ]; then + echo "WARNING: Capture file is empty — no packets matched the filter" +fi + +# Decode pcap to human-readable text summary (first 5000 packets max) +echo "Decoding pcap to text summary..." +# Use -c 5000 instead of piping through head to avoid SIGPIPE under set -euo pipefail +tcpdump -nn -r "$PCAP_FILE" -c 5000 > "$TXT_FILE" 2>/dev/null || true +TXT_SIZE=$(stat -c%s "$TXT_FILE" 2>/dev/null || stat -f%z "$TXT_FILE" 2>/dev/null || echo "0") +PACKET_COUNT=$(wc -l < "$TXT_FILE" 2>/dev/null || echo "0") +echo "Decoded $PACKET_COUNT packets to text (txt_size=$TXT_SIZE)" + +# If decode produced empty output, log diagnostics and retry +if [ "$TXT_SIZE" -eq 0 ] || [ "$PACKET_COUNT" -eq 0 ]; then + echo "WARNING: Text decode produced empty output." + echo "Pcap file details:" + ls -la "$PCAP_FILE" 2>/dev/null || true + file "$PCAP_FILE" 2>/dev/null || true + # Retry with verbose stderr to diagnose + echo "Retry with stderr:" + tcpdump -nn -r "$PCAP_FILE" -c 10 2>&1 || true +fi + +# Generate stats JSON with protocol breakdown and top talkers (using Python for valid JSON) +echo "Generating capture statistics..." +if command -v python3 &>/dev/null; then + python3 - "$PCAP_FILE" "$STATS_FILE" << 'PYSTATS' +import subprocess, json, sys, re +from collections import Counter + +pcap, out = sys.argv[1], sys.argv[2] + +def tcpdump_count(extra_args=None): + cmd = ['tcpdump', '-nn', '-r', pcap] + (extra_args or []) + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + return len([l for l in r.stdout.strip().splitlines() if l]) + except Exception: + return 0 + +def tcpdump_lines(): + try: + r = subprocess.run(['tcpdump', '-nn', '-r', pcap], capture_output=True, text=True, timeout=60) + return [l for l in r.stdout.strip().splitlines() if l] + except Exception: + return [] + +lines = tcpdump_lines() +total = len(lines) + +ip_port_re = re.compile(r'^(\\d+\\.\\d+\\.\\d+\\.\\d+)\\.\\d+$') +src_counter, dst_counter = Counter(), Counter() +for line in lines: + parts = line.split() + if len(parts) >= 5: + m = ip_port_re.match(parts[2]) + if m: + src_counter[m.group(1)] += 1 + dst_raw = parts[4].rstrip(':') + m = ip_port_re.match(dst_raw) + if m: + dst_counter[m.group(1)] += 1 + +retrans = sum(1 for l in lines if 'retransmit' in l.lower() or 'retrans' in l.lower()) + +stats = {{ + "totalPackets": total, + "protocols": {{ + "tcp": tcpdump_count(['tcp']), + "udp": tcpdump_count(['udp']), + "icmp": tcpdump_count(['icmp']), + "arp": tcpdump_count(['arp']), + }}, + "ports": {{ + "dns_53": tcpdump_count(['port', '53']), + "http_80": tcpdump_count(['port', '80']), + "https_443": tcpdump_count(['port', '443']), + }}, + "tcpFlags": {{ + "syn": tcpdump_count(['tcp[tcpflags] & (tcp-syn) != 0']), + "rst": tcpdump_count(['tcp[tcpflags] & (tcp-rst) != 0']), + }}, + "possibleRetransmits": retrans, + "topSourceIPs": dict(src_counter.most_common(10)), + "topDestinationIPs": dict(dst_counter.most_common(10)), +}} + +with open(out, 'w') as f: + json.dump(stats, f, indent=2) +print(f"Stats generated: {{total}} packets") +PYSTATS +else + TOTAL=$(tcpdump -nn -r "$PCAP_FILE" 2>/dev/null | wc -l) + TCP_COUNT=$(tcpdump -nn -r "$PCAP_FILE" tcp 2>/dev/null | wc -l) + UDP_COUNT=$(tcpdump -nn -r "$PCAP_FILE" udp 2>/dev/null | wc -l) + ICMP_COUNT=$(tcpdump -nn -r "$PCAP_FILE" icmp 2>/dev/null | wc -l) + echo '{{"totalPackets":'$TOTAL',"protocols":{{"tcp":'$TCP_COUNT',"udp":'$UDP_COUNT',"icmp":'$ICMP_COUNT'}},"topSourceIPs":{{}},"topDestinationIPs":{{}}}}' > "$STATS_FILE" +fi +if [ ! -f "$STATS_FILE" ] || [ ! -s "$STATS_FILE" ]; then + echo '{{"error":"stats generation failed"}}' > "$STATS_FILE" +fi + +# Upload all artifacts to S3 +set +e +UPLOAD_FAILURES=0 + +echo "Uploading pcap to {s3_uri}..." +aws s3 cp "$PCAP_FILE" "{s3_uri}" --no-progress 2>&1 +if [ $? -eq 0 ]; then echo "UPLOAD_PCAP=ok"; else echo "UPLOAD_PCAP=failed"; UPLOAD_FAILURES=$((UPLOAD_FAILURES + 1)); fi + +echo "Uploading text summary to {s3_uri_txt}..." +aws s3 cp "$TXT_FILE" "{s3_uri_txt}" --quiet 2>&1 +if [ $? -eq 0 ]; then echo "UPLOAD_TXT=ok"; else echo "UPLOAD_TXT=failed"; UPLOAD_FAILURES=$((UPLOAD_FAILURES + 1)); fi + +echo "Uploading stats to {s3_uri_stats}..." +aws s3 cp "$STATS_FILE" "{s3_uri_stats}" --quiet 2>&1 +if [ $? -eq 0 ]; then echo "UPLOAD_STATS=ok"; else echo "UPLOAD_STATS=failed"; UPLOAD_FAILURES=$((UPLOAD_FAILURES + 1)); fi + +if [ "$UPLOAD_FAILURES" -gt 0 ]; then + echo "WARNING: $UPLOAD_FAILURES of 3 uploads failed. Ensure the instance IAM role has s3:PutObject permission to {LOGS_BUCKET}." +fi + +echo "S3_KEY={s3_key}" +echo "S3_KEY_TXT={s3_key_txt}" +echo "S3_KEY_STATS={s3_key_stats}" +echo "FILE_SIZE=$FILE_SIZE" +echo "PACKET_COUNT=$PACKET_COUNT" +echo "UPLOAD_FAILURES=$UPLOAD_FAILURES" + +# Inline the decoded text and stats in stdout so Lambda can parse them even if S3 upload failed +echo "===INLINE_STATS_BEGIN===" +cat "$STATS_FILE" 2>/dev/null || echo '{{"error":"stats file missing"}}' +echo "" +echo "===INLINE_STATS_END===" +echo "===INLINE_TXT_BEGIN===" +head -500 "$TXT_FILE" 2>/dev/null || echo "(no decoded text)" +echo "" +echo "===INLINE_TXT_END===" + +# Cleanup +rm -f "$PCAP_FILE" "$TXT_FILE" "$STATS_FILE" 2>/dev/null || true +echo "DONE" +exit 0 +""" + + try: + response = regional_ssm.send_command( + InstanceIds=[instance_id], + DocumentName='AWS-RunShellScript', + Parameters={ + 'commands': [script], + 'executionTimeout': [str(duration + 120)], + }, + TimeoutSeconds=duration + 180, + Comment=f'tcpdump capture for ECS instance {instance_id} ({duration}s)', + ) + + cmd_id = response['Command']['CommandId'] + + # Store metadata for status polling + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, + Key=_tcpdump_command_metadata_key(cmd_id), + Body=json.dumps({ + 'commandId': cmd_id, + 'instanceId': instance_id, + 'region': target_region, + 's3Key': s3_key, + 's3KeyTxt': s3_key_txt, + 's3KeyStats': s3_key_stats, + 's3Prefix': s3_prefix, + 'durationSeconds': duration, + 'interface': interface, + 'filter': bpf_filter, + 'taskId': ecs_task_id or None, + 'containerName': container_name or None, + 'startedAt': timestamp, + }), + ) + except Exception: + pass # Non-fatal + + return success_response({ + 'message': f'tcpdump capture started ({duration}s){ns_label}', + 'commandId': cmd_id, + 'instanceId': instance_id, + 'region': target_region, + 'durationSeconds': duration, + 'interface': interface, + 'filter': bpf_filter or 'none', + 'taskId': ecs_task_id or None, + 'containerName': container_name or None, + 's3Key': s3_key, + 's3KeyTxt': s3_key_txt, + 's3KeyStats': s3_key_stats, + 's3Bucket': LOGS_BUCKET, + 'estimatedCompletionSeconds': duration + 30, + 'nextStep': f'Poll with tcpdump_capture(commandId="{cmd_id}", instanceId="{instance_id}") after ~{duration + 30}s. Once complete, use tcpdump_analyze(instanceId="{instance_id}", commandId="{cmd_id}") to read the decoded packet summary.', + 'task': { + 'taskId': cmd_id, + 'state': 'running', + 'message': f'tcpdump running for {duration}s on {interface}', + 'progress': 0, + }, + }) + + except Exception as e: + return error_response(500, f'Failed to start tcpdump: {str(e)}') + + +def _poll_tcpdump_status(command_id: str, instance_id: str, arguments: Dict) -> Dict: + """Poll the status of a tcpdump SSM Run Command.""" + + # Try to load stored metadata + metadata = {} + try: + meta_resp = s3_client.get_object( + Bucket=LOGS_BUCKET, + Key=_tcpdump_command_metadata_key(command_id), + ) + metadata = json.loads(meta_resp['Body'].read().decode('utf-8')) + except Exception: + pass + + target_region = metadata.get('region') or resolve_region(arguments, instance_id) + region_error = validate_region(target_region) + if region_error: + return region_error + + try: + regional_ssm = get_regional_client('ssm', target_region) + result = regional_ssm.get_command_invocation( + CommandId=command_id, + InstanceId=instance_id, + ) + + status = result.get('Status', 'Unknown') + stdout = result.get('StandardOutputContent', '') + stderr = result.get('StandardErrorContent', '') + + # Parse output for S3 key and file size + s3_key = metadata.get('s3Key', '') + s3_key_txt = metadata.get('s3KeyTxt', '') + s3_key_stats = metadata.get('s3KeyStats', '') + file_size = 0 + packet_count = 0 + for line in stdout.split('\n'): + if line.startswith('S3_KEY='): + s3_key = line.split('=', 1)[1].strip() + elif line.startswith('S3_KEY_TXT='): + s3_key_txt = line.split('=', 1)[1].strip() + elif line.startswith('S3_KEY_STATS='): + s3_key_stats = line.split('=', 1)[1].strip() + elif line.startswith('FILE_SIZE='): + try: file_size = int(line.split('=', 1)[1].strip()) + except ValueError: pass + elif line.startswith('PACKET_COUNT='): + try: packet_count = int(line.split('=', 1)[1].strip()) + except ValueError: pass + + capture_completed = ( + 'Capture complete.' in stdout + or 'DONE' in stdout + or 'UPLOAD_PCAP=ok' in stdout + or 'UPLOAD_PCAP=failed' in stdout + or ('FILE_SIZE=' in stdout and 'S3_KEY=' in stdout) + ) + if 'FATAL:' in stdout: + capture_completed = False + + upload_failures = 0 + for line in stdout.split('\n'): + if line.startswith('UPLOAD_FAILURES='): + try: upload_failures = int(line.split('=', 1)[1].strip()) + except ValueError: pass + + # Extract inline stats and text from stdout + inline_stats = {} + inline_txt_lines = [] + if '===INLINE_STATS_BEGIN===' in stdout: + try: + stats_block = stdout.split('===INLINE_STATS_BEGIN===')[1].split('===INLINE_STATS_END===')[0].strip() + if stats_block: + inline_stats = json.loads(stats_block) + except (IndexError, json.JSONDecodeError): + pass + if '===INLINE_TXT_BEGIN===' in stdout: + try: + txt_block = stdout.split('===INLINE_TXT_BEGIN===')[1].split('===INLINE_TXT_END===')[0].strip() + if txt_block: + inline_txt_lines = txt_block.split('\n') + except IndexError: + pass + + if status in ('Success',) or (capture_completed and status == 'Failed'): + presigned_url = '' + try: + presigned_url = s3_client.generate_presigned_url( + 'get_object', + Params={'Bucket': LOGS_BUCKET, 'Key': s3_key}, + ExpiresIn=PCAP_PRESIGNED_URL_EXPIRATION, + ) + except Exception: + pass + + # If S3 uploads failed from node, store inline data from Lambda + if (upload_failures > 0 or (status == 'Failed' and capture_completed)) and inline_stats: + try: + s3_client.put_object(Bucket=LOGS_BUCKET, Key=s3_key_stats, + Body=json.dumps(inline_stats, indent=2), ContentType='application/json') + except Exception: + pass + if (upload_failures > 0 or (status == 'Failed' and capture_completed)) and inline_txt_lines: + try: + s3_client.put_object(Bucket=LOGS_BUCKET, Key=s3_key_txt, + Body='\n'.join(inline_txt_lines), ContentType='text/plain') + except Exception: + pass + + warnings = [] + actual_failures = 0 + if upload_failures > 0 or (status == 'Failed' and capture_completed): + actual_failures = upload_failures if upload_failures > 0 else 3 + warnings.append(f'{actual_failures} of 3 S3 uploads failed from the node. Stats and text recovered via Lambda.') + if actual_failures == 3: + warnings.append('pcap file was NOT uploaded — add S3 PutObject permission to the instance IAM role.') + + response_data = { + 'commandId': command_id, + 'instanceId': instance_id, + 'status': 'completed' if not warnings else 'completed_with_warnings', + 's3Key': s3_key, 's3KeyTxt': s3_key_txt, 's3KeyStats': s3_key_stats, + 's3Bucket': LOGS_BUCKET, + 'fileSizeBytes': file_size, 'fileSizeHuman': format_bytes(file_size), + 'packetCount': packet_count, + 'presignedUrl': presigned_url, 'presignedUrlExpiresIn': '1 hour', + 'output': stdout[-2000:] if len(stdout) > 2000 else stdout, + 'nextStep': f'Use tcpdump_analyze(instanceId="{instance_id}", commandId="{command_id}") to read decoded packet data and statistics.', + 'task': {'taskId': command_id, 'state': 'completed', + 'message': f'tcpdump capture completed' + (f' ({actual_failures} S3 uploads failed — recovered via Lambda)' if warnings else f' — uploaded to s3://{LOGS_BUCKET}/{s3_key}'), + 'progress': 100}, + } + if warnings: + response_data['warnings'] = warnings + if inline_stats: + response_data['inlineStats'] = inline_stats + return success_response(response_data) + + elif status in ('InProgress', 'Pending', 'Delayed'): + elapsed = 0 + duration = metadata.get('durationSeconds', 120) + if metadata.get('startedAt'): + for timestamp_format in ('%Y%m%dT%H%M%SZ', '%Y-%m-%dT%H:%M:%SZ'): + try: + start_dt = datetime.strptime(metadata['startedAt'], timestamp_format) + elapsed = (datetime.utcnow() - start_dt).total_seconds() + break + except (TypeError, ValueError): + continue + progress = min(95, int((elapsed / (duration + 30)) * 100)) if duration else 0 + + return success_response({ + 'commandId': command_id, + 'instanceId': instance_id, + 'status': 'in_progress', + 'elapsedSeconds': int(elapsed), + 'durationSeconds': duration, + 'nextStep': 'Poll again in 15-30 seconds', + 'task': {'taskId': command_id, 'state': 'running', + 'message': f'tcpdump capture in progress ({int(elapsed)}s / {duration}s)', + 'progress': progress}, + }) + + else: + return error_response(500, f'tcpdump command {status}', { + 'commandId': command_id, 'status': status, + 'stdout': stdout[-2000:] if stdout else '', + 'stderr': stderr[-2000:] if stderr else '', + 'statusDetails': result.get('StatusDetails', ''), + 'task': {'taskId': command_id, 'state': 'failed', + 'message': f'tcpdump command {status}: {stderr[:200] if stderr else "unknown error"}', + 'progress': 0}, + }) + + except Exception as e: + return error_response(500, f'Failed to poll tcpdump status: {str(e)}') + + +def tcpdump_analyze(arguments: Dict) -> Dict: + """ + Read and analyze a completed tcpdump capture from S3. + Returns decoded packet text, protocol statistics, and top talkers. + + Inputs: + instanceId: EC2 instance ID (required) + commandId: SSM Command ID from tcpdump_capture (required) + section: "summary" (decoded packets), "stats" (protocol breakdown), "all" (default) + maxPackets: Max decoded packet lines to return (default: 500, max: 3000) + filter: Text filter to apply on decoded lines (e.g., "SYN", "RST", "10.0.0.5") + + Returns: + Decoded packet text, protocol stats, top talkers, and anomaly indicators + """ + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + + command_id = arguments.get('commandId') + if not command_id: + return error_response(400, 'commandId is required') + if not isinstance(command_id, str) or not _COMMAND_ID_RE.fullmatch(command_id): + return error_response(400, 'commandId must be a lowercase UUID') + section = arguments.get('section', 'all') + max_packets = min(int(arguments.get('maxPackets', 500)), 3000) + text_filter = arguments.get('filter', '') + + try: + meta_resp = s3_client.get_object( + Bucket=LOGS_BUCKET, + Key=_tcpdump_command_metadata_key(command_id), + ) + metadata = json.loads(meta_resp['Body'].read().decode('utf-8')) + except Exception: + return error_response( + 404, + f'Metadata for commandId {command_id} was not found; refusing latest-capture fallback.', + ) + + s3_key_txt = metadata.get('s3KeyTxt', '') + s3_key_stats = metadata.get('s3KeyStats', '') + s3_key_pcap = metadata.get('s3Key', '') + + results = { + 'instanceId': instance_id, + 'commandId': metadata.get('commandId', command_id or 'unknown'), + 'captureInfo': { + 'interface': metadata.get('interface', 'unknown'), + 'filter': metadata.get('filter', 'none'), + 'durationSeconds': metadata.get('durationSeconds', 0), + 'startedAt': metadata.get('startedAt', 'unknown'), + 'taskId': metadata.get('taskId'), + 'containerName': metadata.get('containerName'), + }, + } + + # Read stats + if section in ('stats', 'all'): + stats = {} + if s3_key_stats: + try: + raw = safe_s3_read_raw(LOGS_BUCKET, s3_key_stats) + if raw: + stats = json.loads(raw.decode('utf-8') if isinstance(raw, bytes) else raw) + except (json.JSONDecodeError, Exception): + stats = {'error': 'Could not parse stats JSON'} + else: + stats = {'error': 'No stats file found — capture may still be in progress'} + + results['statistics'] = stats + + # Anomaly detection + anomalies = [] + if isinstance(stats, dict) and 'totalPackets' in stats: + total = stats.get('totalPackets', 0) + rst_count = stats.get('tcpFlags', {}).get('rst', 0) + syn_count = stats.get('tcpFlags', {}).get('syn', 0) + retrans = stats.get('possibleRetransmits', 0) + + if total > 0: + rst_pct = (rst_count / total) * 100 + if rst_pct > 5: + anomalies.append({ + 'type': 'high_rst_rate', + 'severity': 'warning' if rst_pct < 15 else 'critical', + 'message': f'{rst_pct:.1f}% of packets are TCP RST ({rst_count}/{total}) — possible connection rejection or firewall drops', + }) + if retrans > 0: + retrans_pct = (retrans / total) * 100 + anomalies.append({ + 'type': 'retransmissions', + 'severity': 'warning' if retrans_pct < 5 else 'critical', + 'message': f'{retrans} possible retransmissions ({retrans_pct:.1f}%) — network congestion or packet loss', + }) + if syn_count > 0 and rst_count > syn_count * 0.5: + anomalies.append({ + 'type': 'syn_rst_ratio', + 'severity': 'warning', + 'message': f'High RST-to-SYN ratio ({rst_count} RST vs {syn_count} SYN) — many connections being refused', + }) + icmp_count = stats.get('protocols', {}).get('icmp', 0) + if icmp_count > total * 0.1: + anomalies.append({ + 'type': 'high_icmp', + 'severity': 'info', + 'message': f'{icmp_count} ICMP packets ({(icmp_count/total)*100:.1f}%) — possible ping flood or unreachable destinations', + }) + results['anomalies'] = anomalies + + # Read decoded text summary + if section in ('summary', 'all'): + decoded_lines = [] + if s3_key_txt: + try: + raw = safe_s3_read_raw(LOGS_BUCKET, s3_key_txt) + if raw: + content = raw.decode('utf-8') if isinstance(raw, bytes) else raw + all_lines = content.split('\n') + + if text_filter: + pattern = re.compile(re.escape(text_filter), re.IGNORECASE) + all_lines = [l for l in all_lines if pattern.search(l)] + + total_lines = len(all_lines) + decoded_lines = all_lines[:max_packets] + + results['decodedPackets'] = { + 'lines': decoded_lines, + 'totalPackets': total_lines, + 'returnedPackets': len(decoded_lines), + 'truncated': total_lines > max_packets, + 'filter': text_filter or 'none', + } + else: + results['decodedPackets'] = {'error': 'Text summary file is empty or unreadable'} + except Exception as e: + results['decodedPackets'] = {'error': f'Failed to read text summary: {str(e)}'} + else: + results['decodedPackets'] = {'error': 'No text summary file found — capture may still be in progress'} + + # Presigned URL for pcap download + if s3_key_pcap: + try: + results['pcapDownloadUrl'] = s3_client.generate_presigned_url( + 'get_object', + Params={'Bucket': LOGS_BUCKET, 'Key': s3_key_pcap}, + ExpiresIn=PCAP_PRESIGNED_URL_EXPIRATION, + ) + results['pcapDownloadUrlExpiresIn'] = f'{PCAP_PRESIGNED_URL_EXPIRATION} seconds' + except Exception: + pass + + results['s3Bucket'] = LOGS_BUCKET + results['s3KeyPcap'] = s3_key_pcap + results['s3KeyTxt'] = s3_key_txt + results['s3KeyStats'] = s3_key_stats + + return success_response(results) + + +# ============================================================================= +# ECS HUMAN APPROVAL AND RESTRICTED TCPDUMP OVERRIDES +# ============================================================================= +# Preserve the established direct paths for explicitly supervised deployments. +_direct_start_log_collection = start_log_collection +_direct_get_collection_status = get_collection_status +_direct_validate_bundle_completeness = validate_bundle_completeness +_direct_batch_collect = batch_collect +_direct_batch_status = batch_status +_direct_poll_tcpdump_status = _poll_tcpdump_status +_direct_tcpdump_analyze = tcpdump_analyze + +APPROVAL_STEP_NAME = 'waitForHumanApproval' +TCPDUMP_RUN_STEP_NAME = 'runTcpdump' +APPROVAL_WAIT_SECONDS = 25 +APPROVAL_WAIT_CHECK_SECONDS = 5 +_TASK_ID_RE = re.compile(r'^[0-9a-f]{32}$') +_TASK_ARN_RE = re.compile( + r'^arn:(?:aws|aws-us-gov|aws-cn):ecs:[a-z0-9-]+:\d{12}:task/' + r'(?:[A-Za-z0-9_-]+/)?([0-9a-f]{32})$' +) +_CONTAINER_NAME_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$') +_INSTANCE_ID_RE = re.compile(r'^i-[0-9a-f]{8,17}$') +_EXECUTION_ID_RE = re.compile( + r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' +) +_COMMAND_ID_RE = _EXECUTION_ID_RE +_TERMINAL_FAILURE_STATUSES = frozenset({ + 'Failed', 'TimedOut', 'Cancelled', 'Cancelling', 'Exited', +}) + + +def console_automation_url(region: str, execution_id: str) -> str: + return ( + f'https://{region}.console.aws.amazon.com/systems-manager/automation/' + f'execution/{execution_id}?region={region}' + ) + + +def _approval_configured() -> bool: + return bool(COLLECT_APPROVAL_DOCUMENT and APPROVAL_APPROVERS) + + +def _tcpdump_approval_configured() -> bool: + return bool(TCPDUMP_APPROVAL_DOCUMENT and APPROVAL_APPROVERS) + + +def notify_approvers(tool_name: str, target: str, region: str, + execution_id: str, arguments: Dict) -> None: + if not APPROVAL_TOPIC_ARN: + return + try: + sns_client.publish( + TopicArn=APPROVAL_TOPIC_ARN, + Subject=f'[ECS Diagnostics] Approval needed: {tool_name}'[:100], + Message=( + f"Approval is required for '{tool_name}' on {target} in {region}.\n" + f"Execution: {execution_id}\n" + f"Review: {console_automation_url(region, execution_id)}" + ), + ) + except Exception as exc: + print(f'Warning: approval notification failed: {exc}') + + +def _pending_approval_response(tool_name: str, target: str, region: str, + execution_id: str, + extra: Optional[Dict] = None) -> Dict: + url = console_automation_url(region, execution_id) + notification = ( + 'Approvers were notified by email.' if APPROVAL_EMAILS_CONFIGURED else + 'No approval email subscriptions are configured; share the console link.' + ) + payload = { + 'status': 'pending_approval', + 'executionId': execution_id, + 'tool': tool_name, + 'target': target, + 'region': region, + 'approvalConsoleUrl': url, + 'humanApproval': {'state': 'pending', 'consoleUrl': url}, + 'message': f'Waiting for native SSM human approval. {notification}', + 'suggestedPollIntervalSeconds': 30, + 'task': { + 'taskId': execution_id, + 'state': 'running', + 'message': 'Waiting for human approval', + 'progress': 0, + }, + } + if extra: + payload.update(extra) + return success_response(payload) + + +def enforce_approval_preconditions(target_region: str) -> Optional[Dict]: + if not _approval_configured(): + return error_response( + 503, + 'Human approval is required but COLLECT_APPROVAL_DOCUMENT or ' + 'APPROVAL_APPROVERS is not configured.', + ) + if target_region != DEFAULT_REGION: + return error_response( + 400, + f'Approval wrappers are regional and only configured in {DEFAULT_REGION}; ' + f'requested {target_region}.', + ) + return None + + +def enforce_tcpdump_approval_preconditions(target_region: str) -> Optional[Dict]: + if not _tcpdump_approval_configured(): + return error_response( + 503, + 'Human approval is required but TCPDUMP_APPROVAL_DOCUMENT or ' + 'APPROVAL_APPROVERS is not configured.', + ) + if target_region != DEFAULT_REGION: + return error_response( + 400, + f'Approval wrappers are regional and only configured in {DEFAULT_REGION}; ' + f'requested {target_region}.', + ) + return None + + +def _approval_step_pending(execution: Dict) -> bool: + return any( + step.get('StepName') == APPROVAL_STEP_NAME and + step.get('StepStatus') in ('Pending', 'InProgress', 'Waiting') + for step in execution.get('StepExecutions', []) or [] + ) + + +def wait_for_approval_decision(regional_ssm, execution_id: str, + execution: Dict) -> Dict: + deadline = time.time() + APPROVAL_WAIT_SECONDS + latest = execution + while _approval_step_pending(latest) and time.time() < deadline: + time.sleep(APPROVAL_WAIT_CHECK_SECONDS) + try: + latest = regional_ssm.get_automation_execution( + AutomationExecutionId=execution_id + )['AutomationExecution'] + except Exception: + break + return latest + + +def _is_approval_wrapper(document_name: str) -> bool: + return bool(document_name) and document_name in { + COLLECT_APPROVAL_DOCUMENT, + BATCH_APPROVAL_DOCUMENT, + TCPDUMP_APPROVAL_DOCUMENT, + } + + +def _step_output_values(step: Dict, key: str) -> List[str]: + values = (step.get('Outputs', {}) or {}).get(key, []) + return [value for value in values if isinstance(value, str)] + + +def _parse_batch_children(entries: List[str]) -> List[Dict]: + children = [] + for entry in entries: + if not isinstance(entry, str) or len(entry) > 256: + continue + instance_id, separator, execution_id = entry.partition('|') + instance_id = instance_id.strip() + execution_id = execution_id.strip() + if ( + separator + and _INSTANCE_ID_RE.fullmatch(instance_id) + and _EXECUTION_ID_RE.fullmatch(execution_id) + ): + children.append({'instanceId': instance_id, 'executionId': execution_id}) + return children + + +def _parse_batch_errors(entries: List[str]) -> List[Dict]: + errors = [] + for entry in entries: + if not isinstance(entry, str) or len(entry) > 2048 or any( + character in entry for character in ('\r', '\n', '\x00') + ): + continue + instance_id, separator, message = entry.partition('|') + instance_id = instance_id.strip() + message = message.strip() + if separator and _INSTANCE_ID_RE.fullmatch(instance_id) and message: + errors.append({'instanceId': instance_id, 'error': message}) + return errors + + +def _wrapper_approval_state(execution: Dict, region: str) -> Dict: + steps = execution.get('StepExecutions', []) or [] + approval_step = next( + (step for step in steps if step.get('StepName') == APPROVAL_STEP_NAME), None + ) + if approval_step is None: + return {} + status = approval_step.get('StepStatus', '') + url = console_automation_url(region, execution.get('AutomationExecutionId', '')) + if status in ('Pending', 'InProgress', 'Waiting'): + return {'state': 'pending', 'consoleUrl': url} + if status in ('Failed', 'TimedOut', 'Cancelled'): + return {'state': 'denied_or_expired', 'consoleUrl': url} + return {'state': 'approved', 'consoleUrl': url} + + +def start_collection_with_approval(instance_id: str, target_region: str, + arguments: Dict) -> Dict: + regional_ssm = get_regional_client('ssm', target_region) + try: + response = regional_ssm.start_automation_execution( + DocumentName=COLLECT_APPROVAL_DOCUMENT, + Parameters={ + 'ECSInstanceId': [instance_id], + 'LogDestination': [LOGS_BUCKET], + }, + ) + except Exception as exc: + return error_response(500, f'Failed to start approval wrapper: {exc}') + execution_id = response['AutomationExecutionId'] + if not store_execution_provenance( + execution_id, + tool='collect', + region=target_region, + expected_document=COLLECT_APPROVAL_DOCUMENT, + instance_id=instance_id, + ): + return error_response(500, 'Approval started but secure execution provenance could not be persisted.') + token = arguments.get('idempotencyToken') + if token: + store_idempotency_mapping(instance_id, token, execution_id) + notify_approvers('collect', instance_id, target_region, execution_id, arguments) + return _pending_approval_response( + 'collect', instance_id, target_region, execution_id, + {'instanceId': instance_id, 's3Bucket': LOGS_BUCKET}, + ) + + +def start_log_collection(arguments: Dict) -> Dict: + if not REQUIRE_COLLECTION_APPROVAL: + return _direct_start_log_collection(arguments) + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + if not re.fullmatch(r'i-[0-9a-f]{8,17}', instance_id): + return error_response(400, f'Invalid instanceId format: {instance_id}') + target_region, region_error = resolve_and_validate_region(arguments, instance_id) + if region_error: + return region_error + instance_error = validate_ecs_instance(instance_id, target_region) + if instance_error: + return instance_error + precondition_error = enforce_approval_preconditions(target_region) + if precondition_error: + return precondition_error + token = arguments.get('idempotencyToken') + if token: + existing = find_execution_by_idempotency_token(instance_id, token) + if existing and get_execution_provenance(existing.get('executionId', '')): + return success_response({ + 'executionId': existing['executionId'], + 'instanceId': instance_id, + 'status': existing.get('status', 'InProgress'), + 'idempotent': True, + }) + return start_collection_with_approval(instance_id, target_region, arguments) + + +def get_collection_status(arguments: Dict) -> Dict: + execution_id = arguments.get('executionId') + if not execution_id: + return error_response(400, 'executionId is required') + provenance, provenance_error = require_execution_provenance( + execution_id, allowed_tools={'collect'} + ) + if provenance_error: + return provenance_error + region = provenance['region'] + try: + regional_ssm = get_regional_client('ssm', region) + execution = regional_ssm.get_automation_execution( + AutomationExecutionId=execution_id + )['AutomationExecution'] + except Exception as exc: + return error_response(500, f'Failed to get status: {exc}') + details_error = validate_execution_details(execution, provenance) + if details_error: + return details_error + if not _is_approval_wrapper(execution.get('DocumentName', '')): + return _direct_get_collection_status({ + **arguments, 'region': region, + }) + if _approval_step_pending(execution): + execution = wait_for_approval_decision(regional_ssm, execution_id, execution) + approval = _wrapper_approval_state(execution, region) + result = { + 'executionId': execution_id, + 'status': execution.get('AutomationExecutionStatus', 'Unknown'), + 'documentName': execution.get('DocumentName', ''), + 'humanApproval': approval, + 'approvalConsoleUrl': approval.get('consoleUrl'), + } + if approval.get('state') == 'pending': + result.update({'progress': 0, 'suggestedPollIntervalSeconds': 30}) + elif approval.get('state') == 'denied_or_expired': + result.update({'progress': 0, 'status': 'Denied'}) + else: + for step in execution.get('StepExecutions', []) or []: + child_ids = _step_output_values(step, 'ExecutionId') + if child_ids: + child_id = child_ids[0] + result['childExecutionId'] = child_id + existing_child = get_execution_provenance(child_id) + if not existing_child: + if not store_execution_provenance( + child_id, + tool='collect', + region=region, + expected_document='AWSSupport-CollectECSInstanceLogs', + instance_id=provenance.get('instanceId'), + ): + return error_response( + 500, + 'Child collection started but secure execution provenance could not be persisted.', + ) + child_provenance, child_provenance_error = require_execution_provenance( + child_id, + requested_instance_id=provenance.get('instanceId'), + allowed_tools={'collect'}, + ) + if child_provenance_error: + return child_provenance_error + if child_provenance.get('expectedDocument') != 'AWSSupport-CollectECSInstanceLogs': + return error_response(403, 'Child execution document does not match the collection contract.') + child_response = _direct_get_collection_status({ + 'executionId': child_id, + 'region': region, + 'includeStepDetails': arguments.get('includeStepDetails', True), + }) + if child_response.get('statusCode') == 200: + result['childAutomation'] = json.loads(child_response['body']).get('automation') + break + result['task'] = { + 'taskId': execution_id, + 'state': 'failed' if approval.get('state') == 'denied_or_expired' else 'running', + 'message': f"Human approval: {approval.get('state', 'unknown')}", + 'progress': result.get('progress', 0), + } + return success_response({'automation': result}) + + +def validate_bundle_completeness(arguments: Dict) -> Dict: + """Validate an instance bundle, binding execution-based requests to provenance.""" + execution_id = arguments.get('executionId') + requested_instance = arguments.get('instanceId') + if execution_id: + provenance, provenance_error = require_execution_provenance( + execution_id, + requested_instance_id=requested_instance, + allowed_tools={'collect'}, + ) + if provenance_error: + return provenance_error + region = provenance['region'] + try: + execution = get_regional_client('ssm', region).get_automation_execution( + AutomationExecutionId=execution_id + )['AutomationExecution'] + except Exception as exc: + return error_response(500, f'Failed to get execution details: {exc}') + details_error = validate_execution_details(execution, provenance) + if details_error: + return details_error + instance_id = provenance.get('instanceId') + if not instance_id: + return error_response(403, 'Execution provenance does not identify an instance.') + return _direct_validate_bundle_completeness({'instanceId': instance_id}) + if not requested_instance: + return error_response(400, 'Either executionId or instanceId is required') + instance_error = validate_instance_id(requested_instance) + if instance_error: + return instance_error + return _direct_validate_bundle_completeness({'instanceId': requested_instance}) + + +def batch_collect(arguments: Dict) -> Dict: + adjusted = dict(arguments) + adjusted.setdefault('dryRun', True) + if adjusted.get('dryRun') or not REQUIRE_COLLECTION_APPROVAL: + return _direct_batch_collect(adjusted) + + preview = _direct_batch_collect({**adjusted, 'dryRun': True}) + if preview.get('statusCode') != 200: + return preview + plan = json.loads(preview['body']) + sampled_ids = list(dict.fromkeys( + instance_id + for bucket in plan.get('buckets', []) + for instance_id in bucket.get('sampleInstances', []) + ))[:15] + if not sampled_ids: + return preview + region = plan.get('region', DEFAULT_REGION) + region_error = validate_region(region) + if region_error: + return region_error + cluster_name = plan.get('clusterName') + cluster_error = validate_cluster_name(cluster_name) + if cluster_error: + return cluster_error + for instance_id in sampled_ids: + membership_error = validate_ecs_instance( + instance_id, region, cluster_name=cluster_name + ) + if membership_error: + return membership_error + precondition_error = enforce_approval_preconditions(region) + if precondition_error: + return precondition_error + if not BATCH_APPROVAL_DOCUMENT: + return error_response(503, 'BATCH_APPROVAL_DOCUMENT is not configured') + try: + regional_ssm = get_regional_client('ssm', region) + response = regional_ssm.start_automation_execution( + DocumentName=BATCH_APPROVAL_DOCUMENT, + Parameters={ + 'InstanceIds': sampled_ids, + 'LogDestination': [LOGS_BUCKET], + }, + ) + except Exception as exc: + return error_response(500, f'Failed to start batch approval wrapper: {exc}') + execution_id = response['AutomationExecutionId'] + batch_id = hashlib.sha256( + f"{plan.get('clusterName')}-{datetime.utcnow().isoformat()}".encode() + ).hexdigest()[:12] + metadata = { + 'batchId': batch_id, + 'clusterName': cluster_name, + 'region': region, + 'createdAt': datetime.now(timezone.utc).isoformat(), + 'approvalExecutionId': execution_id, + 'plannedInstanceIds': sampled_ids, + 'buckets': plan.get('buckets', []), + 'executions': [], + } + if not store_execution_provenance( + execution_id, + tool='batch_collect', + region=region, + expected_document=BATCH_APPROVAL_DOCUMENT, + cluster_name=cluster_name, + instance_ids=sampled_ids, + ): + return error_response(500, 'Batch started but secure execution provenance could not be persisted.') + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, + Key=f'{_BATCH_METADATA_PREFIX}{batch_id}.json', + Body=json.dumps(metadata), + ContentType='application/json', + ) + except Exception: + return error_response(500, 'Batch started but secure batch metadata could not be persisted.') + notify_approvers('batch_collect', cluster_name, region, + execution_id, arguments) + return _pending_approval_response( + 'batch_collect', plan.get('clusterName', ''), region, execution_id, + { + 'batchId': batch_id, + 'plannedCollections': len(sampled_ids), + 'plannedInstanceIds': sampled_ids, + 'buckets': plan.get('buckets', []), + }, + ) + + +def _persist_batch_metadata(batch_id: str, metadata: Dict) -> None: + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, + Key=f'{_BATCH_METADATA_PREFIX}{batch_id}.json', + Body=json.dumps(metadata), + ContentType='application/json', + ) + except Exception: + pass + + +def batch_status(arguments: Dict) -> Dict: + batch_id = arguments.get('batchId') + if arguments.get('executionIds') is not None: + return error_response(400, 'executionIds is not accepted; batchId is required') + if not isinstance(batch_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,64}', batch_id): + return error_response(400, 'batchId must be the opaque ID returned by batch_collect') + meta_result = safe_s3_read(f'{_BATCH_METADATA_PREFIX}{batch_id}.json') + if not meta_result.get('success'): + return error_response(404, f'Batch {batch_id} metadata not found') + try: + meta = json.loads(meta_result['content']) + except (KeyError, TypeError, json.JSONDecodeError): + return error_response(500, f'Batch {batch_id} metadata is invalid') + if meta.get('batchId') != batch_id: + return error_response(403, 'Batch metadata identifier mismatch') + region_error = validate_region(meta.get('region')) + if region_error: + return region_error + cluster_error = validate_cluster_name(meta.get('clusterName')) + if cluster_error: + return cluster_error + + executions = [ + { + 'instanceId': str(entry.get('instanceId', '')), + 'executionId': str(entry.get('executionId', '')), + **({'status': entry['status']} if isinstance(entry.get('status'), str) else {}), + } + for entry in meta.get('executions', []) + if isinstance(entry, dict) + and _INSTANCE_ID_RE.fullmatch(str(entry.get('instanceId', ''))) + and _EXECUTION_ID_RE.fullmatch(str(entry.get('executionId', ''))) + ] + fan_out_errors = [ + {'instanceId': str(entry.get('instanceId', '')), 'error': entry['error']} + for entry in meta.get('fanOutErrors', []) + if isinstance(entry, dict) + and _INSTANCE_ID_RE.fullmatch(str(entry.get('instanceId', ''))) + and isinstance(entry.get('error'), str) + and entry['error'].strip() + and len(entry['error']) <= 2048 + and not any(character in entry['error'] for character in ('\r', '\n', '\x00')) + ] + wrapper_id = meta.get('approvalExecutionId') + approval = {'state': 'approved'} if executions else {} + + if wrapper_id: + provenance, provenance_error = require_execution_provenance( + wrapper_id, allowed_tools={'batch_collect'} + ) + if provenance_error: + return provenance_error + region = provenance['region'] + regional_ssm = get_regional_client('ssm', region) + try: + wrapper = regional_ssm.get_automation_execution( + AutomationExecutionId=wrapper_id + )['AutomationExecution'] + except Exception as exc: + return error_response(500, f'Failed to poll batch approval wrapper: {exc}') + details_error = validate_execution_details(wrapper, provenance) + if details_error: + return details_error + if _approval_step_pending(wrapper): + wrapper = wait_for_approval_decision(regional_ssm, wrapper_id, wrapper) + approval = _wrapper_approval_state(wrapper, region) + if approval.get('state') == 'pending': + return success_response({ + 'batchId': batch_id, 'allComplete': False, + 'status': 'pending_approval', 'approvalExecutionId': wrapper_id, + 'humanApproval': approval, 'suggestedPollIntervalSeconds': 30, + }) + if approval.get('state') == 'denied_or_expired': + return success_response({ + 'batchId': batch_id, 'allComplete': True, + 'status': 'denied_or_expired', 'approvalExecutionId': wrapper_id, + 'humanApproval': approval, + }) + + wrapper_status = wrapper.get('AutomationExecutionStatus', '') + fan_out_step = next( + (step for step in wrapper.get('StepExecutions', []) or [] + if step.get('StepName') == 'fanOutCollections'), + None, + ) + if fan_out_step: + raw_executions = _step_output_values(fan_out_step, 'Executions') + raw_errors = _step_output_values(fan_out_step, 'Errors') + if raw_executions or raw_errors: + executions = _parse_batch_children(raw_executions) + fan_out_errors = _parse_batch_errors(raw_errors) + for child in executions: + if not store_execution_provenance( + child['executionId'], + tool='collect', + region=region, + expected_document='AWSSupport-CollectECSInstanceLogs', + instance_id=child['instanceId'], + ): + return error_response(500, 'Could not persist child execution provenance') + meta['executions'] = [ + {**child, 'status': 'Started'} for child in executions + ] + meta['fanOutErrors'] = fan_out_errors + _persist_batch_metadata(batch_id, meta) + executions = meta['executions'] + + fan_out_status = (fan_out_step or {}).get('StepStatus', '') + if wrapper_status in _TERMINAL_FAILURE_STATUSES or ( + fan_out_status in _TERMINAL_FAILURE_STATUSES + ): + return success_response({ + 'batchId': batch_id, 'allComplete': True, 'status': 'failed', + 'approvalExecutionId': wrapper_id, 'humanApproval': approval, + 'failureReason': wrapper.get('FailureMessage') or + (fan_out_step or {}).get('FailureDetails') or + 'Batch approval wrapper or fan-out step failed.', + 'executions': executions, + 'fanOutErrors': fan_out_errors, + 'counts': { + 'planned': len(meta.get('plannedInstanceIds', [])), + 'started': len(executions), + 'startFailed': len(fan_out_errors), + }, + }) + + if not executions: + if fan_out_errors or wrapper_status == 'Success' or fan_out_status == 'Success': + meta['executions'] = [] + meta['fanOutErrors'] = fan_out_errors + _persist_batch_metadata(batch_id, meta) + return success_response({ + 'batchId': batch_id, 'allComplete': True, 'status': 'failed', + 'approvalExecutionId': wrapper_id, 'humanApproval': approval, + 'fanOutErrors': fan_out_errors, + 'counts': { + 'planned': len(meta.get('plannedInstanceIds', [])), + 'started': 0, + 'startFailed': len(fan_out_errors), + }, + }) + return success_response({ + 'batchId': batch_id, 'allComplete': False, + 'status': 'fanout_in_progress', 'approvalExecutionId': wrapper_id, + 'humanApproval': approval, 'fanOutErrors': fan_out_errors, + 'suggestedPollIntervalSeconds': 15, + }) + + if not executions: + return success_response({ + 'batchId': batch_id, 'allComplete': True, 'status': 'failed', + 'approvalExecutionId': wrapper_id, 'humanApproval': approval, + 'fanOutErrors': fan_out_errors, + 'counts': { + 'planned': len(meta.get('plannedInstanceIds', [])), + 'started': 0, + 'startFailed': len(fan_out_errors), + }, + }) + + result = _direct_batch_status({ + 'executionIds': [entry['executionId'] for entry in executions], + }) + if result.get('statusCode') == 200 and wrapper_id: + response_body = json.loads(result['body']) + response_body['batchId'] = batch_id + response_body['approvalExecutionId'] = wrapper_id + response_body['humanApproval'] = approval or {'state': 'approved'} + response_body['fanOutErrors'] = fan_out_errors + response_body['counts'] = { + 'planned': len(meta.get('plannedInstanceIds', [])), + 'started': len(executions), + 'startFailed': len(fan_out_errors), + } + if fan_out_errors: + response_body['status'] = 'partial_failure' + elif response_body.get('allComplete') and response_body.get('summary', {}).get('failed'): + response_body['status'] = 'failed' + else: + response_body['status'] = ( + 'complete' if response_body.get('allComplete') else 'in_progress' + ) + result['body'] = json.dumps(response_body, default=str) + return result + + +def normalize_ecs_task_id(value: str) -> Tuple[Optional[str], Optional[str]]: + """Accept exactly a 32-hex ECS task id or a full task ARN.""" + candidate = (value or '').strip() + if _TASK_ID_RE.fullmatch(candidate): + return candidate, None + match = _TASK_ARN_RE.fullmatch(candidate) + if match: + return match.group(1), None + return None, 'taskId must be a 32-hex ECS task ID or a full ECS task ARN' + + +def _tcpdump_execution_metadata_key(execution_id: str) -> str: + return f'_metadata/tcpdump/executions/{execution_id}.json' + + +def _tcpdump_command_metadata_key(command_id: str) -> str: + return f'_metadata/tcpdump/commands/{command_id}.json' + + +def _read_tcpdump_metadata(key: str) -> Optional[Dict]: + try: + response = s3_client.get_object(Bucket=LOGS_BUCKET, Key=key) + return json.loads(response['Body'].read().decode('utf-8')) + except Exception: + return None + + +def _store_tcpdump_metadata(key: str, metadata: Dict) -> None: + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, Key=key, Body=json.dumps(metadata), + ContentType='application/json', + ) + except Exception: + pass + + +def _build_ecs_tcpdump_script(task_id: str, container_name: str, interface: str, + bpf_filter: str, duration: int, + timestamp: str, s3_keys: Dict[str, str]) -> str: + """Build a fail-closed task-network-namespace capture script.""" + container_arg = container_name or '' + filter_suffix = f' {bpf_filter}' if bpf_filter else '' + lines = [ + '#!/bin/bash', + 'set -euo pipefail', + f'TASK_ID="{task_id}"', + f'REQUESTED_CONTAINER="{container_arg}"', + f'PCAP_FILE="/tmp/ecs_tcpdump_{timestamp}.pcap"', + f'TXT_FILE="/tmp/ecs_tcpdump_{timestamp}.txt"', + f'STATS_FILE="/tmp/ecs_tcpdump_{timestamp}.json"', + 'META_FILE=$(mktemp)', + "trap 'rm -f \"$META_FILE\" \"$PCAP_FILE\" \"$TXT_FILE\" \"$STATS_FILE\"' EXIT", + 'if ! command -v tcpdump >/dev/null 2>&1; then', + ' echo "FATAL: tcpdump is not installed; this tool never installs packages"', + ' exit 1', + 'fi', + 'if ! command -v python3 >/dev/null 2>&1; then', + ' echo "FATAL: python3 is required to parse ECS introspection metadata"', + ' exit 1', + 'fi', + 'curl --fail --silent --show-error http://localhost:51678/v1/tasks > "$META_FILE" || { echo "FATAL: ECS agent introspection unavailable"; exit 1; }', + "RESOLVED=$(python3 - \"$TASK_ID\" \"$REQUESTED_CONTAINER\" \"$META_FILE\" <<'PYMETA'", + 'import json, sys', + 'task_id, requested, path = sys.argv[1:4]', + "with open(path, encoding='utf-8') as handle:", + ' payload = json.load(handle)', + "tasks = payload if isinstance(payload, list) else payload.get('Tasks', [])", + 'matches = []', + 'for task in tasks:', + " arn = task.get('Arn') or task.get('TaskARN') or ''", + " resolved = arn.rsplit('/', 1)[-1]", + ' if resolved == task_id:', + ' matches.append(task)', + 'if len(matches) != 1:', + " raise SystemExit('expected exactly one exact task ID match')", + "containers = matches[0].get('Containers', []) or []", + 'def eligible(container):', + " status = container.get('KnownStatus') or container.get('LastStatus')", + " container_type = str(container.get('Type') or 'NORMAL').upper()", + " return bool(container.get('DockerId')) and status == 'RUNNING' and container_type not in {'CNI_PAUSE', 'RESOURCES_PROVISIONER'}", + 'if requested:', + " selected = [c for c in containers if c.get('Name') == requested]", + ' if len(selected) != 1:', + " raise SystemExit('explicit containerName did not match exactly once')", + ' container = selected[0]', + ' if not eligible(container):', + " raise SystemExit('explicit containerName must identify a RUNNING application container')", + 'else:', + ' selected = [c for c in containers if eligible(c)]', + ' if len(selected) != 1:', + " raise SystemExit('containerName is required unless exactly one RUNNING application container is eligible')", + ' container = selected[0]', + "print(task_id + '\\t' + str(container.get('Name', '')) + '\\t' + container['DockerId'])", + 'PYMETA', + ') || { echo "FATAL: task/container resolution failed"; exit 1; }', + "IFS=$'\\t' read -r RESOLVED_TASK_ID CONTAINER_NAME CONTAINER_ID <<< \"$RESOLVED\"", + '[ "$RESOLVED_TASK_ID" = "$TASK_ID" ] || { echo "FATAL: normalized task mismatch"; exit 1; }', + '[ -n "$CONTAINER_NAME" ] && [ -n "$CONTAINER_ID" ] || { echo "FATAL: incomplete container resolution"; exit 1; }', + 'resolve_pid() {', + ' local expected_runtime="${1:-}" pid="" info="" namespace=""', + ' if [ -z "$expected_runtime" ] || [ "$expected_runtime" = "docker" ]; then', + ' if command -v docker >/dev/null 2>&1; then', + " if pid=$(docker inspect \"$CONTAINER_ID\" 2>/dev/null | python3 -c \"import json,sys; data=json.load(sys.stdin); print(data[0].get('State', {}).get('Pid', 0) if len(data) == 1 and data[0].get('Id') == sys.argv[1] else 0)\" \"$CONTAINER_ID\"); then", + ' if [[ "$pid" =~ ^[0-9]+$ ]] && [ "$pid" -gt 0 ]; then printf "docker\\t%s\\n" "$pid"; return 0; fi', + ' fi', + ' fi', + ' [ -n "$expected_runtime" ] && return 1', + ' fi', + ' local ctr', + ' ctr=$(command -v ctr || true)', + ' [ -n "$ctr" ] || return 1', + ' for namespace in moby default; do', + ' if [ -n "$expected_runtime" ] && [ "$expected_runtime" != "containerd:$namespace" ]; then continue; fi', + ' "$ctr" -n "$namespace" containers info "$CONTAINER_ID" >/dev/null 2>&1 || continue', + ' info=$("$ctr" -n "$namespace" tasks info "$CONTAINER_ID" 2>/dev/null) || continue', + " pid=$(python3 -c \"import json,sys; print(json.load(sys.stdin).get('Pid', 0))\" <<< \"$info\") || continue", + ' if [[ "$pid" =~ ^[0-9]+$ ]] && [ "$pid" -gt 0 ]; then printf "containerd:%s\\t%s\\n" "$namespace" "$pid"; return 0; fi', + ' done', + ' return 1', + '}', + 'PROCESS_START_TIME() {', + " python3 - \"$1\" <<'PYSTART'", + 'import sys', + "text = open('/proc/' + sys.argv[1] + '/stat', encoding='ascii').read()", + "print(text[text.rfind(')') + 2:].split()[19])", + 'PYSTART', + '}', + 'RUNTIME_PID=$(resolve_pid) || { echo "FATAL: exact container runtime task not found"; exit 1; }', + "IFS=$'\\t' read -r CONTAINER_RUNTIME TARGET_PID <<< \"$RUNTIME_PID\"", + "case \"$TARGET_PID\" in ''|*[!0-9]*) echo \"FATAL: invalid container PID\"; exit 1;; esac", + '[ "$TARGET_PID" -gt 0 ] || { echo "FATAL: PID must be positive"; exit 1; }', + '[ -e "/proc/$TARGET_PID/ns/net" ] || { echo "FATAL: container network namespace is unavailable"; exit 1; }', + 'PROCESS_START=$(PROCESS_START_TIME "$TARGET_PID")', + 'NETWORK_NAMESPACE=$(readlink "/proc/$TARGET_PID/ns/net")', + 'NETWORK_NAMESPACE_INODE=$(stat -Lc %i "/proc/$TARGET_PID/ns/net")', + 'HOST_NAMESPACE_INODE=$(stat -Lc %i /proc/1/ns/net)', + '[ "$NETWORK_NAMESPACE_INODE" != "$HOST_NAMESPACE_INODE" ] || { echo "FATAL: task uses the host network namespace"; exit 1; }', + 'echo "RESOLVED_TASK_ID=$RESOLVED_TASK_ID"', + 'echo "CONTAINER_NAME=$CONTAINER_NAME"', + 'echo "CONTAINER_ID=$CONTAINER_ID"', + 'echo "PID=$TARGET_PID"', + 'echo "NETWORK_NAMESPACE=$NETWORK_NAMESPACE"', + '# Re-resolve through the same runtime immediately before namespace entry.', + 'RERESOLVED=$(resolve_pid "$CONTAINER_RUNTIME") || { echo "FATAL: selected container disappeared"; exit 1; }', + "IFS=$'\\t' read -r CURRENT_RUNTIME CURRENT_PID <<< \"$RERESOLVED\"", + '[ "$CURRENT_RUNTIME" = "$CONTAINER_RUNTIME" ] && [ "$CURRENT_PID" = "$TARGET_PID" ] || { echo "FATAL: selected container PID changed"; exit 1; }', + '[ "$(PROCESS_START_TIME "$TARGET_PID")" = "$PROCESS_START" ] || { echo "FATAL: process start time changed"; exit 1; }', + '[ "$(stat -Lc %i "/proc/$TARGET_PID/ns/net")" = "$NETWORK_NAMESPACE_INODE" ] || { echo "FATAL: network namespace inode changed"; exit 1; }', + '[ "$NETWORK_NAMESPACE_INODE" != "$(stat -Lc %i /proc/1/ns/net)" ] || { echo "FATAL: task uses the host network namespace"; exit 1; }', + f'timeout {duration} nsenter -n -t "$TARGET_PID" -- tcpdump -i "{interface}" -w "$PCAP_FILE" -c 100000{filter_suffix} || true', + '[ -f "$PCAP_FILE" ] || { echo "FATAL: capture file not created"; exit 1; }', + 'FILE_SIZE=$(stat -c%s "$PCAP_FILE" 2>/dev/null || stat -f%z "$PCAP_FILE")', + 'tcpdump -nn -r "$PCAP_FILE" -c 5000 > "$TXT_FILE" 2>/dev/null || true', + 'PACKET_COUNT=$(wc -l < "$TXT_FILE" | tr -d " ")', + "python3 - \"$PACKET_COUNT\" > \"$STATS_FILE\" <<'PYSTATS'", + 'import json, sys', + "json.dump({'totalPackets': int(sys.argv[1]), 'protocols': {}, 'tcpFlags': {}}, sys.stdout)", + 'PYSTATS', + f'aws s3 cp "$PCAP_FILE" "s3://{LOGS_BUCKET}/{s3_keys["pcap"]}" --no-progress', + f'aws s3 cp "$TXT_FILE" "s3://{LOGS_BUCKET}/{s3_keys["text"]}" --no-progress', + f'aws s3 cp "$STATS_FILE" "s3://{LOGS_BUCKET}/{s3_keys["stats"]}" --no-progress', + f'echo "S3_KEY={s3_keys["pcap"]}"', + f'echo "S3_KEY_TXT={s3_keys["text"]}"', + f'echo "S3_KEY_STATS={s3_keys["stats"]}"', + 'echo "FILE_SIZE=$FILE_SIZE"', + 'echo "PACKET_COUNT=$PACKET_COUNT"', + 'echo "RESOLVED_TASK_ID=$RESOLVED_TASK_ID"', + 'echo "CONTAINER_NAME=$CONTAINER_NAME"', + 'echo "CONTAINER_ID=$CONTAINER_ID"', + 'echo "PID=$TARGET_PID"', + 'echo "NETWORK_NAMESPACE=$NETWORK_NAMESPACE"', + 'echo "DONE"', + ] + script = '\n'.join(lines) + '\n' + if '{{' in script: + raise ValueError('generated command contains an SSM template sequence') + return script + + +def _capture_metadata_from_output(stdout: str) -> Dict: + markers = {} + mapping = { + 'RESOLVED_TASK_ID': 'resolvedTaskId', + 'CONTAINER_NAME': 'resolvedContainerName', + 'CONTAINER_ID': 'resolvedContainerId', + 'PID': 'targetPid', + 'NETWORK_NAMESPACE': 'networkNamespace', + } + for line in stdout.splitlines(): + key, separator, value = line.partition('=') + if separator and key in mapping and value.strip(): + markers[mapping[key]] = value.strip() + return markers + + +def _start_tcpdump_with_approval(regional_ssm, instance_id: str, region: str, + script: str, duration: int, + metadata: Dict, arguments: Dict) -> Dict: + try: + response = regional_ssm.start_automation_execution( + DocumentName=TCPDUMP_APPROVAL_DOCUMENT, + Parameters={ + 'InstanceId': [instance_id], + 'Commands': [script], + 'ExecutionTimeoutSeconds': [str(duration + 120)], + 'DurationSeconds': [str(duration)], + 'Interface': [metadata['interface']], + 'BpfFilter': [metadata.get('filter') or 'none'], + 'CaptureScope': [metadata['captureScope']], + }, + ) + except Exception as exc: + return error_response(500, f'Failed to start tcpdump approval wrapper: {exc}') + execution_id = response['AutomationExecutionId'] + metadata = {**metadata, 'executionId': execution_id} + _store_tcpdump_metadata(_tcpdump_execution_metadata_key(execution_id), metadata) + if not store_execution_provenance( + execution_id, + tool='tcpdump_capture', + region=region, + expected_document=TCPDUMP_APPROVAL_DOCUMENT, + instance_id=instance_id, + ): + return error_response(500, 'Capture started but secure execution provenance could not be persisted.') + notify_approvers('tcpdump_capture', instance_id, region, execution_id, arguments) + return _pending_approval_response( + 'tcpdump_capture', instance_id, region, execution_id, + { + 'instanceId': instance_id, + 'taskId': metadata['taskId'], + 'containerName': metadata.get('containerName'), + 's3Key': metadata['s3Key'], + }, + ) + + +def _poll_tcpdump_wrapper(execution_id: str, instance_id: str, + arguments: Dict) -> Dict: + provenance, provenance_error = require_execution_provenance( + execution_id, + requested_instance_id=instance_id, + allowed_tools={'tcpdump_capture'}, + ) + if provenance_error: + return provenance_error + region = provenance['region'] + regional_ssm = get_regional_client('ssm', region) + try: + execution = regional_ssm.get_automation_execution( + AutomationExecutionId=execution_id + )['AutomationExecution'] + except Exception as exc: + return error_response(500, f'Failed to poll tcpdump approval wrapper: {exc}') + details_error = validate_execution_details(execution, provenance) + if details_error: + return details_error + if _approval_step_pending(execution): + execution = wait_for_approval_decision(regional_ssm, execution_id, execution) + approval = _wrapper_approval_state(execution, region) + if approval.get('state') == 'pending': + return success_response({ + 'status': 'pending_approval', 'executionId': execution_id, + 'instanceId': instance_id, 'humanApproval': approval, + 'suggestedPollIntervalSeconds': 30, + }) + if approval.get('state') == 'denied_or_expired': + return error_response(403, 'tcpdump approval was denied or expired', { + 'executionId': execution_id, 'humanApproval': approval, + }) + wrapper_status = execution.get('AutomationExecutionStatus', '') + run_step = next( + (step for step in execution.get('StepExecutions', []) or [] + if step.get('StepName') == TCPDUMP_RUN_STEP_NAME), None + ) + command_ids = _step_output_values(run_step or {}, 'CommandId') + if not command_ids: + run_status = (run_step or {}).get('StepStatus', '') + if ( + wrapper_status in _TERMINAL_FAILURE_STATUSES + or wrapper_status == 'Success' + or run_status in _TERMINAL_FAILURE_STATUSES + or run_status == 'Success' + ): + return error_response(500, 'tcpdump wrapper terminated without a command ID', { + 'executionId': execution_id, + 'wrapperStatus': wrapper_status or 'Unknown', + 'runStepStatus': run_status or 'Unknown', + 'humanApproval': approval, + }) + return success_response({ + 'status': 'in_progress', 'executionId': execution_id, + 'instanceId': instance_id, 'humanApproval': approval, + 'message': 'Approved; dispatching the capture command.', + }) + command_id = command_ids[0] + if len(command_ids) != 1 or not _COMMAND_ID_RE.fullmatch(command_id): + return error_response(500, 'tcpdump wrapper returned an invalid command ID', { + 'executionId': execution_id, + }) + metadata = _read_tcpdump_metadata(_tcpdump_execution_metadata_key(execution_id)) or {} + if metadata.get('instanceId') and metadata['instanceId'] != instance_id: + return error_response(403, 'execution metadata belongs to a different instance') + execution_start = (run_step or {}).get('ExecutionStartTime') + if isinstance(execution_start, datetime): + if execution_start.tzinfo: + execution_start = execution_start.astimezone(timezone.utc).replace(tzinfo=None) + metadata['startedAt'] = execution_start.strftime('%Y-%m-%dT%H:%M:%SZ') + metadata.update({'commandId': command_id, 'executionId': execution_id}) + _store_tcpdump_metadata(_tcpdump_execution_metadata_key(execution_id), metadata) + _store_tcpdump_metadata(_tcpdump_command_metadata_key(command_id), metadata) + result = _poll_tcpdump_status(command_id, instance_id, arguments) + if result.get('statusCode') == 200: + body = json.loads(result['body']) + body['executionId'] = execution_id + body['humanApproval'] = approval + result['body'] = json.dumps(body, default=str) + return result + + +def _poll_tcpdump_status(command_id: str, instance_id: str, + arguments: Dict) -> Dict: + if not isinstance(command_id, str) or not _COMMAND_ID_RE.fullmatch(command_id): + return error_response(400, 'commandId must be a lowercase UUID') + metadata = _read_tcpdump_metadata(_tcpdump_command_metadata_key(command_id)) + if not metadata: + return error_response(404, f'tcpdump metadata for command {command_id} was not found') + if metadata.get('instanceId') != instance_id: + return error_response(403, 'command metadata belongs to a different instance') + region_error = validate_region(metadata.get('region')) + if region_error: + return region_error + result = _direct_poll_tcpdump_status( + command_id, instance_id, {**arguments, 'region': metadata['region']} + ) + if result.get('statusCode') != 200: + return result + body = json.loads(result['body']) + markers = _capture_metadata_from_output(body.get('output', '')) + if markers: + metadata.update(markers) + _store_tcpdump_metadata(_tcpdump_command_metadata_key(command_id), metadata) + body.update(markers) + file_size = body.get('fileSizeBytes', 0) or 0 + if file_size > MAX_PCAP_BYTES: + body['pcapOversized'] = True + body['pcapMaxBytes'] = MAX_PCAP_BYTES + body.setdefault('warnings', []).append( + f'pcap exceeds MAX_PCAP_BYTES ({format_bytes(MAX_PCAP_BYTES)}); ' + f'capture size is {format_bytes(file_size)}.' + ) + body['presignedUrlExpiresIn'] = f'{PCAP_PRESIGNED_URL_EXPIRATION} seconds' + result['body'] = json.dumps(body, default=str) + return result + + +def tcpdump_capture(arguments: Dict) -> Dict: + authorization_error = validate_tool_authorization('tcpdump_capture') + if authorization_error: + return authorization_error + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + instance_error = validate_instance_id(instance_id) + if instance_error: + return instance_error + + # Polling is non-mutating and intentionally precedes new-capture validation + # and confirmation. + execution_id = arguments.get('executionId') + if execution_id: + return _poll_tcpdump_wrapper(execution_id, instance_id, arguments) + command_id = arguments.get('commandId') + if command_id: + if not isinstance(command_id, str) or not _COMMAND_ID_RE.fullmatch(command_id): + return error_response(400, 'commandId must be a lowercase UUID') + return _poll_tcpdump_status(command_id, instance_id, arguments) + + task_id, task_error = normalize_ecs_task_id(arguments.get('taskId', '')) + if task_error: + return error_response(400, task_error) + container_name = (arguments.get('containerName') or '').strip() + if container_name and not _CONTAINER_NAME_RE.fullmatch(container_name): + return error_response(400, f'Invalid containerName: {container_name}') + try: + duration = int(arguments.get('durationSeconds', 120)) + except (TypeError, ValueError): + return error_response(400, 'durationSeconds must be an integer') + if duration < 10 or duration > 300: + return error_response(400, 'durationSeconds must be between 10 and 300') + interface = arguments.get('interface', 'any') + if not re.fullmatch(r'[A-Za-z0-9.-]+', interface): + return error_response(400, f'Invalid interface name: {interface}') + bpf_filter = arguments.get('filter', '') + bpf_error = validate_bpf_filter(bpf_filter) + if bpf_error: + return error_response(400, f'Invalid BPF filter: {bpf_error}') + confirm = arguments.get('confirmCapture', False) + if confirm is not True and str(confirm).lower() != 'true': + return error_response(400, 'tcpdump_capture requires confirmCapture=true', { + 'requiresConfirmation': True, + 'instanceId': instance_id, + 'taskId': task_id, + 'containerName': container_name or None, + }) + region, region_error = resolve_and_validate_region(arguments, instance_id) + if region_error: + return region_error + instance_error = validate_ecs_instance(instance_id, region) + if instance_error: + return instance_error + regional_ssm = get_regional_client('ssm', region) + timestamp = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') + prefix = f'tcpdump/{instance_id}/{timestamp}' + keys = { + 'pcap': f'{prefix}/capture.pcap', + 'text': f'{prefix}/capture_summary.txt', + 'stats': f'{prefix}/capture_stats.json', + } + script = _build_ecs_tcpdump_script( + task_id, container_name, interface, bpf_filter, duration, timestamp, keys + ) + metadata = { + 'instanceId': instance_id, + 'region': region, + 'taskId': task_id, + 'containerName': container_name or None, + 'captureScope': f'task/{task_id}/container/{container_name or "auto"}', + 'interface': interface, + 'filter': bpf_filter, + 'durationSeconds': duration, + 'startedAt': timestamp, + 's3Prefix': prefix, + 's3Key': keys['pcap'], + 's3KeyTxt': keys['text'], + 's3KeyStats': keys['stats'], + } + if REQUIRE_COLLECTION_APPROVAL: + precondition_error = enforce_tcpdump_approval_preconditions(region) + if precondition_error: + return precondition_error + return _start_tcpdump_with_approval( + regional_ssm, instance_id, region, script, duration, metadata, arguments + ) + try: + response = regional_ssm.send_command( + InstanceIds=[instance_id], DocumentName='AWS-RunShellScript', + Parameters={ + 'commands': [script], + 'executionTimeout': [str(duration + 120)], + }, + TimeoutSeconds=duration + 180, + Comment=f'tcpdump for ECS task {task_id} on {instance_id}', + ) + except Exception as exc: + return error_response(500, f'Failed to start tcpdump: {exc}') + command_id = response.get('Command', {}).get('CommandId') + if not isinstance(command_id, str) or not _COMMAND_ID_RE.fullmatch(command_id): + return error_response(500, 'SSM returned an invalid command ID') + metadata['commandId'] = command_id + _store_tcpdump_metadata(_tcpdump_command_metadata_key(command_id), metadata) + return success_response({ + 'status': 'in_progress', 'commandId': command_id, + 'instanceId': instance_id, 'taskId': task_id, + 'containerName': container_name or None, + 'captureScope': f'task/{task_id}/container/{container_name or "auto"}', + 's3Key': keys['pcap'], + 'task': { + 'taskId': command_id, 'state': 'running', + 'message': 'Packet capture command started; target namespace is pending command resolution', 'progress': 0, + }, + }) + + +def _validate_tcpdump_artifact_keys(metadata: Dict, instance_id: str) -> Optional[Dict]: + prefix = f'tcpdump/{instance_id}/' + expected_suffixes = { + 's3Key': '/capture.pcap', + 's3KeyTxt': '/capture_summary.txt', + 's3KeyStats': '/capture_stats.json', + } + for field, suffix in expected_suffixes.items(): + key = metadata.get(field) + if not isinstance(key, str) or not key: + return error_response(403, f'Capture metadata field {field} is missing.') + key_parts = key.split('/') + if ( + not key.startswith(prefix) + or not key.endswith(suffix) + or any(part in ('', '.', '..') for part in key_parts) + or any(ord(character) < 0x20 or ord(character) == 0x7f for character in key) + ): + return error_response( + 403, f"Capture metadata field {field} is outside the canonical '{prefix}' path.", + ) + return None + + +def tcpdump_analyze(arguments: Dict) -> Dict: + authorization_error = validate_tool_authorization('tcpdump_analyze') + if authorization_error: + return authorization_error + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + instance_error = validate_instance_id(instance_id) + if instance_error: + return instance_error + command_id = arguments.get('commandId') + if not command_id: + return error_response(400, 'commandId is required') + if not isinstance(command_id, str) or not _COMMAND_ID_RE.fullmatch(command_id): + return error_response(400, 'commandId must be a lowercase UUID') + metadata = _read_tcpdump_metadata(_tcpdump_command_metadata_key(command_id)) + if not metadata: + return error_response( + 404, + f'Metadata for commandId {command_id} was not found; refusing latest-capture fallback.', + ) + if metadata.get('instanceId') != instance_id: + return error_response(403, 'Capture metadata belongs to a different instance') + if metadata.get('commandId') != command_id: + return error_response(403, 'Capture metadata command identifier mismatch') + region_error = validate_region(metadata.get('region')) + if region_error: + return region_error + key_error = _validate_tcpdump_artifact_keys(metadata, instance_id) + if key_error: + return key_error + result = _direct_tcpdump_analyze({**arguments, 'commandId': command_id}) + if result.get('statusCode') != 200: + return result + body = json.loads(result['body']) + started_at = metadata.get('startedAt', '') + parsed = None + for fmt in ('%Y%m%dT%H%M%SZ', '%Y-%m-%dT%H:%M:%SZ'): + try: + parsed = datetime.strptime(started_at, fmt) + break + except (TypeError, ValueError): + continue + if parsed: + age_minutes = (datetime.utcnow() - parsed).total_seconds() / 60 + if age_minutes > 15: + body['stalenessWarning'] = ( + f'This capture is {int(age_minutes)} minutes old; network conditions ' + 'may have changed. Prefer a fresh capture.' + ) + result['body'] = json.dumps(body, default=str) + return result diff --git a/mcp/ecs-instance-log-mcp/tests/__init__.py b/mcp/ecs-instance-log-mcp/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mcp/ecs-instance-log-mcp/tests/conftest.py b/mcp/ecs-instance-log-mcp/tests/conftest.py new file mode 100644 index 0000000..71b1c17 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tests/conftest.py @@ -0,0 +1,13 @@ +""" +Shared pytest configuration for ECS Instance Log MCP tests. +Sets required environment variables before any test module imports the Lambda. +""" +import os + +# These must be set before boto3 clients are created at module-level in the Lambda +os.environ.setdefault('AWS_DEFAULT_REGION', 'us-east-1') +os.environ.setdefault('AWS_REGION', 'us-east-1') +os.environ.setdefault('LOGS_BUCKET_NAME', 'test-bucket') +os.environ.setdefault('SSM_AUTOMATION_ROLE_ARN', 'arn:aws:iam::123456789012:role/test') +os.environ.setdefault('ALLOWED_REGIONS', 'us-east-1,us-west-2') +os.environ.setdefault('ALLOWED_CLUSTER_NAMES', 'cluster-one,cluster-two,cluster') diff --git a/mcp/ecs-instance-log-mcp/tests/test_batch_approval.py b/mcp/ecs-instance-log-mcp/tests/test_batch_approval.py new file mode 100644 index 0000000..048916a --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tests/test_batch_approval.py @@ -0,0 +1,201 @@ +import json +from .test_tcpdump_security import mod, body + +APPROVER = 'arn:aws:iam::123456789012:role/Approver' +EXECUTION = '12345678-1234-1234-1234-123456789abc' +PLAN = {'success': True, 'clusterName': 'cluster', 'region': 'us-east-1', + 'buckets': [{'sampleInstances': ['i-0123456789abcdef0']}], + 'plannedCollections': 1} + + +def response(payload): return {'statusCode': 200, 'body': json.dumps(payload)} + + +def test_batch_defaults_to_dry_run(monkeypatch): + seen = [] + monkeypatch.setattr(mod, '_direct_batch_collect', lambda args: seen.append(args) or response(PLAN)) + mod.batch_collect({'clusterName': 'cluster'}) + assert seen[0]['dryRun'] is True + + +def test_batch_starts_one_approval_wrapper(monkeypatch): + calls = [] + class Ssm: + def start_automation_execution(self, **kwargs): + calls.append(kwargs); return {'AutomationExecutionId': 'batch-wrapper'} + monkeypatch.setattr(mod, 'REQUIRE_COLLECTION_APPROVAL', True) + monkeypatch.setattr(mod, 'COLLECT_APPROVAL_DOCUMENT', 'collect-wrapper') + monkeypatch.setattr(mod, 'BATCH_APPROVAL_DOCUMENT', 'batch-wrapper-doc') + monkeypatch.setattr(mod, 'APPROVAL_APPROVERS', [APPROVER]) + monkeypatch.setattr(mod, '_direct_batch_collect', lambda args: response(PLAN)) + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: Ssm()) + monkeypatch.setattr(mod, 'validate_ecs_instance', lambda *args, **kwargs: None) + monkeypatch.setattr(mod, 'store_execution_provenance', lambda *args, **kwargs: True) + monkeypatch.setattr(mod, 'notify_approvers', lambda *args: None) + monkeypatch.setattr(mod.s3_client, 'put_object', lambda **kwargs: {}) + result = mod.batch_collect({'clusterName': 'cluster', 'dryRun': False}) + assert body(result)['status'] == 'pending_approval' + assert len(calls) == 1 + assert calls[0]['Parameters']['InstanceIds'] == ['i-0123456789abcdef0'] + assert {'AutomationAssumeRole', 'Approvers', 'SNSTopicArn'}.isdisjoint( + calls[0]['Parameters'] + ) + + +def test_batch_child_output_parser(): + assert mod._parse_batch_children([f'i-0123456789abcdef0|{EXECUTION}', 'bad']) == [ + {'instanceId': 'i-0123456789abcdef0', 'executionId': EXECUTION}] + + +def test_batch_status_extracts_and_polls_children(monkeypatch): + meta = {'batchId': 'b1', 'clusterName': 'cluster', 'region': 'us-east-1', + 'approvalExecutionId': 'wrapper', 'executions': []} + wrapper = {'AutomationExecutionId': 'wrapper', + 'AutomationExecutionStatus': 'Success', 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'Success'}, + {'StepName': 'fanOutCollections', 'StepStatus': 'Success', + 'Outputs': {'Executions': [f'i-0123456789abcdef0|{EXECUTION}']}}]} + class Ssm: + def get_automation_execution(self, **kwargs): + return {'AutomationExecution': wrapper} + stored = [] + monkeypatch.setattr(mod, 'safe_s3_read', lambda key: { + 'success': True, 'content': json.dumps(meta)}) + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: Ssm()) + monkeypatch.setattr(mod, 'get_execution_provenance', lambda eid: { + 'executionId': eid, 'tool': 'batch_collect', 'region': 'us-east-1', + 'expectedDocument': 'batch-wrapper-doc', + 'instanceIds': ['i-0123456789abcdef0'], + }) + monkeypatch.setattr(mod, 'validate_execution_details', lambda *args: None) + monkeypatch.setattr(mod, 'store_execution_provenance', lambda *args, **kwargs: True) + monkeypatch.setattr(mod.s3_client, 'put_object', lambda **kwargs: stored.append(kwargs)) + monkeypatch.setattr(mod, '_direct_batch_status', lambda args: response({ + 'allComplete': False, 'executions': [{'executionId': EXECUTION}]})) + result = body(mod.batch_status({'batchId': 'b1'})) + assert result['humanApproval']['state'] == 'approved' + assert result['executions'][0]['executionId'] == EXECUTION + persisted = json.loads(stored[0]['Body']) + assert persisted['executions'][0]['instanceId'] == 'i-0123456789abcdef0' + + +def _batch_meta(): + return { + 'batchId': 'b1', 'clusterName': 'cluster', 'region': 'us-east-1', + 'approvalExecutionId': 'wrapper', 'executions': [], + 'plannedInstanceIds': ['i-0123456789abcdef0', 'i-0aaaaaaaaaaaaaaaa'], + } + + +def _install_wrapper(monkeypatch, wrapper, stored, metadata=None): + class Ssm: + def get_automation_execution(self, **kwargs): + return {'AutomationExecution': wrapper} + monkeypatch.setattr(mod, 'safe_s3_read', lambda key: { + 'success': True, 'content': json.dumps(metadata or _batch_meta())}) + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: Ssm()) + monkeypatch.setattr(mod, 'get_execution_provenance', lambda eid: { + 'executionId': eid, 'tool': 'batch_collect', 'region': 'us-east-1', + 'expectedDocument': 'batch-wrapper-doc', + 'instanceIds': _batch_meta()['plannedInstanceIds'], + }) + monkeypatch.setattr(mod, 'validate_execution_details', lambda *args: None) + monkeypatch.setattr(mod, 'store_execution_provenance', lambda *args, **kwargs: True) + monkeypatch.setattr(mod.s3_client, 'put_object', lambda **kwargs: stored.append(kwargs)) + + +def test_batch_all_starts_failed_is_terminal_and_persists_errors(monkeypatch): + stored = [] + wrapper = { + 'AutomationExecutionStatus': 'Success', + 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'Success'}, + {'StepName': 'fanOutCollections', 'StepStatus': 'Success', 'Outputs': { + 'Executions': ['malformed'], + 'Errors': ['i-0123456789abcdef0|start denied', 'bad|ignored'], + }}, + ], + } + _install_wrapper(monkeypatch, wrapper, stored) + result = body(mod.batch_status({'batchId': 'b1'})) + assert result['allComplete'] is True + assert result['status'] == 'failed' + assert result['counts'] == {'planned': 2, 'started': 0, 'startFailed': 1} + assert result['fanOutErrors'] == [ + {'instanceId': 'i-0123456789abcdef0', 'error': 'start denied'}] + assert json.loads(stored[-1]['Body'])['fanOutErrors'] == result['fanOutErrors'] + + +def test_batch_partial_starts_keep_polling_and_surface_counts(monkeypatch): + stored = [] + wrapper = { + 'AutomationExecutionStatus': 'Success', + 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'Success'}, + {'StepName': 'fanOutCollections', 'StepStatus': 'Success', 'Outputs': { + 'Executions': [f'i-0123456789abcdef0|{EXECUTION}'], + 'Errors': ['i-0aaaaaaaaaaaaaaaa|start failed'], + }}, + ], + } + _install_wrapper(monkeypatch, wrapper, stored) + polled = [] + monkeypatch.setattr(mod, '_direct_batch_status', lambda args: polled.append(args) or response({ + 'allComplete': False, + 'summary': {'total': 1, 'succeeded': 0, 'failed': 0, 'inProgress': 1, 'unknown': 0}, + 'executions': [{'executionId': EXECUTION, 'status': 'InProgress'}], + })) + result = body(mod.batch_status({'batchId': 'b1'})) + assert polled == [{'executionIds': [EXECUTION]}] + assert result['allComplete'] is False + assert result['status'] == 'partial_failure' + assert result['counts'] == {'planned': 2, 'started': 1, 'startFailed': 1} + assert result['fanOutErrors'][0]['instanceId'] == 'i-0aaaaaaaaaaaaaaaa' + + +def test_terminal_fanout_failure_is_terminal_with_parsed_outputs(monkeypatch): + stored = [] + wrapper = { + 'AutomationExecutionStatus': 'Failed', + 'FailureMessage': 'fan-out failed', + 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'Success'}, + {'StepName': 'fanOutCollections', 'StepStatus': 'Failed', 'Outputs': { + 'Executions': [f'i-0123456789abcdef0|{EXECUTION}'], + 'Errors': ['i-0aaaaaaaaaaaaaaaa|start failed'], + }}, + ], + } + _install_wrapper(monkeypatch, wrapper, stored) + monkeypatch.setattr(mod, '_direct_batch_status', lambda args: (_ for _ in ()).throw( + AssertionError('terminal wrapper must not poll children'))) + result = body(mod.batch_status({'batchId': 'b1'})) + assert result['allComplete'] is True + assert result['status'] == 'failed' + assert result['executions'][0]['executionId'] == EXECUTION + assert result['fanOutErrors'][0]['error'] == 'start failed' + assert json.loads(stored[-1]['Body'])['fanOutErrors'] == result['fanOutErrors'] + + +def test_terminal_wrapper_failure_overrides_persisted_children(monkeypatch): + metadata = _batch_meta() + metadata['executions'] = [{ + 'instanceId': 'i-0123456789abcdef0', + 'executionId': EXECUTION, + 'status': 'Started', + }] + wrapper = { + 'AutomationExecutionStatus': 'Failed', + 'FailureMessage': 'wrapper failed after a child start', + 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'Success'}, + {'StepName': 'fanOutCollections', 'StepStatus': 'Failed'}, + ], + } + _install_wrapper(monkeypatch, wrapper, [], metadata) + monkeypatch.setattr(mod, '_direct_batch_status', lambda args: (_ for _ in ()).throw( + AssertionError('terminal wrapper must not poll persisted children'))) + result = body(mod.batch_status({'batchId': 'b1'})) + assert result['allComplete'] is True + assert result['status'] == 'failed' + assert result['counts']['started'] == 1 diff --git a/mcp/ecs-instance-log-mcp/tests/test_collection_approval.py b/mcp/ecs-instance-log-mcp/tests/test_collection_approval.py new file mode 100644 index 0000000..0df5aec --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tests/test_collection_approval.py @@ -0,0 +1,140 @@ +import json +import os +import sys +from pathlib import Path + +os.environ.setdefault('LOGS_BUCKET_NAME', 'test-bucket') +os.environ.setdefault('SSM_AUTOMATION_ROLE_ARN', 'arn:aws:iam::123456789012:role/test') +os.environ.setdefault('AWS_REGION', 'us-east-1') +os.environ.setdefault('ALLOWED_REGIONS', 'us-east-1,us-west-2') +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src', 'lambda')) +mod = __import__('ecs-log-automation') + +INSTANCE = 'i-0123456789abcdef0' +APPROVER = 'arn:aws:iam::123456789012:role/Approver' + + +def body(result): + return json.loads(result['body']) + + +def test_collect_uses_ecs_approval_wrapper(monkeypatch): + calls = [] + class Ssm: + def start_automation_execution(self, **kwargs): + calls.append(kwargs) + return {'AutomationExecutionId': 'approval-1'} + monkeypatch.setattr(mod, 'REQUIRE_COLLECTION_APPROVAL', True) + monkeypatch.setattr(mod, 'COLLECT_APPROVAL_DOCUMENT', 'ecs-collect-approval') + monkeypatch.setattr(mod, 'APPROVAL_APPROVERS', [APPROVER]) + monkeypatch.setattr(mod, 'resolve_and_validate_region', lambda args, iid=None: ('us-east-1', None)) + monkeypatch.setattr(mod, 'validate_ecs_instance', lambda iid, region: None) + monkeypatch.setattr(mod, 'get_regional_client', lambda service, region: Ssm()) + monkeypatch.setattr(mod, 'store_execution_provenance', lambda *args, **kwargs: True) + monkeypatch.setattr(mod, 'notify_approvers', lambda *args: None) + result = mod.start_log_collection({'instanceId': INSTANCE}) + assert body(result)['status'] == 'pending_approval' + assert calls[0]['DocumentName'] == 'ecs-collect-approval' + assert calls[0]['Parameters']['ECSInstanceId'] == [INSTANCE] + assert {'AutomationAssumeRole', 'Approvers', 'SNSTopicArn'}.isdisjoint( + calls[0]['Parameters'] + ) + + +def test_status_surfaces_pending(monkeypatch): + execution = {'AutomationExecutionId': 'approval-1', 'DocumentName': 'ecs-collect-approval', + 'Parameters': {'ECSInstanceId': [INSTANCE]}, + 'AutomationExecutionStatus': 'InProgress', 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'InProgress'}]} + class Ssm: + def get_automation_execution(self, **kwargs): return {'AutomationExecution': execution} + monkeypatch.setattr(mod, 'COLLECT_APPROVAL_DOCUMENT', 'ecs-collect-approval') + monkeypatch.setattr(mod, 'get_execution_provenance', lambda eid: { + 'executionId': eid, 'tool': 'collect', 'instanceId': INSTANCE, + 'instanceIds': [], 'region': 'us-east-1', + 'expectedDocument': 'ecs-collect-approval', + }) + monkeypatch.setattr(mod, 'get_execution_region', lambda eid: 'us-east-1') + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: Ssm()) + monkeypatch.setattr(mod, 'wait_for_approval_decision', lambda *args: execution) + assert body(mod.get_collection_status({'executionId': 'approval-1'}))['automation']['humanApproval']['state'] == 'pending' + + +def test_status_surfaces_denied(monkeypatch): + execution = {'AutomationExecutionId': 'approval-1', 'DocumentName': 'ecs-collect-approval', + 'Parameters': {'ECSInstanceId': [INSTANCE]}, + 'AutomationExecutionStatus': 'Failed', 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'Failed'}]} + class Ssm: + def get_automation_execution(self, **kwargs): return {'AutomationExecution': execution} + monkeypatch.setattr(mod, 'COLLECT_APPROVAL_DOCUMENT', 'ecs-collect-approval') + monkeypatch.setattr(mod, 'get_execution_provenance', lambda eid: { + 'executionId': eid, 'tool': 'collect', 'instanceId': INSTANCE, + 'instanceIds': [], 'region': 'us-east-1', + 'expectedDocument': 'ecs-collect-approval', + }) + monkeypatch.setattr(mod, 'get_execution_region', lambda eid: 'us-east-1') + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: Ssm()) + result = body(mod.get_collection_status({'executionId': 'approval-1'})) + assert result['automation']['humanApproval']['state'] == 'denied_or_expired' + assert result['automation']['status'] == 'Denied' + + +def test_status_surfaces_approved_child(monkeypatch): + execution = {'AutomationExecutionId': 'approval-1', 'DocumentName': 'ecs-collect-approval', + 'Parameters': {'ECSInstanceId': [INSTANCE]}, + 'AutomationExecutionStatus': 'Success', 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'Success'}, + {'StepName': 'collectLogs', 'StepStatus': 'Success', + 'Outputs': {'ExecutionId': ['child-1']}}]} + class Ssm: + def get_automation_execution(self, **kwargs): return {'AutomationExecution': execution} + monkeypatch.setattr(mod, 'COLLECT_APPROVAL_DOCUMENT', 'ecs-collect-approval') + monkeypatch.setattr(mod, 'get_execution_provenance', lambda eid: { + 'executionId': eid, 'tool': 'collect', 'instanceId': INSTANCE, + 'instanceIds': [], 'region': 'us-east-1', + 'expectedDocument': ( + 'ecs-collect-approval' if eid == 'approval-1' + else 'AWSSupport-CollectECSInstanceLogs' + ), + }) + monkeypatch.setattr(mod, 'get_execution_region', lambda eid: 'us-east-1') + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: Ssm()) + monkeypatch.setattr(mod, '_direct_get_collection_status', lambda args: response_status('child-1')) + result = body(mod.get_collection_status({'executionId': 'approval-1'}))['automation'] + assert result['humanApproval']['state'] == 'approved' + assert result['childExecutionId'] == 'child-1' + + +def response_status(execution_id): + return {'statusCode': 200, 'body': json.dumps({'success': True, 'automation': { + 'executionId': execution_id, 'status': 'InProgress'}})} + + +def _construct_source(): + return (Path(__file__).parents[1] / 'src' / 'ecs-log-gateway-construct-v2.ts').read_text() + + +def test_cdk_bakes_deployment_owned_approval_values(): + source = _construct_source() + assert "AutomationAssumeRole: { type: 'String' }" not in source + assert "Approvers: { type: 'StringList' }" not in source + assert "SNSTopicArn: { type: 'String' }" not in source + assert 'assumeRole: this.ssmAutomationRole.roleArn' in source + assert 'NotificationArn: this.collectionApprovalTopic.topicArn' in source + assert 'Approvers: approverArns' in source + assert 'AutomationAssumeRole: this.ssmAutomationRole.roleArn' in source + assert '"AutomationAssumeRole": ["${this.ssmAutomationRole.roleArn}"]' in source + + +def test_cdk_ssm_permissions_are_region_and_document_scoped(): + source = _construct_source() + assert 'SendAutomationSignal' not in source + assert 'automation-definition/AWSSupport-CollectECSInstanceLogs:*' in source + assert 'document/AWSSupport-CollectECSInstanceLogs' in source + assert 'document/AWS-RunShellScript' in source + assert 'automation-definition/${documentName}:*' in source + assert ':document/${documentName}' in source + assert "if (!requireCollectionApproval && enabledRestrictedTools.includes('tcpdump_capture'))" in source + assert "'aws:RequestedRegion': allowedRegions" in source + assert ':document/*' not in source diff --git a/mcp/ecs-instance-log-mcp/tests/test_instance_validation.py b/mcp/ecs-instance-log-mcp/tests/test_instance_validation.py new file mode 100644 index 0000000..49a02c0 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tests/test_instance_validation.py @@ -0,0 +1,144 @@ +"""Tests for exact ECS API membership validation.""" +import json +import os +import sys + +from botocore.exceptions import ClientError + +os.environ.setdefault('LOGS_BUCKET_NAME', 'test-bucket') +os.environ.setdefault('SSM_AUTOMATION_ROLE_ARN', 'arn:aws:iam::123456789012:role/test') +os.environ.setdefault('AWS_REGION', 'us-east-1') +os.environ.setdefault('ALLOWED_REGIONS', 'us-east-1,us-west-2') +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src', 'lambda')) +mod = __import__('ecs-log-automation') + +INSTANCE = 'i-0123456789abcdef0' +OTHER = 'i-0aaaaaaaaaaaaaaaa' + + +def response_body(result): + return json.loads(result['body']) + + +class Ec2: + def __init__(self, instances=None, error=None): + self.instances = instances if instances is not None else [ + {'InstanceId': INSTANCE, 'Tags': [{'Key': 'Name', 'Value': 'ordinary-host'}]} + ] + self.error = error + + def describe_instances(self, **kwargs): + assert kwargs == {'InstanceIds': [INSTANCE]} + if self.error: + raise self.error + return {'Reservations': [{'Instances': self.instances}]} + + +class Ecs: + def __init__(self, active=True, failures=None, list_error=None): + self.active = active + self.failures = failures or [] + self.list_error = list_error + self.cluster_calls = [] + self.container_calls = [] + self.describe_calls = [] + + def list_clusters(self, **kwargs): + self.cluster_calls.append(kwargs) + if self.list_error: + raise self.list_error + if not kwargs: + return {'clusterArns': ['cluster-one'], 'nextToken': 'clusters-2'} + assert kwargs == {'nextToken': 'clusters-2'} + return {'clusterArns': ['cluster-two']} + + def list_container_instances(self, **kwargs): + self.container_calls.append(kwargs) + if self.list_error: + raise self.list_error + cluster = kwargs['cluster'] + if cluster == 'cluster-one' and 'nextToken' not in kwargs: + return {'containerInstanceArns': ['ci-other'], 'nextToken': 'containers-2'} + if cluster == 'cluster-one': + assert kwargs['nextToken'] == 'containers-2' + return {'containerInstanceArns': []} + return {'containerInstanceArns': ['ci-target']} + + def describe_container_instances(self, **kwargs): + self.describe_calls.append(kwargs) + if self.failures: + return {'failures': self.failures, 'containerInstances': []} + arn = kwargs['containerInstances'][0] + if arn == 'ci-target': + return {'containerInstances': [{ + 'containerInstanceArn': arn, + 'ec2InstanceId': INSTANCE, + 'status': 'ACTIVE' if self.active else 'DRAINING', + }]} + return {'containerInstances': [{ + 'containerInstanceArn': arn, + 'ec2InstanceId': OTHER, + 'status': 'ACTIVE', + }]} + + +def install_clients(monkeypatch, ec2=None, ecs=None): + clients = {'ec2': ec2 or Ec2(), 'ecs': ecs or Ecs()} + monkeypatch.setattr(mod, 'get_regional_client', lambda service, region: clients[service]) + return clients + + +def test_accepts_exact_active_membership_across_all_pages(monkeypatch): + monkeypatch.setattr(mod, 'ALLOWED_CLUSTER_NAMES', frozenset({'cluster-one', 'cluster-two'})) + clients = install_clients(monkeypatch) + assert mod.validate_ecs_instance(INSTANCE, 'us-east-1') is None + assert clients['ecs'].cluster_calls == [] + assert {'cluster': 'cluster-one', 'nextToken': 'containers-2'} in clients['ecs'].container_calls + assert clients['ecs'].describe_calls[-1]['cluster'] == 'cluster-two' + assert clients['ecs'].describe_calls[-1]['containerInstances'] == ['ci-target'] + + +def test_rejects_name_and_tag_heuristics_without_membership(monkeypatch): + ec2 = Ec2([{'InstanceId': INSTANCE, 'Tags': [ + {'Key': 'Name', 'Value': 'my-ecs-worker'}, + {'Key': 'aws:ecs:clusterName', 'Value': 'cluster-two'}, + ]}]) + ecs = Ecs() + ecs.list_container_instances = lambda **kwargs: {'containerInstanceArns': []} + install_clients(monkeypatch, ec2, ecs) + result = mod.validate_ecs_instance(INSTANCE, 'us-east-1') + assert result['statusCode'] == 403 + assert 'not an ACTIVE container instance' in response_body(result)['error'] + + +def test_rejects_exact_membership_when_not_active(monkeypatch): + install_clients(monkeypatch, ecs=Ecs(active=False)) + assert mod.validate_ecs_instance(INSTANCE, 'us-east-1')['statusCode'] == 403 + + +def test_rejects_missing_ec2_instance(monkeypatch): + install_clients(monkeypatch, ec2=Ec2([])) + assert mod.validate_ecs_instance(INSTANCE, 'us-east-1')['statusCode'] == 404 + + +def test_fails_closed_on_ecs_list_error(monkeypatch): + install_clients(monkeypatch, ecs=Ecs(list_error=RuntimeError('unavailable'))) + result = mod.validate_ecs_instance(INSTANCE, 'us-east-1') + assert result['statusCode'] == 500 + assert 'Failed to validate instance' in response_body(result)['error'] + + +def test_fails_closed_on_describe_failures(monkeypatch): + install_clients(monkeypatch, ecs=Ecs(failures=[{'reason': 'MISSING'}])) + result = mod.validate_ecs_instance(INSTANCE, 'us-east-1') + assert result['statusCode'] == 500 + assert 'Failed to verify ECS membership' in response_body(result)['error'] + + +def test_maps_invalid_instance_client_error_to_not_found(monkeypatch): + error = ClientError( + {'Error': {'Code': 'InvalidInstanceID.NotFound', 'Message': 'missing'}}, + 'DescribeInstances', + ) + install_clients(monkeypatch, ec2=Ec2(error=error)) + assert mod.validate_ecs_instance(INSTANCE, 'us-east-1')['statusCode'] == 404 diff --git a/mcp/ecs-instance-log-mcp/tests/test_region_validation.py b/mcp/ecs-instance-log-mcp/tests/test_region_validation.py new file mode 100644 index 0000000..3064b36 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tests/test_region_validation.py @@ -0,0 +1,111 @@ +""" +Property-based tests for Lambda region validation (Properties 5, 6). +Tests ALLOWED_REGIONS parsing and validate_region() correctness. +Mirrors eks-node-log-mcp/tests/test_region_validation.py for ECS. +""" +import os +import sys +import json +import pytest +from hypothesis import given, strategies as st, settings, assume + +# Strategy: generate comma-separated region strings with edge cases +region_str = st.from_regex(r'[a-z]{2}-[a-z]+-[0-9]', fullmatch=True) +region_list = st.lists(region_str, min_size=0, max_size=5) + + +# ============================================================================= +# Property 5: Lambda ALLOWED_REGIONS parsing +# ============================================================================= + +@given(regions=region_list) +@settings(max_examples=100) +def test_allowed_regions_parsing_clean(regions): + """Clean comma-separated regions parse to exact set.""" + env_val = ','.join(regions) + parsed = set( + r.strip() for r in env_val.split(',') + if r.strip() + ) + expected = set(r for r in regions if r.strip()) + assert parsed == expected + + +@given(regions=st.lists(region_str, min_size=1, max_size=4)) +@settings(max_examples=100) +def test_allowed_regions_parsing_with_whitespace(regions): + """Regions with extra whitespace and empty segments parse correctly.""" + parts = [] + for r in regions: + parts.append(f' {r} ') + parts.append('') + parts.insert(0, '') + env_val = ','.join(parts) + + parsed = set( + r.strip() for r in env_val.split(',') + if r.strip() + ) + assert parsed == set(regions) + + +def test_allowed_regions_empty_defaults_to_aws_region(): + """Empty ALLOWED_REGIONS defaults to {AWS_REGION}.""" + parsed = set( + r.strip() for r in ''.split(',') + if r.strip() + ) or {'us-east-1'} + assert parsed == {'us-east-1'} + + +def test_allowed_regions_only_commas(): + """String of only commas produces empty set, falls back to default.""" + parsed = set( + r.strip() for r in ',,,'.split(',') + if r.strip() + ) or {'us-west-2'} + assert parsed == {'us-west-2'} + + +# ============================================================================= +# Property 6: Lambda region validation correctness +# ============================================================================= + +@given( + region=region_str, + allowed=st.frozensets(region_str, min_size=1, max_size=5), +) +@settings(max_examples=100) +def test_validate_region_membership(region, allowed): + """validate_region returns None iff region is in allowed set.""" + if region in allowed: + result = None + else: + result = {'statusCode': 403} + + if region in allowed: + assert result is None + else: + assert result is not None + assert result['statusCode'] == 403 + + +@given(allowed=st.frozensets(region_str, min_size=1, max_size=5)) +@settings(max_examples=100) +def test_validate_region_always_accepts_member(allowed): + """Any region that is a member of the allowed set is accepted.""" + for r in allowed: + assert r in allowed + + +@given( + region=region_str, + allowed=st.frozensets(region_str, min_size=1, max_size=3), +) +@settings(max_examples=100) +def test_validate_region_rejects_non_member(region, allowed): + """A region not in the allowed set is rejected with 403.""" + assume(region not in allowed) + result = {'statusCode': 403, 'body': json.dumps({'error': f"Region '{region}' is not permitted"})} + assert result['statusCode'] == 403 + assert region in result['body'] diff --git a/mcp/ecs-instance-log-mcp/tests/test_security_parity.py b/mcp/ecs-instance-log-mcp/tests/test_security_parity.py new file mode 100644 index 0000000..71265e5 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tests/test_security_parity.py @@ -0,0 +1,171 @@ +"""Focused AppSec parity tests for deployment scope and data isolation.""" +import json +import re +import time +from pathlib import Path + +import pytest + +from .test_tcpdump_security import INSTANCE, TASK, body, mod + +OTHER = 'i-0aaaaaaaaaaaaaaaa' + + +def test_cluster_allowlist_fails_closed(monkeypatch): + monkeypatch.setattr(mod, 'ALLOWED_CLUSTER_NAMES', frozenset()) + assert mod.validate_cluster_name('cluster')['statusCode'] == 503 + + +def test_cluster_allowlist_rejects_unlisted(monkeypatch): + monkeypatch.setattr(mod, 'ALLOWED_CLUSTER_NAMES', frozenset({'cluster'})) + assert mod.validate_cluster_name('other')['statusCode'] == 403 + assert mod.validate_cluster_name('cluster') is None + + +@pytest.mark.parametrize('call', [ + lambda: mod.cluster_health_check({'clusterName': 'cluster', 'region': 'eu-west-1'}), + lambda: mod.batch_collect({'clusterName': 'cluster', 'region': 'eu-west-1'}), + lambda: mod.list_collection_history({'region': 'eu-west-1'}), + lambda: mod.network_diagnostics({ + 'instanceId': INSTANCE, 'sections': 'eni', 'region': 'eu-west-1', + }), +]) +def test_public_handlers_reject_disallowed_region(call): + assert call()['statusCode'] == 403 + + +@pytest.mark.parametrize('key,status', [ + (f'ecs_{INSTANCE}/bundle/extracted/ecs-agent.log', None), + (f'ecs_{OTHER}/bundle/extracted/ecs-agent.log', 403), + (f'ecs_{INSTANCE}/../private', 400), + (f'ecs_{INSTANCE}/_metadata/execution.json', 403), + (f'ecs_{INSTANCE}/bundle/capture.pcap', 403), +]) +def test_public_log_keys_are_instance_bound(key, status): + result = mod.validate_log_key(key, INSTANCE) + assert (result is None) if status is None else result['statusCode'] == status + + +def test_artifact_url_lifetime_is_deployment_capped(monkeypatch): + key = f'ecs_{INSTANCE}/bundle/extracted/large.log' + captured = {} + monkeypatch.setattr(mod, 'PRESIGNED_URL_EXPIRATION', 120) + monkeypatch.setattr(mod, 'safe_s3_head', lambda value: { + 'success': True, 'size': 10, 'content_type': 'text/plain', + }) + monkeypatch.setattr(mod.s3_client, 'generate_presigned_url', + lambda *args, **kwargs: captured.update(kwargs) or 'https://example.invalid') + result = body(mod.get_artifact_reference({ + 'instanceId': INSTANCE, 'logKey': key, 'expirationMinutes': 60, + })) + assert captured['ExpiresIn'] == 120 + assert result['expiresIn'] == '120 seconds' + +def test_regex_rejects_catastrophic_patterns(): + assert mod.is_dangerous_regex('(a+)+$') is True + assert mod.is_dangerous_regex('(a|aa)+$') is True + assert mod.is_dangerous_regex(r'CannotPull|OOMKilled') is False + + +def test_compiled_regex_search_and_timeout_cleanup(monkeypatch): + monkeypatch.setattr(mod, 'safe_s3_head_raw', lambda *args, **kwargs: { + 'ContentLength': 30, + }) + monkeypatch.setattr(mod, 'safe_s3_read_raw', + lambda *args, **kwargs: b'normal\nOOMKilled container\n') + matches = mod.search_file_for_pattern( + 'bucket', f'ecs_{INSTANCE}/bundle/log', re.compile('oomkilled', re.I), + ) + assert matches[0]['lineNumber'] == 2 + with pytest.raises(mod.RegexTimeout): + with mod.regex_time_limit(0.01): + time.sleep(0.05) + with mod.regex_time_limit(0.01): + pass + + +def test_compare_instances_rejects_non_instance_identifiers(): + result = mod.compare_instances({'instanceIds': [INSTANCE, '../metadata']}) + assert result['statusCode'] == 400 + + +def test_direct_status_rejects_tampered_persisted_region(monkeypatch): + monkeypatch.setattr(mod, 'get_execution_region', lambda execution_id: 'eu-west-1') + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: pytest.fail('AWS call made')) + assert mod._direct_get_collection_status({'executionId': 'execution'})['statusCode'] == 403 + + +def test_cdk_hardening_contracts_are_present(): + source = (Path(__file__).parents[1] / 'src' / 'ecs-log-gateway-construct-v2.ts').read_text() + assert 'KMS encryption is mandatory and cannot be disabled' in source + assert '`allowedClusterNames` must contain at least one valid ECS cluster name' in source + assert '`ecsInstanceRoleArns` must contain at least one explicit IAM role ARN' in source + assert 'ALLOWED_CLUSTER_NAMES: allowedClusterNames.join' in source + assert 'noncurrentVersionExpiration' in source + assert 'abortIncompleteMultipartUploadAfter' in source + assert 'expiredObjectDeleteMarker: true' in source + assert 'AnyPrincipal' not in source + assert "sid: 'AllowEC2InstancesBucketPreflight'" in source + assert "actions: ['s3:GetBucketPolicyStatus', 's3:GetBucketAcl', 's3:ListBucket']" in source + assert "actions: ['s3:PutObject']" in source + assert 'DescribeUserPoolClient' not in source + assert 'ClientSecretRetriever' not in source + + +def test_deploy_script_validates_instance_upload_policy(): + deploy = (Path(__file__).parents[1] / 'deploy.sh').read_text() + assert 'Validating ECS instance upload permissions' in deploy + assert "required_bucket_actions = {'s3:GetBucketAcl', 's3:GetBucketPolicyStatus', 's3:ListBucket'}" in deploy + assert "required_object_actions = {'s3:PutObject'}" in deploy + + +def test_unzip_pipeline_supports_managed_tgz_in_canonical_namespace(): + source = (Path(__file__).parents[1] / 'src' / 'ecs-log-gateway-construct-v2.ts').read_text() + assert "{ suffix: '.tgz' }" in source + assert 'from urllib.parse import unquote_plus' in source + assert 'MANAGED_BUNDLE_RE = re.compile' in source + assert 'return f"ecs_{instance_id}/{execution_id}/extracted/"' in source + assert "key.lower().endswith('.tgz')" in source + assert 'Validating archive extraction notifications' in ( + Path(__file__).parents[1] / 'deploy.sh' + ).read_text() + + +def test_tool_schemas_bind_generic_s3_access_to_instance(): + source = (Path(__file__).parents[1] / 'src' / 'ecs-log-gateway-construct-v2.ts').read_text() + assert source.count("Required: ['instanceId', 'logKey']") >= 2 + assert "executionIds is not accepted; batchId is required" in ( + Path(__file__).parents[1] / 'src' / 'lambda' / 'ecs-log-automation.py' + ).read_text() + +def test_batch_hard_cap_rejects_more_than_fifteen(): + result = mod.batch_collect({ + 'clusterName': 'cluster', 'dryRun': True, 'maxTotalCollections': 16, + }) + assert result['statusCode'] == 400 + + +def test_batch_excludes_non_active_container_instances(monkeypatch): + class Paginator: + def paginate(self, **kwargs): + return [{'containerInstanceArns': ['ci-draining']}] + + class Ecs: + def get_paginator(self, name): + return Paginator() + + def describe_container_instances(self, **kwargs): + return {'containerInstances': [{ + 'ec2InstanceId': INSTANCE, 'status': 'DRAINING', + 'agentConnected': True, 'runningTasksCount': 0, + }]} + + monkeypatch.setattr( + mod, 'get_regional_client', + lambda service, region: Ecs() if service == 'ecs' else object(), + ) + result = body(mod.batch_collect({ + 'clusterName': 'cluster', 'filter': 'all', 'dryRun': True, + })) + assert result['totalInstances'] == 0 + assert result['plannedCollections'] == 0 diff --git a/mcp/ecs-instance-log-mcp/tests/test_tcpdump_approval.py b/mcp/ecs-instance-log-mcp/tests/test_tcpdump_approval.py new file mode 100644 index 0000000..e93f1d0 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tests/test_tcpdump_approval.py @@ -0,0 +1,184 @@ +import json +from datetime import datetime, timezone +from pathlib import Path +from .test_tcpdump_security import mod, TASK, INSTANCE, body + +APPROVER = 'arn:aws:iam::123456789012:role/Approver' +COMMAND = '12345678-1234-1234-1234-123456789abc' + + +def capture_metadata(instance_id=INSTANCE, command_id=COMMAND): + prefix = f'tcpdump/{instance_id}/20250101T000000Z' + return { + 'commandId': command_id, + 'instanceId': instance_id, + 'region': 'us-east-1', + 'startedAt': '20250101T000000Z', + 's3Key': f'{prefix}/capture.pcap', + 's3KeyTxt': f'{prefix}/capture_summary.txt', + 's3KeyStats': f'{prefix}/capture_stats.json', + } + + +def install_wrapper_provenance(monkeypatch): + monkeypatch.setattr(mod, 'get_execution_provenance', lambda execution_id: { + 'executionId': execution_id, 'tool': 'tcpdump_capture', + 'instanceId': INSTANCE, 'instanceIds': [], 'region': 'us-east-1', + 'expectedDocument': 'ecs-tcpdump-approval', + }) + monkeypatch.setattr(mod, 'validate_execution_details', lambda *args: None) + + +def test_tcpdump_requires_task_and_confirmation(monkeypatch): + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', {'tcpdump_capture'}) + assert mod.tcpdump_capture({'instanceId': INSTANCE})['statusCode'] == 400 + result = mod.tcpdump_capture({'instanceId': INSTANCE, 'taskId': TASK}) + assert body(result)['details']['requiresConfirmation'] is True + + +def test_capture_starts_approval_not_command(monkeypatch): + calls = [] + class Ssm: + def start_automation_execution(self, **kwargs): + calls.append(('automation', kwargs)) + return {'AutomationExecutionId': 'tcp-approval'} + def send_command(self, **kwargs): calls.append(('command', kwargs)) + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', {'tcpdump_capture'}) + monkeypatch.setattr(mod, 'REQUIRE_COLLECTION_APPROVAL', True) + monkeypatch.setattr(mod, 'TCPDUMP_APPROVAL_DOCUMENT', 'ecs-tcpdump-approval') + monkeypatch.setattr(mod, 'APPROVAL_APPROVERS', [APPROVER]) + monkeypatch.setattr(mod, 'resolve_and_validate_region', lambda args, iid=None: ('us-east-1', None)) + monkeypatch.setattr(mod, 'validate_ecs_instance', lambda *args: None) + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: Ssm()) + monkeypatch.setattr(mod, '_store_tcpdump_metadata', lambda *args: None) + monkeypatch.setattr(mod, 'store_execution_region', lambda *args: None) + monkeypatch.setattr(mod, 'store_execution_provenance', lambda *args, **kwargs: True) + monkeypatch.setattr(mod, 'notify_approvers', lambda *args: None) + result = mod.tcpdump_capture({'instanceId': INSTANCE, 'taskId': TASK, + 'containerName': 'web', 'confirmCapture': True}) + assert body(result)['status'] == 'pending_approval' + assert [kind for kind, _ in calls] == ['automation'] + parameters = calls[0][1]['Parameters'] + assert parameters['CaptureScope'] == [f'task/{TASK}/container/web'] + assert {'AutomationAssumeRole', 'Approvers', 'SNSTopicArn'}.isdisjoint(parameters) + script = parameters['Commands'][0] + assert '{{' not in script + + calls.clear() + result = mod.tcpdump_capture({ + 'instanceId': INSTANCE, 'taskId': TASK, 'confirmCapture': True}) + assert body(result)['status'] == 'pending_approval' + assert calls[0][1]['Parameters']['CaptureScope'] == [ + f'task/{TASK}/container/auto'] + + +def test_analyze_rejects_cross_instance_metadata(monkeypatch): + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', {'tcpdump_analyze'}) + monkeypatch.setattr( + mod, '_read_tcpdump_metadata', + lambda key: capture_metadata('i-0aaaaaaaaaaaaaaaa'), + ) + result = mod.tcpdump_analyze({'instanceId': INSTANCE, 'commandId': COMMAND}) + assert result['statusCode'] == 403 + + +def test_analyze_rejects_missing_command_metadata(monkeypatch): + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', {'tcpdump_analyze'}) + monkeypatch.setattr(mod, '_read_tcpdump_metadata', lambda key: None) + result = mod.tcpdump_analyze({'instanceId': INSTANCE, 'commandId': COMMAND}) + assert result['statusCode'] == 404 + assert 'refusing latest-capture fallback' in body(result)['error'] + + +def test_analyze_rejects_artifact_outside_instance_prefix(monkeypatch): + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', {'tcpdump_analyze'}) + metadata = capture_metadata() + metadata['s3Key'] = 'tcpdump/other-instance/t/capture.pcap' + monkeypatch.setattr(mod, '_read_tcpdump_metadata', lambda key: metadata) + assert mod.tcpdump_analyze({ + 'instanceId': INSTANCE, 'commandId': COMMAND})['statusCode'] == 403 + + +def test_analyze_requires_valid_command_id(monkeypatch): + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', {'tcpdump_analyze'}) + assert mod.tcpdump_analyze({'instanceId': INSTANCE})['statusCode'] == 400 + assert mod.tcpdump_analyze({ + 'instanceId': INSTANCE, 'commandId': 'latest'})['statusCode'] == 400 + + +def test_capture_poll_rejects_invalid_command_id(monkeypatch): + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', {'tcpdump_capture'}) + result = mod.tcpdump_capture({'instanceId': INSTANCE, 'commandId': 'bad'}) + assert result['statusCode'] == 400 + + +def test_wrapper_terminal_without_command_id_fails(monkeypatch): + wrapper = { + 'AutomationExecutionStatus': 'Failed', + 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'Success'}, + {'StepName': 'runTcpdump', 'StepStatus': 'Failed'}, + ], + } + class Ssm: + def get_automation_execution(self, **kwargs): + return {'AutomationExecution': wrapper} + install_wrapper_provenance(monkeypatch) + monkeypatch.setattr(mod, 'get_execution_region', lambda execution_id: 'us-east-1') + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: Ssm()) + result = mod._poll_tcpdump_wrapper('wrapper', INSTANCE, {}) + assert result['statusCode'] == 500 + assert 'without a command ID' in body(result)['error'] + + +def test_wrapper_refreshes_started_at_from_run_step(monkeypatch): + started = datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + wrapper = { + 'AutomationExecutionStatus': 'Success', + 'StepExecutions': [ + {'StepName': 'waitForHumanApproval', 'StepStatus': 'Success'}, + {'StepName': 'runTcpdump', 'StepStatus': 'Success', + 'ExecutionStartTime': started, + 'Outputs': {'CommandId': [COMMAND]}}, + ], + } + class Ssm: + def get_automation_execution(self, **kwargs): + return {'AutomationExecution': wrapper} + stored = [] + install_wrapper_provenance(monkeypatch) + monkeypatch.setattr(mod, 'get_execution_region', lambda execution_id: 'us-east-1') + monkeypatch.setattr(mod, 'get_regional_client', lambda *args: Ssm()) + monkeypatch.setattr(mod, '_read_tcpdump_metadata', lambda key: { + **capture_metadata(), 'startedAt': '20240101T000000Z'}) + monkeypatch.setattr(mod, '_store_tcpdump_metadata', + lambda key, metadata: stored.append((key, dict(metadata)))) + monkeypatch.setattr(mod, '_poll_tcpdump_status', lambda *args: { + 'statusCode': 200, 'body': json.dumps({ + 'status': 'in_progress', 'commandId': COMMAND})}) + result = body(mod._poll_tcpdump_wrapper('wrapper', INSTANCE, {})) + assert result['commandId'] == COMMAND + assert len(stored) == 2 + assert all(item[1]['startedAt'] == '2025-01-02T03:04:05Z' for item in stored) + + +def test_cdk_capture_scope_contract_matches_lambda_format(): + source = (Path(__file__).parents[1] / 'src' / 'ecs-log-gateway-construct-v2.ts').read_text() + assert "allowedPattern: '^task/[0-9a-f]{32}/container/[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$'" in source + + +def test_analyze_rejects_command_metadata_mismatch(monkeypatch): + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', {'tcpdump_analyze'}) + metadata = capture_metadata(command_id='aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa') + monkeypatch.setattr(mod, '_read_tcpdump_metadata', lambda key: metadata) + assert mod.tcpdump_analyze({ + 'instanceId': INSTANCE, 'commandId': COMMAND})['statusCode'] == 403 + + +def test_analyze_rejects_incomplete_artifact_metadata(monkeypatch): + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', {'tcpdump_analyze'}) + metadata = capture_metadata() + metadata.pop('s3KeyStats') + monkeypatch.setattr(mod, '_read_tcpdump_metadata', lambda key: metadata) + assert mod.tcpdump_analyze({ + 'instanceId': INSTANCE, 'commandId': COMMAND})['statusCode'] == 403 diff --git a/mcp/ecs-instance-log-mcp/tests/test_tcpdump_security.py b/mcp/ecs-instance-log-mcp/tests/test_tcpdump_security.py new file mode 100644 index 0000000..c455388 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tests/test_tcpdump_security.py @@ -0,0 +1,67 @@ +import json +import os +import sys +import pytest + +os.environ.setdefault('LOGS_BUCKET_NAME', 'test-bucket') +os.environ.setdefault('SSM_AUTOMATION_ROLE_ARN', 'arn:aws:iam::123456789012:role/test') +os.environ.setdefault('AWS_REGION', 'us-east-1') +os.environ.setdefault('ALLOWED_REGIONS', 'us-east-1,us-west-2') +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src', 'lambda')) +mod = __import__('ecs-log-automation') +TASK = '0123456789abcdef0123456789abcdef' +INSTANCE = 'i-0123456789abcdef0' + + +def body(result): return json.loads(result['body']) + + +def test_restricted_tools_fail_closed(monkeypatch): + monkeypatch.setattr(mod, 'ENABLED_RESTRICTED_TOOLS', set()) + assert mod.validate_tool_authorization('tcpdump_capture')['statusCode'] == 403 + + +@pytest.mark.parametrize('value,expected', [ + (TASK, TASK), + ('arn:aws:ecs:us-east-1:123456789012:task/cluster/' + TASK, TASK), +]) +def test_task_id_normalization(value, expected): + assert mod.normalize_ecs_task_id(value) == (expected, None) + + +@pytest.mark.parametrize('value', ['short', TASK + '0', 'prefix-' + TASK, + 'arn:aws:ecs:us-east-1:123:task/' + TASK]) +def test_task_id_rejects_non_exact_values(value): + assert mod.normalize_ecs_task_id(value)[1] + + +@pytest.mark.parametrize('expr', ['', 'port 53', 'udp port 53', + 'host 10.0.0.1 and port 443', 'net 10.0.0.0/16']) +def test_bpf_safe(expr): assert mod.validate_bpf_filter(expr) is None + + +@pytest.mark.parametrize('expr', ['port 53; id', 'port `id`', 'port 53 $(id)', + 'port 53\nreboot', 'tcp[tcpflags] & 2 != 0']) +def test_bpf_rejects_shell_and_complex_flags(expr): + assert mod.validate_bpf_filter(expr) + + +def test_script_is_exact_task_namespace_and_has_no_template_sequence(): + keys = {'pcap': f'tcpdump/{INSTANCE}/t/capture.pcap', + 'text': f'tcpdump/{INSTANCE}/t/capture.txt', + 'stats': f'tcpdump/{INSTANCE}/t/stats.json'} + script = mod._build_ecs_tcpdump_script(TASK, 'web', 'any', 'port 443', 30, 't', keys) + assert '{{' not in script + assert "resolved == task_id" in script + assert "c.get('Name') == requested" in script + assert 'explicit containerName did not match exactly once' in script + assert 'explicit containerName must identify a RUNNING application container' in script + assert 'containerName is required unless exactly one RUNNING application container is eligible' in script + assert 'stat -Lc %i /proc/1/ns/net' in script + assert 'process start time changed' in script + assert 'network namespace inode changed' in script + assert 'nsenter -n -t "$TARGET_PID" -- tcpdump' in script + assert 'docker inspect --format' not in script + assert "data[0].get('Id') == sys.argv[1]" in script + assert 'containers ls -q' not in script + assert 'yum install' not in script and 'apt-get install' not in script diff --git a/mcp/ecs-instance-log-mcp/tests/test_tool_validation_wiring.py b/mcp/ecs-instance-log-mcp/tests/test_tool_validation_wiring.py new file mode 100644 index 0000000..64cdd0e --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tests/test_tool_validation_wiring.py @@ -0,0 +1,142 @@ +""" +Unit tests for MCP tool function validation wiring. +Verifies that tool functions call region and ECS instance validation. +Mirrors eks-node-log-mcp/tests/test_tool_validation_wiring.py for ECS. +""" +import os +import sys +import json +import pytest +from unittest.mock import patch, MagicMock + +# Set required env vars before importing the module +os.environ.setdefault('LOGS_BUCKET_NAME', 'test-bucket') +os.environ.setdefault('SSM_AUTOMATION_ROLE_ARN', 'arn:aws:iam::123456789012:role/test') +os.environ.setdefault('AWS_REGION', 'us-east-1') +os.environ.setdefault('AWS_DEFAULT_REGION', 'us-east-1') +os.environ.setdefault('ALLOWED_REGIONS', 'us-east-1,us-west-2') + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src', 'lambda')) + + +class TestValidationFunctionsExist: + """Tests that all validation functions are defined in the module.""" + + def test_validation_functions_exist(self): + mod = __import__('ecs-log-automation') + assert callable(getattr(mod, 'validate_region', None)) + assert callable(getattr(mod, 'resolve_and_validate_region', None)) + assert callable(getattr(mod, 'validate_ecs_instance', None)) + assert callable(getattr(mod, '_parse_presigned_url_expiration', None)) + assert callable(getattr(mod, '_parse_pcap_presigned_url_expiration', None)) + assert callable(getattr(mod, '_parse_max_pcap_bytes', None)) + assert callable(getattr(mod, 'validate_tool_authorization', None)) + assert callable(getattr(mod, 'validate_bpf_filter', None)) + assert callable(getattr(mod, 'normalize_ecs_task_id', None)) + assert callable(getattr(mod, '_build_ecs_tcpdump_script', None)) + assert callable(getattr(mod, '_poll_tcpdump_wrapper', None)) + assert callable(getattr(mod, 'TimeWindowResolver', None)) or hasattr(mod, 'TimeWindowResolver') + + +class TestRegionValidation: + """Tests that validate_region works correctly.""" + + def test_validate_region_accepts_allowed(self): + mod = __import__('ecs-log-automation') + result = mod.validate_region('us-east-1') + assert result is None + + def test_validate_region_rejects_disallowed(self): + mod = __import__('ecs-log-automation') + result = mod.validate_region('ap-south-1') + assert result is not None + assert result['statusCode'] == 403 + body = json.loads(result['body']) + assert 'not permitted' in body['error'] + + def test_allowed_regions_parsed_from_env(self): + mod = __import__('ecs-log-automation') + assert 'us-east-1' in mod.ALLOWED_REGIONS + assert 'us-west-2' in mod.ALLOWED_REGIONS + + +class TestPresignedUrlExpiration: + """Tests that presigned URL expiration is configurable.""" + + def test_default_expiration_is_900(self): + mod = __import__('ecs-log-automation') + # Default is 900 for ECS (15 minutes) + assert mod.PRESIGNED_URL_EXPIRATION == 900 + + def test_parse_valid_value(self): + mod = __import__('ecs-log-automation') + with patch.dict(os.environ, {'PRESIGNED_URL_EXPIRATION_SECONDS': '120'}): + result = mod._parse_presigned_url_expiration() + assert result == 120 + + def test_parse_invalid_value_defaults(self): + mod = __import__('ecs-log-automation') + with patch.dict(os.environ, {'PRESIGNED_URL_EXPIRATION_SECONDS': 'abc'}): + result = mod._parse_presigned_url_expiration() + assert result == 900 + + def test_parse_zero_defaults(self): + mod = __import__('ecs-log-automation') + with patch.dict(os.environ, {'PRESIGNED_URL_EXPIRATION_SECONDS': '0'}): + result = mod._parse_presigned_url_expiration() + assert result == 900 + + def test_parse_negative_defaults(self): + mod = __import__('ecs-log-automation') + with patch.dict(os.environ, {'PRESIGNED_URL_EXPIRATION_SECONDS': '-5'}): + result = mod._parse_presigned_url_expiration() + assert result == 900 + + +class TestTimeWindowResolver: + """Tests that TimeWindowResolver works correctly.""" + + def test_default_window_is_10_minutes(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({}) + assert window['resolution_reason'].startswith('no incident time') + delta = window['window_end_utc'] - window['window_start_utc'] + assert 9 * 60 <= delta.total_seconds() <= 11 * 60 # ~10 minutes + + def test_explicit_window(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({ + 'start_time': '2025-01-15T10:00:00Z', + 'end_time': '2025-01-15T10:30:00Z', + }) + assert 'explicit' in window['resolution_reason'] + delta = window['window_end_utc'] - window['window_start_utc'] + assert delta.total_seconds() == 30 * 60 + + def test_incident_time_padding(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({ + 'incident_time': '2025-01-15T10:00:00Z', + }) + assert 'padding' in window['resolution_reason'] + delta = window['window_end_utc'] - window['window_start_utc'] + assert delta.total_seconds() == 10 * 60 # +/- 5 min = 10 min + + def test_max_window_clamped(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({ + 'start_time': '2025-01-01T00:00:00Z', + 'end_time': '2025-01-10T00:00:00Z', + }) + assert 'clamped' in window['resolution_reason'] + delta = window['window_end_utc'] - window['window_start_utc'] + assert delta.total_seconds() <= 24 * 3600 + + def test_swapped_start_end(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({ + 'start_time': '2025-01-15T11:00:00Z', + 'end_time': '2025-01-15T10:00:00Z', + }) + assert 'swapped' in window['resolution_reason'] + assert window['window_start_utc'] < window['window_end_utc'] diff --git a/mcp/ecs-instance-log-mcp/tsconfig.json b/mcp/ecs-instance-log-mcp/tsconfig.json new file mode 100644 index 0000000..22a6ee1 --- /dev/null +++ b/mcp/ecs-instance-log-mcp/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020"], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "outDir": "./lib" + }, + "include": ["src/**/*", "bin/**/*"], + "exclude": ["node_modules", "cdk.out"] +} From 9bbdc3c7c89185401ca89051dbcdc177345963d4 Mon Sep 17 00:00:00 2001 From: Shyam Kulkarni Date: Fri, 14 Aug 2026 13:07:29 +0530 Subject: [PATCH 2/2] chore(mcp): Align ECS package with repo Apache-2.0 license --- mcp/ecs-instance-log-mcp/LICENSE | 16 ---------------- mcp/ecs-instance-log-mcp/README.md | 4 ---- mcp/ecs-instance-log-mcp/package.json | 2 +- 3 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 mcp/ecs-instance-log-mcp/LICENSE diff --git a/mcp/ecs-instance-log-mcp/LICENSE b/mcp/ecs-instance-log-mcp/LICENSE deleted file mode 100644 index 56a66b6..0000000 --- a/mcp/ecs-instance-log-mcp/LICENSE +++ /dev/null @@ -1,16 +0,0 @@ -MIT No Attribution - -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/mcp/ecs-instance-log-mcp/README.md b/mcp/ecs-instance-log-mcp/README.md index 23e0cce..e280059 100644 --- a/mcp/ecs-instance-log-mcp/README.md +++ b/mcp/ecs-instance-log-mcp/README.md @@ -383,7 +383,3 @@ cdk destroy > The S3 bucket has `removalPolicy: DESTROY` with `autoDeleteObjects: true`, so it will be cleaned up with the stack. --- - -## License - -This project is licensed under the MIT No Attribution (MIT-0) License. See the [LICENSE](LICENSE) file. diff --git a/mcp/ecs-instance-log-mcp/package.json b/mcp/ecs-instance-log-mcp/package.json index 3c6f5cf..29ebea3 100644 --- a/mcp/ecs-instance-log-mcp/package.json +++ b/mcp/ecs-instance-log-mcp/package.json @@ -24,7 +24,7 @@ "troubleshooting" ], "author": "AWS", - "license": "MIT-0", + "license": "Apache-2.0", "devDependencies": { "@types/node": "^20.0.0", "aws-cdk": "^2.170.0",