From fbe90941d233d14b7b0f6afcb7a8577b7868eb21 Mon Sep 17 00:00:00 2001 From: Mridul Chopra Date: Mon, 10 Aug 2026 11:14:04 -0400 Subject: [PATCH] feat(mcp): Add ECS Instance Log MCP Server MCP Server for AWS DevOps Agent to collect and analyze diagnostic logs from ECS container instances using SSM Automation. Covers ECS agent, Docker/containerd, container logs, system logs, dmesg, networking, cgroups, instance metadata, and GPU diagnostics. Features: - 19 MCP tools across 5 tiers (Core, Analysis, Cluster, Capture, SOPs) - Hub-and-spoke cross-region design (one deployment, all regions) - 36 structured runbooks for ECS-specific failure categories - KMS encryption, Cognito OAuth2, region allow-list, ECS instance validation - BucketDeployment for auto-deploying SOPs - Interactive deploy script with cluster/role auto-detection - Property-based tests with hypothesis Architecturally analogous to mcp/aws-eks-node-diagnostics-mcp but tailored for ECS EC2 container instances. --- mcp/aws-ecs-instance-log-mcp/.gitignore | 42 + mcp/aws-ecs-instance-log-mcp/LICENSE | 16 + mcp/aws-ecs-instance-log-mcp/README.md | 349 ++ mcp/aws-ecs-instance-log-mcp/bin/app.ts | 23 + mcp/aws-ecs-instance-log-mcp/cdk.json | 22 + mcp/aws-ecs-instance-log-mcp/deploy.sh | 480 ++ .../docs/ARCHITECTURE.md | 258 + mcp/aws-ecs-instance-log-mcp/get-config.sh | 52 + .../package-lock.json | 470 ++ mcp/aws-ecs-instance-log-mcp/package.json | 37 + .../requirements-dev.txt | 4 + .../runbooks/A1-task-startup-resource-init.md | 78 + .../A2-task-startup-container-runtime.md | 52 + .../sops/runbooks/B1-ecr-image-pull-auth.md | 53 + .../sops/runbooks/B2-image-not-found.md | 48 + .../sops/runbooks/B3-docker-hub-rate-limit.md | 42 + .../C1-task-execution-role-permissions.md | 53 + .../runbooks/C2-secrets-manager-retrieval.md | 50 + .../sops/runbooks/D1-oom-kill-memory.md | 53 + .../sops/runbooks/D2-disk-space-exhaustion.md | 41 + .../sops/runbooks/D3-cpu-throttling.md | 40 + .../runbooks/E1-eni-allocation-subnet-ip.md | 49 + .../runbooks/E2-dns-resolution-failures.md | 44 + .../sops/runbooks/E3-connection-timeout.md | 45 + .../runbooks/F1-container-health-check.md | 47 + .../sops/runbooks/F2-elb-target-health.md | 43 + .../sops/runbooks/G1-agent-disconnected.md | 52 + .../G2-instance-registration-failure.md | 49 + .../sops/runbooks/H1-cloudwatch-log-driver.md | 49 + .../runbooks/I1-deployment-circuit-breaker.md | 48 + .../runbooks/J1-oci-runtime-entrypoint.md | 53 + .../K1-spot-interruption-instance-draining.md | 81 + .../K10-api-throttling-service-quotas.md | 86 + .../K11-fargate-metadata-credential-errors.md | 87 + .../K12-essential-container-exited-nonzero.md | 96 + .../runbooks/K13-windows-container-issues.md | 91 + .../runbooks/K14-task-latency-performance.md | 94 + .../K15-docker-daemon-agent-errors.md | 88 + .../runbooks/K2-task-placement-failures.md | 90 + .../K3-service-steady-state-failures.md | 87 + .../K4-service-auto-scaling-issues.md | 87 + .../sops/runbooks/K5-ecs-exec-failures.md | 83 + .../K6-service-connect-discovery-failures.md | 90 + .../K7-ebs-efs-volume-mount-failures.md | 86 + .../sops/runbooks/K8-task-stuck-pending.md | 84 + .../K9-fargate-platform-ephemeral-storage.md | 88 + .../runbooks/Z1-general-troubleshooting.md | 55 + .../src/ecs-log-gateway-construct-v2.ts | 1450 +++++ .../src/ecs-log-gateway-stack-v2.ts | 31 + mcp/aws-ecs-instance-log-mcp/src/index.ts | 2 + .../src/lambda/ecs-log-automation.py | 4837 +++++++++++++++++ .../tests/__init__.py | 0 .../tests/conftest.py | 12 + .../tests/test_instance_validation.py | 123 + .../tests/test_region_validation.py | 111 + .../tests/test_tool_validation_wiring.py | 135 + mcp/aws-ecs-instance-log-mcp/tsconfig.json | 24 + 57 files changed, 10840 insertions(+) create mode 100644 mcp/aws-ecs-instance-log-mcp/.gitignore create mode 100644 mcp/aws-ecs-instance-log-mcp/LICENSE create mode 100644 mcp/aws-ecs-instance-log-mcp/README.md create mode 100644 mcp/aws-ecs-instance-log-mcp/bin/app.ts create mode 100644 mcp/aws-ecs-instance-log-mcp/cdk.json create mode 100755 mcp/aws-ecs-instance-log-mcp/deploy.sh create mode 100644 mcp/aws-ecs-instance-log-mcp/docs/ARCHITECTURE.md create mode 100755 mcp/aws-ecs-instance-log-mcp/get-config.sh create mode 100644 mcp/aws-ecs-instance-log-mcp/package-lock.json create mode 100644 mcp/aws-ecs-instance-log-mcp/package.json create mode 100644 mcp/aws-ecs-instance-log-mcp/requirements-dev.txt create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/A1-task-startup-resource-init.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/A2-task-startup-container-runtime.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/B1-ecr-image-pull-auth.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/B2-image-not-found.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/B3-docker-hub-rate-limit.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/C1-task-execution-role-permissions.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/C2-secrets-manager-retrieval.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/D1-oom-kill-memory.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/D2-disk-space-exhaustion.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/D3-cpu-throttling.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/E1-eni-allocation-subnet-ip.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/E2-dns-resolution-failures.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/E3-connection-timeout.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/F1-container-health-check.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/F2-elb-target-health.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/G1-agent-disconnected.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/G2-instance-registration-failure.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/H1-cloudwatch-log-driver.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/I1-deployment-circuit-breaker.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/J1-oci-runtime-entrypoint.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K1-spot-interruption-instance-draining.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K10-api-throttling-service-quotas.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K11-fargate-metadata-credential-errors.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K12-essential-container-exited-nonzero.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K13-windows-container-issues.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K14-task-latency-performance.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K15-docker-daemon-agent-errors.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K2-task-placement-failures.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K3-service-steady-state-failures.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K4-service-auto-scaling-issues.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K5-ecs-exec-failures.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K6-service-connect-discovery-failures.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K7-ebs-efs-volume-mount-failures.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K8-task-stuck-pending.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/K9-fargate-platform-ephemeral-storage.md create mode 100644 mcp/aws-ecs-instance-log-mcp/sops/runbooks/Z1-general-troubleshooting.md create mode 100644 mcp/aws-ecs-instance-log-mcp/src/ecs-log-gateway-construct-v2.ts create mode 100644 mcp/aws-ecs-instance-log-mcp/src/ecs-log-gateway-stack-v2.ts create mode 100644 mcp/aws-ecs-instance-log-mcp/src/index.ts create mode 100644 mcp/aws-ecs-instance-log-mcp/src/lambda/ecs-log-automation.py create mode 100644 mcp/aws-ecs-instance-log-mcp/tests/__init__.py create mode 100644 mcp/aws-ecs-instance-log-mcp/tests/conftest.py create mode 100644 mcp/aws-ecs-instance-log-mcp/tests/test_instance_validation.py create mode 100644 mcp/aws-ecs-instance-log-mcp/tests/test_region_validation.py create mode 100644 mcp/aws-ecs-instance-log-mcp/tests/test_tool_validation_wiring.py create mode 100644 mcp/aws-ecs-instance-log-mcp/tsconfig.json diff --git a/mcp/aws-ecs-instance-log-mcp/.gitignore b/mcp/aws-ecs-instance-log-mcp/.gitignore new file mode 100644 index 0000000..38caf32 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/.gitignore @@ -0,0 +1,42 @@ +# Dependencies +node_modules/ + +# Build output +lib/ +*.js +*.d.ts + +# CDK output +cdk.out/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log +npm-debug.log* + +# Test coverage +coverage/ + +# Python cache +__pycache__/ +*.pyc +.pytest_cache/ + +# Hypothesis test framework cache +.hypothesis/ + +# Deployment-specific outputs (contain account IDs, secrets) +mcp-config.txt +cdk-outputs.json + +# Lambda packaging +lambda.zip diff --git a/mcp/aws-ecs-instance-log-mcp/LICENSE b/mcp/aws-ecs-instance-log-mcp/LICENSE new file mode 100644 index 0000000..56a66b6 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/LICENSE @@ -0,0 +1,16 @@ +MIT No Attribution + +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/mcp/aws-ecs-instance-log-mcp/README.md b/mcp/aws-ecs-instance-log-mcp/README.md new file mode 100644 index 0000000..fd68f10 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/README.md @@ -0,0 +1,349 @@ +# ECS Instance Log MCP + +> **⚠️ Proof of Concept (POC):** This project is a proof of concept and should be tested in non-production environments first. Validate thoroughly in a staging or development account before using with production workloads. + +MCP Server for AWS DevOps Agent to collect and analyze diagnostic logs from ECS container instances using SSM Automation. Covers ECS agent, Docker/containerd, container logs, system logs, dmesg, networking, cgroups, instance metadata, and GPU diagnostics — artifacts that live on the instance OS and aren't accessible through the ECS API or CloudWatch. + +> **Want to understand the internals?** See [Architecture & Design](docs/ARCHITECTURE.md) for a deep dive into how the components work, data flows, tool design, and security model. + +--- + +## Prerequisites + +### 1. Node.js (v18.x or later) + +**macOS (Homebrew):** +```bash +brew install node +``` + +**Linux (Ubuntu/Debian):** +```bash +curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - +sudo apt-get install -y nodejs +``` + +### 2. AWS CLI v2 + +**macOS:** +```bash +brew install awscli +``` + +**Linux:** +```bash +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip awscliv2.zip +sudo ./aws/install +``` + +### 3. AWS CDK CLI + +```bash +npm install -g aws-cdk +``` + +### 4. Python 3 + +Most systems have it pre-installed: +```bash +python3 --version +``` + +### 5. AWS Credentials + +You need permissions to create IAM Roles, Lambda Functions, S3 Buckets, KMS Keys, Cognito User Pools, and BedrockAgentCore Gateways. + +```bash +aws configure +# Or use AWS SSO: +aws sso login --profile your-profile +export AWS_PROFILE=your-profile +``` + +--- + +## Deployment + +```bash +# Clone the repository +git clone +cd ecs-instance-log-mcp + +# Make the script executable +chmod +x deploy.sh + +# Deploy (defaults to us-east-1) +./deploy.sh + +# Or deploy to a specific region +AWS_REGION=us-west-2 ./deploy.sh +``` + +### Interactive Deployment Flow + +The deploy script walks you through three interactive prompts: + +**Step 1 — Region selection:** +``` +Which AWS regions should be scanned for ECS clusters? + + 1) All enabled regions + 2) Current deploy region only (us-east-1) + 3) Enter a specific region + +Select [1/2/3] (default: 1): +``` + +**Step 2 — Cluster selection:** +``` +Found 4 ECS cluster(s): + + 1) prod-cluster (us-east-1) + 2) dev-cluster (us-east-1) + 3) analytics (us-west-2) + 4) eu-cluster (eu-west-1) + + a) All clusters + +Select clusters (comma-separated numbers, or 'a' for all) [default: a]: +``` + +**Step 3 — Instance role selection:** +``` +Found 3 unique container instance role(s): + + 1) arn:aws:iam::123456789012:role/ecsInstanceRole + └─ ecsInstanceRole (prod-cluster / us-east-1) + 2) arn:aws:iam::123456789012:role/ecs-dev-role + └─ ecs-dev-role (dev-cluster / us-east-1) + 3) arn:aws:iam::123456789012:role/ecs-eu-role + └─ ecs-eu-role (eu-cluster / eu-west-1) + + a) All roles + +Select instance roles (comma-separated numbers, or 'a' for all) [default: a]: +``` + +**Fallback — Manual ARN entry:** + +If no ECS clusters or instance roles are found (e.g., Fargate-only account), the script prompts for manual entry: +``` +WARNING: No ECS clusters found in the selected region(s). + +Would you like to manually enter instance role ARN(s)? [y/N]: y +Enter comma-separated role ARNs (e.g. arn:aws:iam::123456789012:role/ecsInstanceRole): +> +``` + +### Non-Interactive / CI Mode + +Skip all prompts by providing role ARNs directly: + +```bash +# Via environment variable +ECS_INSTANCE_ROLE_ARNS="arn:aws:iam::123456789012:role/ecsInstanceRole" ./deploy.sh + +# Or as a positional argument +./deploy.sh EcsInstanceLogMcpStack arn:aws:iam::123456789012:role/ecsInstanceRole + +# Multiple roles (comma-separated) +ECS_INSTANCE_ROLE_ARNS="arn:aws:iam::123456789012:role/Role1,arn:aws:iam::123456789012:role/Role2" ./deploy.sh +``` + +### What Gets Deployed + +| Resource | Purpose | +|----------|---------| +| S3 Bucket (KMS encrypted) | Stores collected log bundles | +| S3 Bucket (SOPs) | Stores 36 runbooks, auto-deployed via CDK | +| Lambda (ECS Log Automation) | Handles all 19 MCP tool invocations | +| Lambda (Unzip) | Auto-extracts uploaded archives | +| Lambda (Findings Indexer) | Pre-indexes errors for fast retrieval | +| SSM Automation Role | Runs log collection on EC2 instances | +| Cognito User Pool | OAuth2 authentication for MCP Gateway | +| BedrockAgentCore Gateway | MCP protocol endpoint | +| KMS Key | Encrypts all data at rest | + +--- + +## Post-Deployment: ECS Instance IAM Setup + +### What's Automatic + +If you selected instance roles during the interactive deploy flow (or passed them via `ECS_INSTANCE_ROLE_ARNS`), the CDK stack automatically grants: + +- S3 bucket policy: `s3:PutObject`, `s3:GetBucketPolicyStatus`, `s3:GetBucketAcl` on the logs bucket +- KMS key policy: `kms:GenerateDataKey`, `kms:Encrypt` on the encryption key + +No manual S3 or KMS setup is needed for those roles. + +If no instance roles were provided during deployment, the stack falls back to an account-scoped policy (any principal in the account can upload). This is less restrictive but still functional. + +### What You May Still Need + +The only thing the CDK stack does not attach is the SSM Agent managed policy. ECS-optimized AMIs include SSM Agent by default, but the IAM role needs the policy: + +```bash +# Only needed if not already attached +aws iam attach-role-policy \ + --role-name \ + --policy-arn arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore +``` + +### Adding Instance Roles After Deployment + +If you add new ECS clusters later, re-run the deploy script — it will detect the new instance roles and update the S3 bucket and KMS key policies automatically. + +Alternatively, pass the new roles directly: + +```bash +ECS_INSTANCE_ROLE_ARNS="arn:aws:iam::123456789012:role/ExistingRole,arn:aws:iam::123456789012:role/NewRole" ./deploy.sh +``` + +### Checklist Per Cluster + +- [ ] Instance role was selected during deployment (or added via re-deploy) +- [ ] Instance role has `AmazonSSMManagedInstanceCore` managed policy (for SSM Agent) +- [ ] SSM Agent is running on the instances (default on ECS-optimized AMIs) +- [ ] `AWSSupport-CollectECSInstanceLogs` SSM document exists in the target region + +--- + +## Configuration in DevOps Agent + +After deployment, the script outputs all values needed for the MCP Server configuration: + +| Setting | Value | +|---------|-------| +| MCP Server URL | `https://.gateway.bedrock-agentcore..amazonaws.com/mcp` | +| OAuth Client ID | Cognito Client ID from output | +| OAuth Client Secret | Cognito Client Secret from output | +| Token URL | `https://-.auth..amazoncognito.com/oauth2/token` | +| Scope | `ecs-log-gateway-id/gateway:read` | + +Values are also saved to `mcp-config.txt` for reference. + +--- + +## How It Works + +The server gives MCP-compatible agents the ability to collect full diagnostic bundles from ECS container instances, pre-index errors with severity classification, stream multi-GB log files without truncation, correlate events across log sources, run live tcpdump captures, compare instances, and follow structured runbooks — all through 19 MCP tools organized in 5 tiers. + +For a detailed walkthrough of the architecture, data flows, tool design, cross-region mechanics, security model, and anti-hallucination design, see: + +**[Architecture & Design →](docs/ARCHITECTURE.md)** + +### MCP Tools (Quick Reference) + +| Tier | Tools | Purpose | +|------|-------|---------| +| 1 — Core | `collect`, `status`, `validate`, `errors`, `read` | Log collection, findings, streaming | +| 2 — Analysis | `search`, `correlate`, `artifact`, `summarize`, `history` | Deep investigation, correlation, summaries | +| 3 — Cluster | `cluster_health`, `compare_instances`, `batch_collect`, `batch_status`, `network_diagnostics` | Multi-instance operations | +| 4 — Capture | `tcpdump_capture`, `tcpdump_analyze` | Live packet capture and analysis | +| 5 — SOPs | `list_sops`, `get_sop` | 36 structured runbooks | + +### Agent Workflow + +``` +collect → status (poll) → validate → errors → search → correlate → read → summarize +``` + +### Runbook Library (36 SOPs) + +| Category | Coverage | +|----------|----------| +| A — Task Startup | Resource initialization, container runtime failures | +| B — Image Pull | ECR auth, image not found, Docker Hub rate limits | +| C — IAM/Secrets | Task execution role, Secrets Manager retrieval | +| D — Resource Exhaustion | OOM kills, disk space, CPU throttling | +| E — Networking | ENI allocation, DNS resolution, connection timeouts | +| F — Health Checks | Container health checks, ELB target health | +| G — ECS Agent | Agent disconnected, instance registration failure | +| H — Logging | CloudWatch log driver configuration | +| I — Deployment | Circuit breaker triggers, rollbacks | +| J — Container Runtime | OCI runtime, entrypoint issues, architecture mismatch | +| K — Extended | Spot interruption, task placement, steady state, auto scaling, ECS Exec, Service Connect, volume mounts, stuck pending, Fargate, API throttling, Windows, performance, Docker daemon | +| Z — Catch-All | General troubleshooting | + +--- + +## Usage Examples + +### Basic Investigation +``` +Container instance i-0abc123def in us-west-2 has tasks failing to start. +Collect its logs and correlate what happened in the last 10 minutes. +``` + +### Cluster-Wide Triage +``` +We have a 50-instance ECS cluster and something is off. Do a dry run batch +collection first — show me which instances you'd sample. Then collect from +the unhealthy ones. +``` + +### Live Packet Capture +``` +Tasks on instance i-0abc123def can't reach the backend service. Run a 2-minute +tcpdump filtered on port 8080, then analyze — show me RST counts and retransmissions. +``` + +### Task-Level Capture +``` +Task abc123 on instance i-0abc123def has connection timeouts. Capture traffic +scoped to that task's network namespace for 60 seconds on port 443. +``` + +### SOP-Guided +``` +I don't know what's wrong — just investigate. List the available SOPs, run a +general triage, and follow whichever runbook matches. +``` + +--- + +## CloudFormation Outputs + +| Output | Description | +|--------|-------------| +| `GatewayId` | AgentCore Gateway ID | +| `GatewayUrl` | MCP Server URL | +| `CognitoUserPoolId` | Cognito User Pool ID | +| `CognitoClientId` | OAuth Client ID | +| `OAuthExchangeUrl` | OAuth Token URL | +| `OAuthScope` | OAuth Scope | +| `LogsBucketName` | S3 bucket for logs | +| `SOPBucketName` | S3 bucket for runbooks | +| `SSMAutomationRoleArn` | SSM Automation role ARN | +| `EncryptionKeyArn` | KMS key ARN | + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `collect` returns "document not found" | SSM document not in target region | Use a supported region or pass `region` explicitly | +| Upload step fails | Instance role missing S3/KMS permissions | Re-run deploy with the instance role selected | +| `status` returns wrong region | Region metadata not persisted | Pass `region` explicitly | +| Auto-detection times out | Instance in uncommon region | Pass `region` explicitly | +| `errors` returns empty | Findings indexer hasn't run yet | Wait a few seconds after `validate`, or use `search` | +| `tcpdump_capture` uploads fail | Instance role missing S3 PutObject | Re-run deploy with the instance role selected | + +--- + +## Cleanup + +```bash +cdk destroy +``` + +> The S3 bucket has `removalPolicy: DESTROY` with `autoDeleteObjects: true`, so it will be cleaned up with the stack. + +--- + +## License + +This project is licensed under the MIT No Attribution (MIT-0) License. See the [LICENSE](LICENSE) file. diff --git a/mcp/aws-ecs-instance-log-mcp/bin/app.ts b/mcp/aws-ecs-instance-log-mcp/bin/app.ts new file mode 100644 index 0000000..4a0fafc --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/bin/app.ts @@ -0,0 +1,23 @@ +#!/usr/bin/env node +import 'source-map-support/register'; +import * as cdk from 'aws-cdk-lib'; +import { EcsLogGatewayStackV2 } from '../src/ecs-log-gateway-stack-v2'; + +const app = new cdk.App(); + +new EcsLogGatewayStackV2(app, 'EcsInstanceLogMcpStack', { + description: 'ECS Instance Log MCP Server - Production-grade log collection for DevOps Agent with byte-range streaming and incident analysis', + env: { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION, + }, + gatewayProps: { + gatewayName: 'EcsInstanceLogMcpGW', + logRetentionDays: 1, + enableKmsEncryption: true, + ssmDefaultHostRoleArn: process.env.SSM_DEFAULT_HOST_ROLE_ARN, + ecsInstanceRoleArns: process.env.ECS_INSTANCE_ROLE_ARNS + ? process.env.ECS_INSTANCE_ROLE_ARNS.split(',').filter(Boolean) + : undefined, + }, +}); diff --git a/mcp/aws-ecs-instance-log-mcp/cdk.json b/mcp/aws-ecs-instance-log-mcp/cdk.json new file mode 100644 index 0000000..6760f7a --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/cdk.json @@ -0,0 +1,22 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/app.ts", + "watch": { + "include": ["**"], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": ["aws", "aws-cn"] + } +} diff --git a/mcp/aws-ecs-instance-log-mcp/deploy.sh b/mcp/aws-ecs-instance-log-mcp/deploy.sh new file mode 100755 index 0000000..76348aa --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/deploy.sh @@ -0,0 +1,480 @@ +#!/bin/bash +set -e + +# ECS Instance Log MCP - Deploy and Configure Script +# This script deploys the CDK stack and outputs all values needed for DevOps Agent configuration + +STACK_NAME="${1:-EcsInstanceLogMcpStack}" +REGION="${AWS_REGION:-us-east-1}" + +# Optional: pass ECS instance role ARNs directly (comma-separated) +# Usage: ./deploy.sh EcsInstanceLogMcpStack arn:aws:iam::123456789012:role/ecsInstanceRole +# Or: ECS_INSTANCE_ROLE_ARNS=arn:aws:iam::123456789012:role/ecsInstanceRole ./deploy.sh +if [ -n "$2" ]; then + export ECS_INSTANCE_ROLE_ARNS="$2" +fi + +echo "==============================================" +echo "ECS Instance Log MCP - Deployment Script" +echo "==============================================" +echo "Stack Name: $STACK_NAME" +echo "Region: $REGION" +echo "" + +# Check prerequisites +command -v npm >/dev/null 2>&1 || { echo "Error: npm is required but not installed."; exit 1; } +command -v aws >/dev/null 2>&1 || { echo "Error: AWS CLI is required but not installed."; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "Error: python3 is required but not installed."; exit 1; } + +# Install dependencies +echo "Installing dependencies..." +npm install --silent + +# Build TypeScript +echo "Building TypeScript..." +npm run build + +# Bootstrap CDK (if needed) +echo "Bootstrapping CDK (if needed)..." +npx cdk bootstrap --quiet 2>/dev/null || true + +# ======================================================================== +# DETECT / CREATE SSM DEFAULT HOST MANAGEMENT ROLE +# ======================================================================== +echo "" +echo "Setting up SSM Default Host Management role..." + +SSM_ROLE_NAME="AWSSystemsManagerDefaultEC2InstanceManagementRole" +EPOXY_ROLE_NAME="EpoxyAWSSystemsManagerDefaultEC2InstanceManagementRole" + +# Check for existing role (standard or Epoxy-prefixed) +SSM_DEFAULT_HOST_ROLE_ARN=$(aws iam get-role \ + --role-name "$SSM_ROLE_NAME" \ + --query 'Role.Arn' --output text 2>/dev/null || true) + +if [ -z "$SSM_DEFAULT_HOST_ROLE_ARN" ] || [ "$SSM_DEFAULT_HOST_ROLE_ARN" = "None" ]; then + SSM_DEFAULT_HOST_ROLE_ARN=$(aws iam get-role \ + --role-name "$EPOXY_ROLE_NAME" \ + --query 'Role.Arn' --output text 2>/dev/null || true) +fi + +if [ -z "$SSM_DEFAULT_HOST_ROLE_ARN" ] || [ "$SSM_DEFAULT_HOST_ROLE_ARN" = "None" ]; then + echo "SSM Default Host Management role not found. Creating $SSM_ROLE_NAME..." + + TRUST_POLICY=$(cat <<'TRUST' +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ssm.amazonaws.com" }, + "Action": "sts:AssumeRole" + } + ] +} +TRUST +) + + aws iam create-role \ + --role-name "$SSM_ROLE_NAME" \ + --assume-role-policy-document "$TRUST_POLICY" \ + --description "Default EC2 instance management role for SSM" \ + --region "$REGION" >/dev/null + + aws iam attach-role-policy \ + --role-name "$SSM_ROLE_NAME" \ + --policy-arn "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + + SSM_DEFAULT_HOST_ROLE_ARN=$(aws iam get-role \ + --role-name "$SSM_ROLE_NAME" \ + --query 'Role.Arn' --output text) + + echo "Created role: $SSM_DEFAULT_HOST_ROLE_ARN" + echo "Waiting 10s for IAM propagation..." + sleep 10 +else + echo "Found existing role: $SSM_DEFAULT_HOST_ROLE_ARN" +fi + +export SSM_DEFAULT_HOST_ROLE_ARN + +# ======================================================================== +# AUTO-DETECT ECS CONTAINER INSTANCE ROLE ARNS (interactive) +# ======================================================================== +echo "" +if [ -n "$ECS_INSTANCE_ROLE_ARNS" ]; then + echo "Using provided ECS instance role ARNs: $ECS_INSTANCE_ROLE_ARNS" +else + # --- Step 1: Region selection --- + echo "Which AWS regions should be scanned for ECS clusters?" + echo "" + echo " 1) All enabled regions" + echo " 2) Current deploy region only ($REGION)" + echo " 3) Enter a specific region" + echo "" + read -rp "Select [1/2/3] (default: 1): " REGION_CHOICE + REGION_CHOICE="${REGION_CHOICE:-1}" + + case "$REGION_CHOICE" in + 1) + echo "" + echo "Fetching all enabled regions..." + SCAN_REGIONS=$(aws ec2 describe-regions --query 'Regions[].RegionName' --output text 2>/dev/null || echo "$REGION") + ;; + 2) + SCAN_REGIONS="$REGION" + ;; + 3) + read -rp "Enter region (e.g. us-west-2): " CUSTOM_REGION + if [ -z "$CUSTOM_REGION" ]; then + echo "No region entered, falling back to $REGION" + CUSTOM_REGION="$REGION" + fi + SCAN_REGIONS="$CUSTOM_REGION" + ;; + *) + echo "Invalid choice, falling back to all regions." + SCAN_REGIONS=$(aws ec2 describe-regions --query 'Regions[].RegionName' --output text 2>/dev/null || echo "$REGION") + ;; + esac + + # --- Step 2: Discover ECS clusters across selected regions --- + echo "" + echo "Scanning for ECS clusters..." + + CLUSTER_LIST=() + CLUSTER_DISPLAY=() + + for SCAN_REGION in $SCAN_REGIONS; do + CLUSTERS=$(aws ecs list-clusters --region "$SCAN_REGION" --query 'clusterArns[*]' --output text 2>/dev/null || true) + if [ -n "$CLUSTERS" ]; then + for CLUSTER_ARN in $CLUSTERS; do + CLUSTER_NAME="${CLUSTER_ARN##*/}" + CLUSTER_LIST+=("${SCAN_REGION}/${CLUSTER_NAME}") + done + fi + done + + if [ ${#CLUSTER_LIST[@]} -eq 0 ]; then + echo "WARNING: No ECS clusters found in the selected region(s)." + echo "" + read -rp "Would you like to manually enter instance role ARN(s)? [y/N]: " MANUAL_ENTRY + if [ "$MANUAL_ENTRY" = "y" ] || [ "$MANUAL_ENTRY" = "Y" ]; then + echo "Enter comma-separated role ARNs (e.g. arn:aws:iam::123456789012:role/ecsInstanceRole):" + read -rp "> " MANUAL_ARNS + MANUAL_ARNS=$(echo "$MANUAL_ARNS" | tr -d ' ') + if [ -n "$MANUAL_ARNS" ]; then + ECS_INSTANCE_ROLE_ARNS="$MANUAL_ARNS" + echo "Using manually provided roles: $ECS_INSTANCE_ROLE_ARNS" + export ECS_INSTANCE_ROLE_ARNS + else + echo "No ARNs entered. The S3 bucket policy will use an account-scoped fallback." + fi + else + echo "Skipping. The S3 bucket policy will use an account-scoped fallback (less restrictive)." + fi + else + # --- Step 3: Display clusters and let user choose --- + echo "" + echo "Found ${#CLUSTER_LIST[@]} ECS cluster(s):" + echo "" + IDX=1 + for ENTRY in "${CLUSTER_LIST[@]}"; do + C_REGION="${ENTRY%%/*}" + C_NAME="${ENTRY#*/}" + echo " ${IDX}) ${C_NAME} (${C_REGION})" + IDX=$((IDX + 1)) + done + echo "" + echo " a) All clusters" + echo "" + read -rp "Select clusters (comma-separated numbers, or 'a' for all) [default: a]: " CLUSTER_CHOICE + CLUSTER_CHOICE="${CLUSTER_CHOICE:-a}" + + SELECTED_CLUSTERS=() + if [ "$CLUSTER_CHOICE" = "a" ] || [ "$CLUSTER_CHOICE" = "A" ]; then + SELECTED_CLUSTERS=("${CLUSTER_LIST[@]}") + else + IFS=',' read -ra PICKS <<< "$CLUSTER_CHOICE" + for PICK in "${PICKS[@]}"; do + PICK=$(echo "$PICK" | tr -d ' ') + if [[ "$PICK" =~ ^[0-9]+$ ]] && [ "$PICK" -ge 1 ] && [ "$PICK" -le ${#CLUSTER_LIST[@]} ]; then + SELECTED_CLUSTERS+=("${CLUSTER_LIST[$((PICK - 1))]}") + else + echo " Skipping invalid selection: $PICK" + fi + done + fi + + if [ ${#SELECTED_CLUSTERS[@]} -eq 0 ]; then + echo "No valid clusters selected. Skipping instance role detection." + else + echo "" + echo "Detecting container instance roles for ${#SELECTED_CLUSTERS[@]} cluster(s)..." + + # --- Step 4: Collect unique instance role ARNs from selected clusters --- + ALL_ROLE_ARNS=() + ROLE_SOURCES=() + + for ENTRY in "${SELECTED_CLUSTERS[@]}"; do + C_REGION="${ENTRY%%/*}" + C_NAME="${ENTRY#*/}" + + # List container instances in the cluster + CI_ARNS=$(aws ecs list-container-instances --cluster "$C_NAME" --region "$C_REGION" \ + --query 'containerInstanceArns[*]' --output text 2>/dev/null || true) + + if [ -n "$CI_ARNS" ]; then + # Describe container instances to get EC2 instance IDs + CI_DETAILS=$(aws ecs describe-container-instances --cluster "$C_NAME" --region "$C_REGION" \ + --container-instances $CI_ARNS \ + --query 'containerInstances[].ec2InstanceId' --output text 2>/dev/null || true) + + for EC2_ID in $CI_DETAILS; do + # Get the IAM instance profile role from the EC2 instance + ROLE_ARN=$(aws ec2 describe-instances --instance-ids "$EC2_ID" --region "$C_REGION" \ + --query 'Reservations[0].Instances[0].IamInstanceProfile.Arn' --output text 2>/dev/null || true) + + if [ -n "$ROLE_ARN" ] && [ "$ROLE_ARN" != "None" ]; then + # Convert instance profile ARN to role ARN + PROFILE_NAME="${ROLE_ARN##*/}" + ACTUAL_ROLE_ARN=$(aws iam get-instance-profile --instance-profile-name "$PROFILE_NAME" \ + --query 'InstanceProfile.Roles[0].Arn' --output text 2>/dev/null || true) + + if [ -n "$ACTUAL_ROLE_ARN" ] && [ "$ACTUAL_ROLE_ARN" != "None" ]; then + # Deduplicate + ALREADY_ADDED=false + for EXISTING in "${ALL_ROLE_ARNS[@]}"; do + if [ "$EXISTING" = "$ACTUAL_ROLE_ARN" ]; then + ALREADY_ADDED=true + break + fi + done + if [ "$ALREADY_ADDED" = false ]; then + ALL_ROLE_ARNS+=("$ACTUAL_ROLE_ARN") + ROLE_NAME="${ACTUAL_ROLE_ARN##*/}" + ROLE_SOURCES+=("${ROLE_NAME} (${C_NAME} / ${C_REGION})") + fi + fi + fi + done + fi + done + + if [ ${#ALL_ROLE_ARNS[@]} -eq 0 ]; then + echo "WARNING: No container instance roles found in selected clusters." + echo " (Clusters may be empty or using Fargate launch type)" + echo "" + read -rp "Would you like to manually enter instance role ARN(s)? [y/N]: " MANUAL_ENTRY + if [ "$MANUAL_ENTRY" = "y" ] || [ "$MANUAL_ENTRY" = "Y" ]; then + echo "Enter comma-separated role ARNs (e.g. arn:aws:iam::123456789012:role/ecsInstanceRole):" + read -rp "> " MANUAL_ARNS + MANUAL_ARNS=$(echo "$MANUAL_ARNS" | tr -d ' ') + if [ -n "$MANUAL_ARNS" ]; then + ECS_INSTANCE_ROLE_ARNS="$MANUAL_ARNS" + echo "Using manually provided roles: $ECS_INSTANCE_ROLE_ARNS" + export ECS_INSTANCE_ROLE_ARNS + else + echo "No ARNs entered. The S3 bucket policy will use an account-scoped fallback." + fi + else + echo "Skipping. The S3 bucket policy will use an account-scoped fallback (less restrictive)." + fi + else + # --- Step 5: Let user choose which instance roles to include --- + echo "" + echo "Found ${#ALL_ROLE_ARNS[@]} unique container instance role(s):" + echo "" + IDX=1 + for i in "${!ALL_ROLE_ARNS[@]}"; do + echo " ${IDX}) ${ALL_ROLE_ARNS[$i]}" + echo " └─ ${ROLE_SOURCES[$i]}" + IDX=$((IDX + 1)) + done + echo "" + echo " a) All roles" + echo "" + read -rp "Select instance roles (comma-separated numbers, or 'a' for all) [default: a]: " ROLE_CHOICE + ROLE_CHOICE="${ROLE_CHOICE:-a}" + + SELECTED_ROLES=() + if [ "$ROLE_CHOICE" = "a" ] || [ "$ROLE_CHOICE" = "A" ]; then + SELECTED_ROLES=("${ALL_ROLE_ARNS[@]}") + else + IFS=',' read -ra PICKS <<< "$ROLE_CHOICE" + for PICK in "${PICKS[@]}"; do + PICK=$(echo "$PICK" | tr -d ' ') + if [[ "$PICK" =~ ^[0-9]+$ ]] && [ "$PICK" -ge 1 ] && [ "$PICK" -le ${#ALL_ROLE_ARNS[@]} ]; then + SELECTED_ROLES+=("${ALL_ROLE_ARNS[$((PICK - 1))]}") + else + echo " Skipping invalid selection: $PICK" + fi + done + fi + + # Build comma-separated string + ECS_INSTANCE_ROLE_ARNS="" + for ROLE in "${SELECTED_ROLES[@]}"; do + if [ -z "$ECS_INSTANCE_ROLE_ARNS" ]; then + ECS_INSTANCE_ROLE_ARNS="$ROLE" + else + ECS_INSTANCE_ROLE_ARNS="$ECS_INSTANCE_ROLE_ARNS,$ROLE" + fi + done + + if [ -n "$ECS_INSTANCE_ROLE_ARNS" ]; then + echo "" + echo "Using ECS instance roles: $ECS_INSTANCE_ROLE_ARNS" + export ECS_INSTANCE_ROLE_ARNS + else + echo "No roles selected." + read -rp "Would you like to manually enter instance role ARN(s) instead? [y/N]: " MANUAL_ENTRY + if [ "$MANUAL_ENTRY" = "y" ] || [ "$MANUAL_ENTRY" = "Y" ]; then + echo "Enter comma-separated role ARNs (e.g. arn:aws:iam::123456789012:role/ecsInstanceRole):" + read -rp "> " MANUAL_ARNS + MANUAL_ARNS=$(echo "$MANUAL_ARNS" | tr -d ' ') + if [ -n "$MANUAL_ARNS" ]; then + ECS_INSTANCE_ROLE_ARNS="$MANUAL_ARNS" + echo "Using manually provided roles: $ECS_INSTANCE_ROLE_ARNS" + export ECS_INSTANCE_ROLE_ARNS + else + echo "No ARNs entered. The S3 bucket policy will use an account-scoped fallback." + fi + else + echo "The S3 bucket policy will use an account-scoped fallback." + fi + fi + fi + fi + fi +fi + +# Deploy the stack +echo "" +echo "Deploying CDK stack..." +npx cdk deploy "$STACK_NAME" --require-approval never --outputs-file cdk-outputs.json + +echo "" +echo "==============================================" +echo "Deployment Complete! Retrieving configuration..." +echo "==============================================" + +# Read from cdk-outputs.json using python3 for reliable JSON parsing +if [ ! -f cdk-outputs.json ]; then + echo "Error: cdk-outputs.json not found" + exit 1 +fi + +# Parse values from cdk-outputs.json +GATEWAY_URL=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'GatewayUrl' in k][0])" 2>/dev/null || echo "NOT_FOUND") +CLIENT_ID=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'CognitoClientId' in k][0])" 2>/dev/null || echo "NOT_FOUND") +USER_POOL_ID=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'CognitoUserPoolId' in k][0])" 2>/dev/null || echo "NOT_FOUND") +TOKEN_URL=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'OAuthExchangeUrl' in k][0])" 2>/dev/null || echo "NOT_FOUND") +OAUTH_SCOPE=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'OAuthScope' in k][0])" 2>/dev/null || echo "NOT_FOUND") +LOGS_BUCKET=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'LogsBucketName' in k][0])" 2>/dev/null || echo "NOT_FOUND") + +# Get Cognito Client Secret +echo "Retrieving Cognito Client Secret..." +if [ "$USER_POOL_ID" != "NOT_FOUND" ] && [ "$CLIENT_ID" != "NOT_FOUND" ]; then + CLIENT_SECRET=$(aws cognito-idp describe-user-pool-client \ + --user-pool-id "$USER_POOL_ID" \ + --client-id "$CLIENT_ID" \ + --region "$REGION" \ + --query "UserPoolClient.ClientSecret" \ + --output text 2>/dev/null || echo "NOT_FOUND") +else + CLIENT_SECRET="NOT_FOUND" +fi + +echo "" +echo "==============================================" +echo "DEVOPS AGENT MCP SERVER CONFIGURATION" +echo "==============================================" +echo "" +echo "Copy these values to configure the MCP Server in DevOps Agent Console:" +echo "" +echo "┌─────────────────────────────────────────────────────────────────────┐" +echo "│ MCP Server URL: │" +echo "│ $GATEWAY_URL" +echo "├─────────────────────────────────────────────────────────────────────┤" +echo "│ OAuth Client ID: │" +echo "│ $CLIENT_ID" +echo "├─────────────────────────────────────────────────────────────────────┤" +echo "│ OAuth Client Secret: │" +echo "│ $CLIENT_SECRET" +echo "├─────────────────────────────────────────────────────────────────────┤" +echo "│ Token URL: │" +echo "│ $TOKEN_URL" +echo "├─────────────────────────────────────────────────────────────────────┤" +echo "│ Scope (use only ONE): │" +echo "│ $OAUTH_SCOPE" +echo "└─────────────────────────────────────────────────────────────────────┘" +echo "" +echo "Additional Info:" +echo " Logs Bucket: $LOGS_BUCKET" +echo " Region: $REGION" +echo "" + +# Save configuration to file +CONFIG_FILE="mcp-config.txt" +cat > "$CONFIG_FILE" << EOF +# ECS Instance Log MCP - DevOps Agent Configuration +# Generated: $(date) +# Stack: $STACK_NAME +# Region: $REGION + +MCP_SERVER_URL=$GATEWAY_URL +OAUTH_CLIENT_ID=$CLIENT_ID +OAUTH_CLIENT_SECRET=$CLIENT_SECRET +TOKEN_URL=$TOKEN_URL +OAUTH_SCOPE=$OAUTH_SCOPE +LOGS_BUCKET=$LOGS_BUCKET +EOF + +echo "Configuration saved to: $CONFIG_FILE" +echo "" +echo "==============================================" +echo "AVAILABLE MCP TOOLS" +echo "==============================================" +echo "" +echo "TIER 1: CORE OPERATIONS" +echo "------------------------" +echo "1. collect - Start log collection from an ECS container instance" +echo "2. status - Get detailed status with progress tracking" +echo "3. validate - Verify all expected files were extracted" +echo "4. errors - Get pre-indexed error findings by severity" +echo "5. read - Byte-range streaming for multi-GB files" +echo "" +echo "TIER 2: ADVANCED ANALYSIS" +echo "-------------------------" +echo "6. search - Full-text regex search across all logs" +echo "7. correlate - Cross-file timeline correlation" +echo "8. artifact - Secure presigned URLs for large artifacts" +echo "9. summarize - Finding-grounded incident summary" +echo "10. history - Audit trail of past collections" +echo "" +echo "TIER 3: CLUSTER-LEVEL INTELLIGENCE" +echo "-----------------------------------" +echo "11. cluster_health - Health overview across all instances in a cluster" +echo "12. compare_instances - Diff findings between 2+ instances" +echo "13. batch_collect - Smart batch collection with sampling" +echo "14. batch_status - Poll status of multiple collections" +echo "15. network_diagnostics - Structured networking analysis" +echo "" +echo "TIER 4: LIVE PACKET CAPTURE" +echo "---------------------------" +echo "16. tcpdump_capture - Run tcpdump via SSM (supports task-scoped captures)" +echo "17. tcpdump_analyze - Decoded packets, stats, anomaly detection" +echo "" +echo "TIER 5: SOPs" +echo "------------" +echo "18. list_sops - List all 36 structured runbooks" +echo "19. get_sop - Get a specific runbook by name" +echo "" +echo "==============================================" +echo "EXAMPLE PROMPT FOR DEVOPS AGENT" +echo "==============================================" +echo "" +echo "\"I'm investigating an ECS container instance issue on i-0123456789abcdef0." +echo " Collect logs, find any critical errors, and give me a summary.\"" +echo "" diff --git a/mcp/aws-ecs-instance-log-mcp/docs/ARCHITECTURE.md b/mcp/aws-ecs-instance-log-mcp/docs/ARCHITECTURE.md new file mode 100644 index 0000000..4e78f67 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/docs/ARCHITECTURE.md @@ -0,0 +1,258 @@ +# ECS Instance Log MCP — Architecture & Design + +This document explains the internal design of the ECS Instance Log MCP server: how the components fit together, how data flows from an ECS container instance to an AI agent's context window, and the design decisions behind each layer. + +## Table of Contents + +- [System Overview](#system-overview) +- [Component Deep Dive](#component-deep-dive) +- [Data Flow](#data-flow) +- [Cross-Region Design](#cross-region-design) +- [Tool Architecture](#tool-architecture) +- [Time-Bounded Analysis](#time-bounded-analysis) +- [Anti-Hallucination Design](#anti-hallucination-design) +- [SOP Runbook System](#sop-runbook-system) +- [Security Model](#security-model) +- [CDK Construct Design](#cdk-construct-design) +- [Deploy Script Design](#deploy-script-design) + +--- + +## System Overview + +The server bridges the gap between AI agents and the OS-level diagnostic data on ECS container instances. The ECS API and CloudWatch don't expose Docker daemon config, iptables rules, cgroup memory events, container runtime state, or ECS agent internal logs — but these are exactly what's needed to diagnose task startup failures, agent disconnects, OOM kills, and networking issues. + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ MCP Client │────▶│ MCP Gateway │────▶│ Lambda │ +│ (DevOps Agent) │◀────│ (AgentCore) │◀────│ (19 tools) │ +└─────────────────┘ └──────────────────┘ └────────┬────────┘ + │ OAuth2 │ + ▼ │ + ┌──────────┐ ┌───────────┼───────────┐ + │ Cognito │ │ │ │ + │ User Pool│ ▼ ▼ ▼ + └──────────┘ ┌──────────┐ ┌──────────┐ ┌──────────┐ + │ SSM │ │ S3 │ │ S3 │ + │ Automati-│ │ Logs │ │ SOPs │ + │ on │ │ (KMS) │ │ Bucket │ + └────┬─────┘ └──────────┘ └──────────┘ + │ + ┌─────────┼─────────┐ + ▼ ▼ ▼ + ┌────────┐┌────────┐┌────────┐ + │ECS Inst││ECS Inst││ECS Inst│ + │Region A││Region B││Region C│ + └────────┘└────────┘└────────┘ +``` + +Hub-and-spoke: one central deployment serves container instances across all AWS regions. + +--- + +## Component Deep Dive + +### MCP Gateway (Bedrock AgentCore) + +Entry point for all MCP tool calls. Handles MCP protocol (JSON-RPC over HTTP), OAuth2 token validation via Cognito, and request routing to Lambda. Tool names are kept short (e.g., `collect`, `errors`, `read`) to stay under the 64-character limit. + +### Lambda Function (Tool Router) + +A single Python Lambda (~4400 lines) implementing all 19 MCP tools. Key design: +- **Single Lambda**: All tools share one function to avoid cold start multiplication +- **Regional clients**: `get_regional_client()` creates boto3 clients per-region with caching +- **Auto-detection**: `detect_instance_region()` tries default region first, then scans common regions +- **ECS instance validation**: `validate_ecs_instance()` verifies the target belongs to an ECS cluster via `aws:ecs:clusterName` tag or ECS API fallback +- **Region allow-list**: `ALLOWED_REGIONS` env var restricts which regions the tools can operate in + +### SSM Automation + +Log collection uses `AWSSupport-CollectECSInstanceLogs`, which: +- Runs on the target EC2 instance via SSM Agent +- Collects ECS agent logs, Docker/containerd, container logs, system logs, dmesg, networking, cgroups, metadata, GPU info +- Packages into a tar.gz archive and uploads to the central S3 bucket + +### S3 Log Storage + +Two S3 buckets: + +1. **Logs bucket** (KMS-encrypted): + ``` + ecs_{instance-id}/ + ├── {timestamp}.tar.gz # Raw bundle from SSM + ├── extracted/ # Extracted files (by Unzip Lambda) + │ ├── var/log/ecs/ecs-agent.log + │ ├── var/log/docker + │ ├── iptables-rules.txt + │ └── manifest.json + ├── findings_index.json # Pre-indexed errors + └── baselines/{cluster}/ # Baseline noise profiles + + idempotency/{instance-id}/ # Dedup mappings + execution-regions/ # Region metadata + ``` + +2. **SOPs bucket**: 36 runbook markdown files, auto-deployed via CDK. + +### Findings Indexer + +Separate Lambda triggered by S3 events on `manifest.json`. Scans extracted files for ECS-specific error patterns (agent disconnects, image pull failures, OOM kills, etc.), assigns severity and stable finding IDs (F-001), writes `findings_index.json`. + +### Unzip Lambda + +Triggered by `.tar.gz` uploads. Extracts files, generates `manifest.json`, which triggers the Findings Indexer. + +### KMS Encryption + +Customer-managed key encrypts all S3 objects. S3 client uses SigV4 explicitly for presigned URL compatibility. Key policy scopes to specific instance roles when provided, falls back to account-wide access otherwise. + +--- + +## Data Flow + +### Log Collection Flow + +``` +Agent calls collect(instanceId, region?) + → Lambda validates region (ALLOWED_REGIONS) and instance (ECS cluster tag) + → Lambda calls SSM StartAutomationExecution in target region + → SSM Agent collects logs, packages tar.gz, uploads to central S3 + → Unzip Lambda extracts → manifest.json triggers Findings Indexer + → Agent polls status(executionId) until complete + → Agent calls errors(instanceId) → pre-indexed findings +``` + +### Live Packet Capture Flow + +``` +Agent calls tcpdump_capture(instanceId, taskId?, filter?) + → Lambda calls SSM SendCommand (RunShellScript) + → If taskId: resolves container PID via ECS agent introspection + nsenter + → Captures for durationSeconds, uploads pcap + decoded text to S3 + → Agent polls tcpdump_capture(commandId) until complete + → Agent calls tcpdump_analyze(instanceId, commandId) → stats + anomalies +``` + +--- + +## Cross-Region Design + +``` +Central Region (us-east-1) + MCP Gateway → Lambda → S3 Bucket (KMS) + │ SSM StartAutomation + ├──→ us-west-2 (ECS Instance) + ├──→ eu-west-1 (ECS Instance) + └──→ ap-southeast-1 (ECS Instance) +``` + +- Region resolution: explicit param > auto-detect > Lambda's region +- Region metadata persisted in S3 for subsequent calls +- S3 writes are cross-region (S3 is global) +- IAM scoped to `ALLOWED_REGIONS` via `aws:RequestedRegion` conditions + +--- + +## Tool Architecture + +| Tier | Tools | Purpose | +|------|-------|---------| +| 1 — Core | `collect`, `status`, `validate`, `errors`, `read` | Log collection, findings, streaming | +| 2 — Analysis | `search`, `correlate`, `artifact`, `summarize`, `history` | Deep investigation, correlation | +| 3 — Cluster | `cluster_health`, `compare_instances`, `batch_collect`, `batch_status`, `network_diagnostics` | Multi-instance ops | +| 4 — Capture | `tcpdump_capture`, `tcpdump_analyze` | Live packet capture | +| 5 — SOPs | `list_sops`, `get_sop` | 36 structured runbooks | + +--- + +## Time-Bounded Analysis + +`TimeWindowResolver` enforces time windows on all analysis: +1. Explicit `start_time` + `end_time` → used as-is +2. `incident_time` → ± 5 minutes +3. Nothing → last 10 minutes +4. Max: 24 hours (safety cap) + +--- + +## Anti-Hallucination Design + +1. **Finding IDs**: Every error gets a stable ID (F-001). `summarize` requires finding IDs — unresolved IDs are flagged. +2. **ECS instance validation**: `collect` verifies the target belongs to an ECS cluster before running SSM. +3. **Region allow-list**: Tools reject requests to disallowed regions. +4. **Baseline subtraction**: Known noise is annotated, not removed. +5. **Confidence scores**: `correlate` reports confidence and data gaps. +6. **Network diagnostics guardrails**: `network_diagnostics` returns an `ecsContext` block with domain-specific guardrails (Docker bridge vs awsvpc, ECS Agent vs container networking, SG per network mode, DNS per network mode, conntrack attribution, ENI limits, Service Connect vs Discovery, container vs ELB health checks). +7. **False positive suppression**: Error scanning filters out `error_count=0`, conditional error handling, etc. +8. **Configurable presigned URLs**: `PRESIGNED_URL_EXPIRATION_SECONDS` env var controls URL lifetime. +9. **S3 SigV4**: Explicit SigV4 signing for KMS-encrypted bucket compatibility. + +--- + +## SOP Runbook System + +36 runbooks covering ECS-specific failure categories (A-K, Z). Each follows a 3-phase structure: +- Phase 1 — Triage (MUST): Check cluster/task state, collect logs, get findings +- Phase 2 — Enrich (SHOULD): Deep search, correlate, domain-specific diagnostics +- Phase 3 — Report (MUST): Grounded summary with finding IDs, root cause, remediation + +Auto-matched via `recommendedSOPs` in `errors`, `correlate`, and `summarize` responses. + +--- + +## Security Model + +| Layer | Mechanism | +|-------|-----------| +| Authentication | Cognito OAuth2 client credentials grant | +| Encryption at rest | KMS customer-managed key | +| Encryption in transit | HTTPS enforced on S3 | +| Public access | S3 Block Public Access | +| IAM | Least-privilege with region restrictions via `aws:RequestedRegion` | +| Instance scoping | S3 bucket policy and KMS key policy scoped to specific instance role ARNs (when provided) | +| Presigned URLs | Configurable expiration (default 900s) with SigV4 | +| Instance validation | ECS cluster membership verified before SSM execution | +| Region validation | `ALLOWED_REGIONS` enforced on all tools | +| Idempotency | Instance-scoped token mapping | +| Audit | CloudWatch logs for all Lambda invocations | + +--- + +## CDK Construct Design + +Single CDK construct (`EcsLogGatewayConstructV2`) provisions everything. Accepts optional props: +- `ecsInstanceRoleArns`: Scopes S3 bucket policy and KMS key policy to specific roles +- `allowedRegions`: Restricts IAM policies and Lambda env var +- `presignedUrlExpirationSeconds`: Controls URL lifetime +- `ssmDefaultHostRoleArn`: Grants SSM Default Host Management role access + +When `ecsInstanceRoleArns` is provided, the construct creates tight S3/KMS policies. When omitted, it falls back to account-scoped access for backward compatibility. + +--- + +## Deploy Script Design + +Interactive 5-step flow matching the EKS deploy pattern: + +``` +Step 1: Region Selection + ├── 1) All enabled regions + ├── 2) Current deploy region only + └── 3) Enter a specific region + +Step 2: ECS Cluster Discovery + └── Lists all ECS clusters across selected regions + +Step 3: Cluster Selection + ├── a) All clusters + └── 1,2,5) Comma-separated picks + +Step 4: Instance Role Detection + └── Discovers IAM roles from container instance profiles + +Step 5: Role Selection + ├── a) All roles + └── 1,3) Comma-separated picks +``` + +Automation mode: `ECS_INSTANCE_ROLE_ARNS="arn:..." ./deploy.sh` diff --git a/mcp/aws-ecs-instance-log-mcp/get-config.sh b/mcp/aws-ecs-instance-log-mcp/get-config.sh new file mode 100755 index 0000000..523025a --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/get-config.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# ECS Instance Log MCP - Get Configuration Values +# Run this after deployment to retrieve all configuration values + +STACK_NAME="${1:-EcsInstanceLogMcpStack}" +REGION="${AWS_REGION:-us-east-1}" + +if [ ! -f cdk-outputs.json ]; then + echo "Error: cdk-outputs.json not found. Run ./deploy.sh first." + exit 1 +fi + +# Parse values from cdk-outputs.json +GATEWAY_URL=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'GatewayUrl' in k][0])" 2>/dev/null) +CLIENT_ID=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'CognitoClientId' in k][0])" 2>/dev/null) +USER_POOL_ID=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'CognitoUserPoolId' in k][0])" 2>/dev/null) +TOKEN_URL=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'OAuthExchangeUrl' in k][0])" 2>/dev/null) +OAUTH_SCOPE=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'OAuthScope' in k][0])" 2>/dev/null) +LOGS_BUCKET=$(python3 -c "import json; d=json.load(open('cdk-outputs.json')); print([v for k,v in d.get('$STACK_NAME',{}).items() if 'LogsBucketName' in k][0])" 2>/dev/null) + +# Get Cognito Client Secret +CLIENT_SECRET=$(aws cognito-idp describe-user-pool-client \ + --user-pool-id "$USER_POOL_ID" \ + --client-id "$CLIENT_ID" \ + --region "$REGION" \ + --query "UserPoolClient.ClientSecret" \ + --output text 2>/dev/null) + +echo "" +echo "==============================================" +echo "DEVOPS AGENT MCP SERVER CONFIGURATION" +echo "==============================================" +echo "" +echo "MCP Server URL:" +echo " $GATEWAY_URL" +echo "" +echo "OAuth Client ID:" +echo " $CLIENT_ID" +echo "" +echo "OAuth Client Secret:" +echo " $CLIENT_SECRET" +echo "" +echo "Token URL:" +echo " $TOKEN_URL" +echo "" +echo "Scope (use only ONE):" +echo " $OAUTH_SCOPE" +echo "" +echo "Logs Bucket:" +echo " $LOGS_BUCKET" +echo "" +echo "==============================================" diff --git a/mcp/aws-ecs-instance-log-mcp/package-lock.json b/mcp/aws-ecs-instance-log-mcp/package-lock.json new file mode 100644 index 0000000..2814c10 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/package-lock.json @@ -0,0 +1,470 @@ +{ + "name": "ecs-instance-log-mcp", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ecs-instance-log-mcp", + "version": "1.0.0", + "license": "MIT-0", + "dependencies": { + "aws-cdk-lib": "^2.170.0", + "constructs": "^10.0.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "aws-cdk": "^2.170.0", + "typescript": "^5.0.0" + } + }, + "node_modules/@aws-cdk/asset-awscli-v1": { + "version": "2.2.263", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.263.tgz", + "integrity": "sha512-X9JvcJhYcb7PHs8R7m4zMablO5C9PGb/hYfLnxds9h/rKJu6l7MiXE/SabCibuehxPnuO/vk+sVVJiUWrccarQ==", + "license": "Apache-2.0" + }, + "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.0.tgz", + "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==", + "license": "Apache-2.0" + }, + "node_modules/@aws-cdk/cloud-assembly-schema": { + "version": "48.20.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-48.20.0.tgz", + "integrity": "sha512-+eeiav9LY4wbF/EFuCt/vfvi/Zoxo8bf94PW5clbMraChEliq83w4TbRVy0jB9jE0v1ooFTtIjSQkowSPkfISg==", + "bundleDependencies": [ + "jsonschema", + "semver" + ], + "license": "Apache-2.0", + "dependencies": { + "jsonschema": "~1.4.1", + "semver": "^7.7.2" + }, + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": { + "version": "7.7.2", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@types/node": { + "version": "20.19.32", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.32.tgz", + "integrity": "sha512-Ez8QE4DMfhjjTsES9K2dwfV258qBui7qxUsoaixZDiTzbde4U12e1pXGNu/ECsUIOi5/zoCxAQxIhQnaUQ2VvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/aws-cdk": { + "version": "2.1105.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.1105.0.tgz", + "integrity": "sha512-1RY2UZJv31XYobEGFHQEb7c2HXNzDbHuHqdnfdYyygvZW4Nrm8MJCW42lqItQCn+wF52Ixc7r2VR5eR4YGtVhA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "cdk": "bin/cdk" + }, + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/aws-cdk-lib": { + "version": "2.237.1", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.237.1.tgz", + "integrity": "sha512-RH8mWHLBtc14stkeUF0gFLaWdS5iS2AHUHnoT5B5LLZfEkYP9G43LjJs0AMmpdlQ5/ZQVvCZ+VB83Q9vLxILRw==", + "bundleDependencies": [ + "@balena/dockerignore", + "case", + "fs-extra", + "ignore", + "jsonschema", + "minimatch", + "punycode", + "semver", + "table", + "yaml", + "mime-types" + ], + "license": "Apache-2.0", + "dependencies": { + "@aws-cdk/asset-awscli-v1": "2.2.263", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.0", + "@aws-cdk/cloud-assembly-schema": "^48.20.0", + "@balena/dockerignore": "^1.0.2", + "case": "1.6.3", + "fs-extra": "^11.3.3", + "ignore": "^5.3.2", + "jsonschema": "^1.5.0", + "mime-types": "^2.1.35", + "minimatch": "^3.1.2", + "punycode": "^2.3.1", + "semver": "^7.7.3", + "table": "^6.9.0", + "yaml": "1.10.2" + }, + "engines": { + "node": ">= 18.0.0" + }, + "peerDependencies": { + "constructs": "^10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { + "version": "1.0.2", + "inBundle": true, + "license": "Apache-2.0" + }, + "node_modules/aws-cdk-lib/node_modules/ajv": { + "version": "8.17.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-regex": { + "version": "5.0.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-styles": { + "version": "4.3.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/astral-regex": { + "version": "2.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/balanced-match": { + "version": "1.0.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/brace-expansion": { + "version": "1.1.12", + "inBundle": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/aws-cdk-lib/node_modules/case": { + "version": "1.6.3", + "inBundle": true, + "license": "(MIT OR GPL-3.0-or-later)", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/color-convert": { + "version": "2.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/color-name": { + "version": "1.1.4", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/concat-map": { + "version": "0.0.1", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/emoji-regex": { + "version": "8.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { + "version": "3.1.3", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fast-uri": { + "version": "3.1.0", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "inBundle": true, + "license": "BSD-3-Clause" + }, + "node_modules/aws-cdk-lib/node_modules/fs-extra": { + "version": "11.3.3", + "inBundle": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/aws-cdk-lib/node_modules/graceful-fs": { + "version": "4.2.11", + "inBundle": true, + "license": "ISC" + }, + "node_modules/aws-cdk-lib/node_modules/ignore": { + "version": "5.3.2", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { + "version": "1.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/jsonfile": { + "version": "6.2.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/jsonschema": { + "version": "1.5.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { + "version": "4.4.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/mime-db": { + "version": "1.52.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/mime-types": { + "version": "2.1.35", + "inBundle": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/minimatch": { + "version": "3.1.2", + "inBundle": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/punycode": { + "version": "2.3.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/aws-cdk-lib/node_modules/require-from-string": { + "version": "2.0.2", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/semver": { + "version": "7.7.3", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/aws-cdk-lib/node_modules/slice-ansi": { + "version": "4.0.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/string-width": { + "version": "4.2.3", + "inBundle": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/strip-ansi": { + "version": "6.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/table": { + "version": "6.9.0", + "inBundle": true, + "license": "BSD-3-Clause", + "dependencies": { + "ajv": "^8.0.1", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/universalify": { + "version": "2.0.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/yaml": { + "version": "1.10.2", + "inBundle": true, + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/constructs": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.4.5.tgz", + "integrity": "sha512-fOoP70YLevMZr5avJHx2DU3LNYmC6wM8OwdrNewMZou1kZnPGOeVzBrRjZNgFDHUlulYUjkpFRSpTE3D+n+ZSg==", + "license": "Apache-2.0" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/mcp/aws-ecs-instance-log-mcp/package.json b/mcp/aws-ecs-instance-log-mcp/package.json new file mode 100644 index 0000000..3c6f5cf --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/package.json @@ -0,0 +1,37 @@ +{ + "name": "ecs-instance-log-mcp", + "version": "1.0.0", + "description": "MCP Server for DevOps Agent to collect ECS container instance logs via SSM Automation", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "test": "jest", + "cdk": "cdk", + "synth": "cdk synth", + "deploy": "cdk deploy", + "destroy": "cdk destroy" + }, + "keywords": [ + "aws", + "cdk", + "ssm", + "automation", + "mcp", + "devops-agent", + "ecs", + "troubleshooting" + ], + "author": "AWS", + "license": "MIT-0", + "devDependencies": { + "@types/node": "^20.0.0", + "aws-cdk": "^2.170.0", + "typescript": "^5.0.0" + }, + "dependencies": { + "aws-cdk-lib": "^2.170.0", + "constructs": "^10.0.0" + } +} diff --git a/mcp/aws-ecs-instance-log-mcp/requirements-dev.txt b/mcp/aws-ecs-instance-log-mcp/requirements-dev.txt new file mode 100644 index 0000000..5aa3a1d --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/requirements-dev.txt @@ -0,0 +1,4 @@ +pytest>=7.0 +hypothesis>=6.0 +boto3>=1.26 +botocore>=1.29 diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/A1-task-startup-resource-init.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/A1-task-startup-resource-init.md new file mode 100644 index 0000000..0cb168d --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/A1-task-startup-resource-init.md @@ -0,0 +1,78 @@ +--- +title: "A1 — Task Startup / ResourceInitializationError" +description: "Diagnose and remediate ECS tasks failing to start due to ResourceInitializationError" +status: active +severity: CRITICAL +triggers: + - "ResourceInitializationError" + - "TaskFailedToStart" + - "CannotStartContainerError" + - "CannotCreateContainerError" +owner: devops-agent +objective: "Identify the root cause of task startup failure and restore task placement" +context: "ResourceInitializationError occurs when ECS cannot set up the required resources (ENI, secrets, volumes) before starting the container. Common in awsvpc networking mode and Fargate tasks." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to get pre-indexed findings +- Use `search` tool with instanceId and query=`ResourceInitializationError|TaskFailedToStart|CannotStartContainerError` to find startup failure evidence + +SHOULD: +- Use `search` tool with query=`unable to pull secrets|failed to retrieve|ENI.*timeout` to identify the specific resource that failed +- Use `network_diagnostics` tool with instanceId to check ENI and subnet IP availability + +MAY: +- Use `cluster_health` tool with clusterName to check if multiple instances are affected +- Use `compare_instances` tool to diff healthy vs failing instances + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline around the startup failure +- Determine which resource failed: ENI provisioning, secrets retrieval, or volume mount +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`subnet|InsufficientFreeAddresses` if ENI-related +- Use `search` tool with query=`secretsmanager|ssm:GetParameters|AccessDenied` if secrets-related + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the startup failure findings +- State root cause: which resource initialization step failed and why +- Recommend immediate mitigation based on failure type + +SHOULD: +- Include timeline from correlate showing the sequence of events +- Provide specific remediation steps (e.g., add IPs to subnet, fix IAM role) + +## Guardrails + +escalation_conditions: + - "All tasks in a service failing to start" + - "Subnet IP exhaustion affecting multiple services" + - "IAM role changes needed require approval" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "IAM role changes: YELLOW — operator action" + - "Subnet/VPC changes: RED — requires approval" + +## Common Issues + +- symptoms: "ResourceInitializationError: unable to pull secrets or registry auth" + diagnosis: "Task execution role lacks permissions for Secrets Manager or SSM Parameter Store" + resolution: "Add secretsmanager:GetSecretValue or ssm:GetParameters to task execution role" + +- symptoms: "ResourceInitializationError: failed to configure ENI" + diagnosis: "Subnet has no available IP addresses or ENI limit reached" + resolution: "Add IPs to subnet or use a larger subnet CIDR" + +- symptoms: "CannotStartContainerError: exec format error" + diagnosis: "Image architecture mismatch (e.g., ARM image on x86 instance)" + resolution: "Use correct image architecture or change instance type" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/A2-task-startup-container-runtime.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/A2-task-startup-container-runtime.md new file mode 100644 index 0000000..0d67cc5 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/A2-task-startup-container-runtime.md @@ -0,0 +1,52 @@ +--- +title: "A2 — Task Startup / Container Runtime Failure" +description: "Diagnose ECS tasks failing due to container runtime errors (Docker/containerd)" +status: active +severity: CRITICAL +triggers: + - "ContainerRuntimeError" + - "ContainerRuntimeTimeoutError" + - "CannotCreateContainerError" +owner: devops-agent +objective: "Identify container runtime issue preventing task startup and restore service" +context: "Container runtime errors occur when Docker or containerd cannot create or start the container process. May indicate daemon issues, resource exhaustion, or configuration problems." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical to find runtime errors +- Use `search` tool with instanceId and query=`ContainerRuntimeError|docker.*daemon|containerd.*error|OCI runtime` to find runtime evidence + +SHOULD: +- Use `search` tool with query=`no space left|disk full|inode` to check disk exhaustion +- Use `search` tool with query=`docker.*restart|containerd.*restart` to check daemon restarts + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline +- Use `validate` tool to confirm docker and containerd logs are present + +SHOULD: +- Use `search` tool with query=`docker.*version|containerd.*version` to check versions + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: Docker/containerd daemon issue, disk exhaustion, or config error +- Recommend restart of container runtime or instance replacement + +## Guardrails + +escalation_conditions: + - "Container runtime down on multiple instances" + - "Disk exhaustion requiring volume resize" + +safety_ratings: + - "Log collection, search: GREEN (read-only)" + - "Docker daemon restart: YELLOW — operator action" + - "Instance replacement: RED — requires drain first" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/B1-ecr-image-pull-auth.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/B1-ecr-image-pull-auth.md new file mode 100644 index 0000000..133cf80 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/B1-ecr-image-pull-auth.md @@ -0,0 +1,53 @@ +--- +title: "B1 — ECR Image Pull Authentication Failure" +description: "Diagnose and remediate ECR image pull failures due to authentication or authorization" +status: active +severity: CRITICAL +triggers: + - "CannotPullECRContainerError" + - "ecr:GetAuthorizationToken.*denied" + - "ecr:BatchGetImage.*not authorized" + - "pull.*access.*denied" +owner: devops-agent +objective: "Restore ECR image pull capability by fixing authentication/authorization" +context: "ECR image pulls require the task execution role to have ecr:GetAuthorizationToken, ecr:BatchGetImage, and ecr:GetDownloadUrlForLayer permissions. Cross-account pulls need additional ECR repository policy." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical to find image pull errors +- Use `search` tool with instanceId and query=`CannotPullECRContainerError|ecr.*denied|pull.*access.*denied|authorization.*token` to find auth evidence + +SHOULD: +- Use `search` tool with query=`ecr:GetAuthorizationToken|ecr:BatchGetImage|ecr:GetDownloadUrlForLayer` to check which permission is missing +- Use `search` tool with query=`cross-account|registry.*id` to check for cross-account pull attempts + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline +- Determine if this is a permission issue (IAM) or a network issue (VPC endpoint) + +SHOULD: +- Use `network_diagnostics` tool to check if VPC endpoints for ECR are configured +- Use `search` tool with query=`vpc.*endpoint|443.*timeout` to check ECR connectivity + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: missing IAM permissions, expired token, or network connectivity +- Recommend specific IAM policy additions or VPC endpoint configuration + +## Common Issues + +- symptoms: "ecr:GetAuthorizationToken denied" + diagnosis: "Task execution role missing ECR auth permissions" + resolution: "Attach AmazonEC2ContainerRegistryReadOnly policy to task execution role" + +- symptoms: "ecr:BatchGetImage not authorized for cross-account" + diagnosis: "ECR repository policy does not allow cross-account access" + resolution: "Add cross-account principal to ECR repository policy" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/B2-image-not-found.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/B2-image-not-found.md new file mode 100644 index 0000000..89f044a --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/B2-image-not-found.md @@ -0,0 +1,48 @@ +--- +title: "B2 — Image Not Found / Manifest Not Found" +description: "Diagnose image pull failures due to missing image or tag" +status: active +severity: HIGH +triggers: + - "manifest.*not.*found" + - "repository.*does.*not.*exist" + - "image.*not.*found" + - "failed to resolve ref.*not found" +owner: devops-agent +objective: "Identify why the container image cannot be found and fix the reference" +context: "Image not found errors occur when the image URI, tag, or digest in the task definition does not match any image in the registry." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`manifest.*not found|repository.*not exist|image.*not found|resolve ref` to find the exact image URI that failed + +SHOULD: +- Use `search` tool with query=`image=|imageUri|container.*image` to extract the full image URI from task definition logs + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if this is a new issue or recurring +- Confirm the exact image URI and tag that failed + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: image tag deleted, typo in URI, or repository does not exist +- Recommend verifying image exists in registry + +## Common Issues + +- symptoms: "manifest for :latest not found" + diagnosis: "Image tag was overwritten or deleted from registry" + resolution: "Use immutable tags or image digests instead of :latest" + +- symptoms: "repository does not exist" + diagnosis: "ECR repository name is wrong or repository was deleted" + resolution: "Verify ECR repository name matches task definition" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/B3-docker-hub-rate-limit.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/B3-docker-hub-rate-limit.md new file mode 100644 index 0000000..df2875b --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/B3-docker-hub-rate-limit.md @@ -0,0 +1,42 @@ +--- +title: "B3 — Docker Hub Rate Limit" +description: "Diagnose image pull failures due to Docker Hub rate limiting" +status: active +severity: HIGH +triggers: + - "toomanyrequests.*Too Many Requests" + - "You have reached your pull rate limit" +owner: devops-agent +objective: "Mitigate Docker Hub rate limiting and restore image pulls" +context: "Docker Hub enforces pull rate limits: 100 pulls/6h for anonymous, 200 pulls/6h for authenticated. ECS tasks pulling public images can hit these limits quickly." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`toomanyrequests|rate limit|Too Many Requests` to confirm rate limiting + +SHOULD: +- Use `search` tool with query=`docker.io|hub.docker.com|registry-1.docker.io` to identify which images are from Docker Hub + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check frequency of pull attempts +- Determine if multiple tasks/services are pulling the same public images + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- Recommend migrating public images to ECR (pull-through cache or manual copy) +- Recommend configuring Docker Hub authentication for higher limits + +## Common Issues + +- symptoms: "toomanyrequests: Too Many Requests" + diagnosis: "Anonymous Docker Hub pull rate limit exceeded" + resolution: "Use ECR pull-through cache for Docker Hub images, or authenticate with Docker Hub credentials via Secrets Manager" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/C1-task-execution-role-permissions.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/C1-task-execution-role-permissions.md new file mode 100644 index 0000000..e93ecb5 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/C1-task-execution-role-permissions.md @@ -0,0 +1,53 @@ +--- +title: "C1 — Task Execution Role Permission Issues" +description: "Diagnose IAM permission failures for ECS task execution role" +status: active +severity: CRITICAL +triggers: + - "AccessDeniedException" + - "is not authorized to perform" + - "No valid providers in chain" + - "AssumeRoleUnauthorizedAccess" + - "execution role.*does not have" +owner: devops-agent +objective: "Identify missing IAM permissions and restore task execution" +context: "The task execution role is used by the ECS agent to pull images, retrieve secrets, and send logs. Missing permissions cause ResourceInitializationError or AccessDenied errors." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`AccessDenied|not authorized|No valid providers|execution role` to find IAM errors + +SHOULD: +- Use `search` tool with query=`ecr:GetAuthorizationToken|secretsmanager:GetSecretValue|ssm:GetParameters|logs:CreateLogStream` to identify which specific API call is denied + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine if IAM errors are the root cause or a symptom +- Identify the exact IAM action and resource that was denied + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: specific missing IAM permission +- Recommend exact IAM policy statement to add + +## Common Issues + +- symptoms: "No valid providers in chain" + diagnosis: "Task execution role ARN is invalid or role does not exist" + resolution: "Verify task execution role ARN in task definition and ensure role exists" + +- symptoms: "is not authorized to perform ecr:GetAuthorizationToken" + diagnosis: "Task execution role missing ECR permissions" + resolution: "Attach AmazonEC2ContainerRegistryReadOnly managed policy" + +- symptoms: "AccessDenied for secretsmanager:GetSecretValue" + diagnosis: "Task execution role missing Secrets Manager permissions" + resolution: "Add secretsmanager:GetSecretValue permission for the specific secret ARN" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/C2-secrets-manager-retrieval.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/C2-secrets-manager-retrieval.md new file mode 100644 index 0000000..3794839 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/C2-secrets-manager-retrieval.md @@ -0,0 +1,50 @@ +--- +title: "C2 — Secrets Manager / SSM Parameter Retrieval Failure" +description: "Diagnose failures retrieving secrets or parameters during task startup" +status: active +severity: CRITICAL +triggers: + - "unable to pull secrets" + - "unable to retrieve secret from asm" + - "SecretNotFound" + - "ParameterNotFound" + - "secretsmanager:GetSecretValue.*denied" + - "ssm:GetParameters.*denied" +owner: devops-agent +objective: "Restore secret/parameter retrieval for ECS task startup" +context: "ECS tasks can reference Secrets Manager secrets and SSM parameters in container definitions. Failures occur due to missing permissions, deleted secrets, or VPC endpoint issues." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`unable to pull secrets|SecretNotFound|ParameterNotFound|retrieve secret|retrieve ecr registry auth` to find secret retrieval errors + +SHOULD: +- Use `search` tool with query=`secretsmanager|ssm:GetParameters|AccessDenied` to identify permission vs not-found issues + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if secrets were recently deleted or rotated +- Determine if this is a permission issue or a resource-not-found issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: secret deleted, permission denied, or VPC endpoint missing +- Recommend specific fix + +## Common Issues + +- symptoms: "SecretNotFound" + diagnosis: "Secret ARN in task definition references a deleted or non-existent secret" + resolution: "Verify secret exists in Secrets Manager and ARN matches task definition" + +- symptoms: "unable to pull secrets or registry auth: execution resource retrieval failed" + diagnosis: "VPC endpoint for Secrets Manager not configured in private subnet" + resolution: "Create VPC endpoint for secretsmanager or ensure NAT gateway is configured" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/D1-oom-kill-memory.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/D1-oom-kill-memory.md new file mode 100644 index 0000000..4de3e0f --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/D1-oom-kill-memory.md @@ -0,0 +1,53 @@ +--- +title: "D1 — OOM Kill / Memory Exhaustion" +description: "Diagnose container or instance OOM kills in ECS" +status: active +severity: CRITICAL +triggers: + - "OutOfMemoryError" + - "oom.*kill" + - "Memory cgroup out of memory" + - "invoked oom-killer" + - "exit code 137" +owner: devops-agent +objective: "Identify OOM-killed process, determine memory pressure source, and prevent recurrence" +context: "OOM kills occur when a container exceeds its memory limit (hard limit) or the instance runs out of memory. Exit code 137 indicates SIGKILL from OOM killer." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical to find OOM findings +- Use `search` tool with instanceId and query=`oom-killer|OOMKilled|out of memory|Memory cgroup|exit code 137` to find OOM evidence in dmesg + +SHOULD: +- Use `search` tool with query=`Killed process.*total-vm|memory.*limit|memory.*usage` to find which process was killed and memory stats +- Use `search` tool with query=`insufficient.*memory|memory.*pressure` to check for instance-level memory pressure + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId and pivotEvent=`oom-killer` to build timeline +- Determine if OOM is at container level (cgroup limit) or instance level (system OOM) + +SHOULD: +- Use `search` tool with query=`memory.*hard.*limit|memoryReservation|memory=` to check task definition memory settings + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: container memory limit too low or memory leak +- Recommend increasing memory limit or investigating memory leak + +## Common Issues + +- symptoms: "exit code 137, Memory cgroup out of memory" + diagnosis: "Container exceeded its hard memory limit" + resolution: "Increase container memory limit in task definition, or investigate memory leak" + +- symptoms: "invoked oom-killer on instance level, multiple containers affected" + diagnosis: "Instance total memory exhausted by sum of all containers" + resolution: "Use larger instance type or reduce number of tasks per instance" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/D2-disk-space-exhaustion.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/D2-disk-space-exhaustion.md new file mode 100644 index 0000000..efc1fa7 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/D2-disk-space-exhaustion.md @@ -0,0 +1,41 @@ +--- +title: "D2 — Disk Space Exhaustion" +description: "Diagnose disk full errors on ECS container instances" +status: active +severity: CRITICAL +triggers: + - "no.*space.*left.*device" + - "disk.*full" +owner: devops-agent +objective: "Identify disk space consumers and restore available space" +context: "Disk exhaustion on ECS instances prevents new container creation, image pulls, and log writing. Common causes: accumulated container images, large log files, or container writable layers." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`no space left|disk full|disk.*pressure` to find disk errors + +SHOULD: +- Use `search` tool with query=`docker.*prune|image.*size|overlay.*size` to check image storage usage + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine when disk filled up +- Use `validate` tool to check if disk-related logs are present + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- Recommend: docker system prune, increase EBS volume, or configure image cleanup + +## Common Issues + +- symptoms: "no space left on device during image pull" + diagnosis: "Docker storage driver partition full from accumulated images" + resolution: "Operator: run docker system prune, or enable ECS image cleanup (ECS_IMAGE_CLEANUP_INTERVAL)" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/D3-cpu-throttling.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/D3-cpu-throttling.md new file mode 100644 index 0000000..a510d9d --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/D3-cpu-throttling.md @@ -0,0 +1,40 @@ +--- +title: "D3 — CPU Throttling" +description: "Diagnose CPU throttling affecting ECS container performance" +status: active +severity: HIGH +triggers: + - "cpu.*throttl" + - "insufficient.*cpu" +owner: devops-agent +objective: "Identify CPU-throttled containers and optimize resource allocation" +context: "CPU throttling occurs when containers hit their CPU limit. Unlike memory, CPU throttling doesn't kill containers but degrades performance, causing health check failures and timeouts." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`cpu.*throttl|insufficient.*cpu|cpu.*limit` to find CPU throttling evidence + +SHOULD: +- Use `search` tool with query=`health.*check.*failed|timeout|slow` to check for symptoms of CPU throttling + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if CPU throttling correlates with health check failures + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- Recommend increasing CPU units in task definition or using larger instance type + +## Common Issues + +- symptoms: "cpu throttling detected, health checks failing intermittently" + diagnosis: "Container CPU limit too low for workload" + resolution: "Increase cpu units in task definition (e.g., 256 -> 512)" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/E1-eni-allocation-subnet-ip.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/E1-eni-allocation-subnet-ip.md new file mode 100644 index 0000000..61e9be1 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/E1-eni-allocation-subnet-ip.md @@ -0,0 +1,49 @@ +--- +title: "E1 — ENI Allocation / Subnet IP Exhaustion" +description: "Diagnose ENI provisioning failures and subnet IP exhaustion in ECS" +status: active +severity: CRITICAL +triggers: + - "ENI.*allocation.*failed" + - "InsufficientFreeAddressesInSubnet" + - "Timeout waiting for network interface" + - "failed.*create.*network.*interface" +owner: devops-agent +objective: "Restore ENI provisioning for ECS tasks using awsvpc networking" +context: "Tasks using awsvpc mode require an ENI per task. Failures occur when the subnet runs out of IPs, ENI limits are reached, or security group limits are exceeded." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`ENI.*alloc|InsufficientFreeAddresses|network interface.*fail|Timeout.*network` to find ENI errors +- Use `network_diagnostics` tool with instanceId and sections=eni to check ENI attachment status + +SHOULD: +- Use `search` tool with query=`subnet|available.*ip|ip.*address` to check subnet capacity + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine when ENI failures started +- Use `network_diagnostics` tool with sections=eni,security-groups to get full network picture + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: subnet IP exhaustion, ENI limit, or security group limit +- Recommend adding IPs to subnet or using ENI trunking + +## Common Issues + +- symptoms: "InsufficientFreeAddressesInSubnet" + diagnosis: "Subnet CIDR too small for number of tasks" + resolution: "Use larger subnet, enable ENI trunking, or reduce task count" + +- symptoms: "Timeout waiting for network interface provisioning" + diagnosis: "ENI creation taking too long, possible API throttling" + resolution: "Check AWS service health, reduce concurrent task launches" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/E2-dns-resolution-failures.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/E2-dns-resolution-failures.md new file mode 100644 index 0000000..9424efb --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/E2-dns-resolution-failures.md @@ -0,0 +1,44 @@ +--- +title: "E2 — DNS Resolution Failures" +description: "Diagnose DNS resolution failures affecting ECS containers" +status: active +severity: HIGH +triggers: + - "DNS.*failed" + - "name.*resolution.*failed" + - "no.*route.*host" +owner: devops-agent +objective: "Restore DNS resolution for ECS containers" +context: "DNS failures in ECS can affect service discovery, ECR image pulls, and application connectivity. VPC DNS settings, Docker DNS config, and Route 53 resolver rules all play a role." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`DNS.*failed|resolve.*fail|SERVFAIL|NXDOMAIN|no route to host` to find DNS errors +- Use `network_diagnostics` tool with instanceId and sections=dns to check resolv.conf + +SHOULD: +- Use `search` tool with query=`nameserver|resolv.conf|search.*domain` to check DNS configuration + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if DNS failures correlate with other network issues +- Use `network_diagnostics` tool with sections=dns,routes to verify network path to DNS server + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: VPC DNS disabled, security group blocking UDP 53, or DNS server unreachable +- Recommend enabling VPC DNS or fixing security group rules + +## Common Issues + +- symptoms: "DNS resolution failed for ecr.*.amazonaws.com" + diagnosis: "VPC DNS resolution disabled or security group blocks UDP/TCP 53" + resolution: "Enable enableDnsSupport and enableDnsHostnames on VPC, check security group outbound rules" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/E3-connection-timeout.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/E3-connection-timeout.md new file mode 100644 index 0000000..53bae7a --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/E3-connection-timeout.md @@ -0,0 +1,45 @@ +--- +title: "E3 — Connection Timeout / Network Unreachable" +description: "Diagnose connection timeouts and network unreachable errors in ECS" +status: active +severity: HIGH +triggers: + - "connection.*timeout" + - "network.*unreachable" + - "dial.*tcp.*timeout" + - "i/o timeout" +owner: devops-agent +objective: "Identify network connectivity issue and restore communication" +context: "Connection timeouts in ECS can be caused by security group rules, NACLs, route table misconfigurations, or VPC endpoint issues." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`connection.*timeout|network.*unreachable|dial.*tcp.*timeout|i/o timeout` to find timeout errors +- Use `network_diagnostics` tool with instanceId and sections=all to get full network picture + +SHOULD: +- Use `search` tool with query=`security.*group|nacl|route.*table` to check network config + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine if timeouts are intermittent or persistent +- Use `network_diagnostics` tool with sections=routes,security-groups to check routing and SG rules + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: security group, NACL, route table, or VPC endpoint issue +- Recommend specific network configuration fix + +## Common Issues + +- symptoms: "connection timeout to 443 for ECR/S3 endpoints" + diagnosis: "Missing VPC endpoints or NAT gateway for private subnet" + resolution: "Create VPC endpoints for ECR, S3, and CloudWatch Logs, or configure NAT gateway" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/F1-container-health-check.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/F1-container-health-check.md new file mode 100644 index 0000000..6cae373 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/F1-container-health-check.md @@ -0,0 +1,47 @@ +--- +title: "F1 — Container Health Check Failures" +description: "Diagnose ECS container health check failures" +status: active +severity: HIGH +triggers: + - "health.*check.*failed" + - "UNHEALTHY" + - "failed container health checks" +owner: devops-agent +objective: "Identify why container health checks are failing and restore healthy state" +context: "ECS container health checks (HEALTHCHECK in Dockerfile or healthCheck in task definition) mark containers as UNHEALTHY when the check command fails consecutively. This can trigger task replacement." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`health.*check.*failed|UNHEALTHY|health.*status` to find health check failures + +SHOULD: +- Use `search` tool with query=`HEALTHCHECK|healthCheck|curl.*localhost|wget` to find health check command configuration +- Use `search` tool with query=`connection.*refused|timeout|exit code` to find why the check command fails + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if health check failures correlate with resource exhaustion or network issues + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: application not ready, port mismatch, or resource exhaustion +- Recommend fixing health check command, increasing startPeriod, or fixing application + +## Common Issues + +- symptoms: "container health check failed immediately after start" + diagnosis: "Health check startPeriod too short for application startup time" + resolution: "Increase startPeriod in health check configuration" + +- symptoms: "health check curl: connection refused" + diagnosis: "Application not listening on the expected port" + resolution: "Verify containerPort matches application listen port" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/F2-elb-target-health.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/F2-elb-target-health.md new file mode 100644 index 0000000..c1c0e08 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/F2-elb-target-health.md @@ -0,0 +1,43 @@ +--- +title: "F2 — ELB Target Health Check Failures" +description: "Diagnose ALB/NLB target health check failures for ECS services" +status: active +severity: HIGH +triggers: + - "target.*unhealthy" + - "failed ELB health checks" + - "Instance.*port.*is unhealthy" +owner: devops-agent +objective: "Restore ELB target health for ECS service" +context: "ELB health checks are separate from container health checks. Failed ELB health checks cause targets to be deregistered, reducing service capacity. Common causes: security group rules, health check path returning non-200, or application startup time." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`target.*unhealthy|ELB health|Instance.*unhealthy|deregistering.*target` to find ELB health failures + +SHOULD: +- Use `network_diagnostics` tool with instanceId and sections=security-groups to verify port access +- Use `search` tool with query=`health.*path|health.*check.*port` to check health check configuration + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if ELB failures correlate with deployments or resource issues + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: security group blocking health check port, wrong health check path, or slow startup +- Recommend specific fix + +## Common Issues + +- symptoms: "target unhealthy, health check on port 80 returning 502" + diagnosis: "Application not ready or returning errors on health check path" + resolution: "Verify health check path returns 200, increase deregistration delay and health check grace period" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/G1-agent-disconnected.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/G1-agent-disconnected.md new file mode 100644 index 0000000..4f3ac24 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/G1-agent-disconnected.md @@ -0,0 +1,52 @@ +--- +title: "G1 — ECS Agent Disconnected" +description: "Diagnose ECS agent disconnection from the cluster" +status: active +severity: CRITICAL +triggers: + - "agent.*connected.*false" + - "AGENT_DISCONNECTED" + - "websocket.*unable to dial" + - "Error getting ECS instance credentials" +owner: devops-agent +objective: "Restore ECS agent connectivity to the cluster" +context: "When the ECS agent disconnects, the instance cannot receive new task placements and existing tasks may become orphaned. Common causes: network issues, IAM credential expiry, or agent crash." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`AGENT_DISCONNECTED|agent.*connected.*false|websocket.*unable|ECS Agent failed` to find agent disconnection evidence + +SHOULD: +- Use `search` tool with query=`ecs-agent.*restart|ecs-init|agent.*start` to check agent restart history +- Use `network_diagnostics` tool with instanceId to check network connectivity + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to determine when agent disconnected and what else happened +- Determine if this is a network issue, IAM issue, or agent crash + +SHOULD: +- Use `search` tool with query=`credential|token.*expir|sts.*assume` to check IAM credential issues + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: network connectivity, IAM credentials, or agent crash +- Recommend restarting ECS agent or replacing instance + +## Common Issues + +- symptoms: "AGENT_DISCONNECTED, websocket unable to dial" + diagnosis: "Network connectivity to ECS service endpoint lost" + resolution: "Check VPC endpoints, NAT gateway, and security group outbound rules for HTTPS (443)" + +- symptoms: "Error getting ECS instance credentials" + diagnosis: "Instance profile or IAM role issue" + resolution: "Verify instance profile is attached and has AmazonEC2ContainerServiceforEC2Role policy" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/G2-instance-registration-failure.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/G2-instance-registration-failure.md new file mode 100644 index 0000000..e017382 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/G2-instance-registration-failure.md @@ -0,0 +1,49 @@ +--- +title: "G2 — Instance Registration Failure" +description: "Diagnose ECS container instance registration failures" +status: active +severity: CRITICAL +triggers: + - "failed.*register.*container.*instance" + - "No container instances were found" + - "ECS Agent failed to start" + - "client version.*is too old" +owner: devops-agent +objective: "Restore instance registration with ECS cluster" +context: "Instance registration failures prevent the instance from joining the cluster. Causes include wrong cluster name in ECS config, outdated agent version, or IAM permission issues." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`failed.*register|No container instances|Agent failed to start|client version.*too old` to find registration errors + +SHOULD: +- Use `search` tool with query=`ECS_CLUSTER|ecs.config|cluster.*name` to check cluster configuration +- Use `search` tool with query=`agent.*version|ecs-init.*version` to check agent version + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of registration attempts +- Determine if this is a config issue, version issue, or permission issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: wrong cluster name, outdated agent, or missing permissions +- Recommend specific fix + +## Common Issues + +- symptoms: "failed to register container instance: cluster not found" + diagnosis: "ECS_CLUSTER in /etc/ecs/ecs.config points to wrong or non-existent cluster" + resolution: "Fix ECS_CLUSTER value in /etc/ecs/ecs.config and restart ECS agent" + +- symptoms: "client version is too old" + diagnosis: "ECS agent version incompatible with cluster features" + resolution: "Update ECS agent: sudo yum update -y ecs-init && sudo systemctl restart ecs" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/H1-cloudwatch-log-driver.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/H1-cloudwatch-log-driver.md new file mode 100644 index 0000000..2a7f3e4 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/H1-cloudwatch-log-driver.md @@ -0,0 +1,49 @@ +--- +title: "H1 — CloudWatch Log Driver Issues" +description: "Diagnose CloudWatch logging failures for ECS containers" +status: active +severity: HIGH +triggers: + - "log.*driver.*error" + - "failed.*send.*logs" + - "awslogs.*error" + - "failed to initialize logging driver" + - "logs:CreateLogStream.*denied" +owner: devops-agent +objective: "Restore CloudWatch log delivery for ECS containers" +context: "The awslogs log driver sends container stdout/stderr to CloudWatch Logs. Failures can be caused by missing IAM permissions, log group not existing, or network connectivity to CloudWatch endpoint." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=all +- Use `search` tool with instanceId and query=`log.*driver.*error|awslogs.*error|CreateLogStream.*denied|initialize logging driver` to find logging errors + +SHOULD: +- Use `search` tool with query=`logs:CreateLogGroup|logs:CreateLogStream|logs:PutLogEvents` to check which permission is missing + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if logging failures cause task failures +- Determine if this is a permission issue, log group issue, or network issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: missing IAM permissions, log group not created, or VPC endpoint missing +- Recommend specific fix + +## Common Issues + +- symptoms: "failed to initialize logging driver: AccessDeniedException" + diagnosis: "Task execution role missing CloudWatch Logs permissions" + resolution: "Add logs:CreateLogGroup, logs:CreateLogStream, logs:PutLogEvents to task execution role" + +- symptoms: "awslogs error: ResourceNotFoundException" + diagnosis: "Log group does not exist and auto-create is not enabled" + resolution: "Create log group or set awslogs-create-group=true in log configuration" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/I1-deployment-circuit-breaker.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/I1-deployment-circuit-breaker.md new file mode 100644 index 0000000..33afd13 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/I1-deployment-circuit-breaker.md @@ -0,0 +1,48 @@ +--- +title: "I1 — Deployment Circuit Breaker Triggered" +description: "Diagnose ECS deployment failures that triggered the circuit breaker" +status: active +severity: CRITICAL +triggers: + - "deployment circuit breaker.*triggered" + - "ECS Deployment Circuit Breaker was triggered" + - "deployment circuit breaker.*rolling back" +owner: devops-agent +objective: "Identify why the deployment failed and fix the underlying issue before redeploying" +context: "The ECS deployment circuit breaker automatically rolls back a deployment when tasks repeatedly fail to reach RUNNING state. This prevents bad deployments from taking down the entire service." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`circuit breaker|rolling back|deployment.*fail|unable to place a task` to find deployment failure evidence + +SHOULD: +- Use `search` tool with query=`TaskFailedToStart|ResourceInitializationError|CannotPullContainerError` to find the underlying task failure +- Use `cluster_health` tool with clusterName to check overall cluster state + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of deployment events +- Identify the root cause: image pull failure, resource exhaustion, health check failure, or permission issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: the underlying task failure that triggered the circuit breaker +- Recommend fixing the root cause before redeploying + +## Common Issues + +- symptoms: "circuit breaker triggered, tasks failing with CannotPullContainerError" + diagnosis: "New image tag does not exist or ECR permissions changed" + resolution: "Verify image exists in registry, fix permissions, then redeploy" + +- symptoms: "circuit breaker triggered, tasks failing health checks" + diagnosis: "New application version has a bug or misconfiguration" + resolution: "Check application logs, fix the issue, then redeploy" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/J1-oci-runtime-entrypoint.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/J1-oci-runtime-entrypoint.md new file mode 100644 index 0000000..2be35a2 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/J1-oci-runtime-entrypoint.md @@ -0,0 +1,53 @@ +--- +title: "J1 — OCI Runtime / Entrypoint Failures" +description: "Diagnose container runtime failures related to OCI, entrypoint, or architecture mismatch" +status: active +severity: CRITICAL +triggers: + - "OCI runtime create failed" + - "exec format error" + - "no such file or directory.*entrypoint" + - "permission denied.*entrypoint" + - "container_linux.go.*starting container process" +owner: devops-agent +objective: "Fix container startup failure at the runtime level" +context: "OCI runtime errors occur at the lowest level of container creation. Common causes: wrong CPU architecture (ARM vs x86), missing entrypoint binary, or permission issues on the entrypoint script." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs +- Use `status` tool with executionId to poll until complete +- Use `errors` tool with instanceId and severity=critical +- Use `search` tool with instanceId and query=`OCI runtime|exec format error|entrypoint.*not found|permission denied.*entrypoint|container_linux.go` to find runtime errors + +SHOULD: +- Use `search` tool with query=`architecture|platform|amd64|arm64` to check for architecture mismatch + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to check if this affects all tasks or specific images +- Determine if this is an architecture mismatch, missing binary, or permission issue + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids +- State root cause: architecture mismatch, missing entrypoint, or permission denied +- Recommend building correct image architecture or fixing Dockerfile + +## Common Issues + +- symptoms: "exec format error" + diagnosis: "Image built for different CPU architecture (e.g., ARM image on x86 instance)" + resolution: "Build multi-arch image or use correct platform: docker build --platform linux/amd64" + +- symptoms: "no such file or directory: /app/entrypoint.sh" + diagnosis: "Entrypoint binary missing from image" + resolution: "Verify ENTRYPOINT/CMD in Dockerfile, ensure binary is included in image" + +- symptoms: "permission denied: /app/entrypoint.sh" + diagnosis: "Entrypoint script not executable" + resolution: "Add RUN chmod +x /app/entrypoint.sh in Dockerfile" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K1-spot-interruption-instance-draining.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K1-spot-interruption-instance-draining.md new file mode 100644 index 0000000..271efd0 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K1-spot-interruption-instance-draining.md @@ -0,0 +1,81 @@ +--- +title: "K1 — Spot Interruption / Instance Draining" +description: "Diagnose and remediate ECS task disruptions caused by Spot Instance interruptions or container instance draining" +status: active +severity: HIGH +triggers: + - "SpotInterruption" + - "DRAINING" + - "instance is being terminated" + - "Spot Instance interruption notice" + - "managed instance draining" + - "ECS_ENABLE_SPOT_INSTANCE_DRAINING" +owner: devops-agent +objective: "Identify Spot interruption or draining events and ensure tasks are gracefully rescheduled" +context: "When EC2 Spot capacity is reclaimed or an instance enters DRAINING state, ECS stops scheduling new tasks on it and attempts to replace running service tasks. Standalone tasks are NOT automatically replaced. Two-minute warning for Spot terminations." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=high to find Spot/draining related errors +- Use `search` tool with instanceId and query=`SpotInterruption|DRAINING|instance.*terminat|spot.*interrupt` to find interruption evidence + +SHOULD: +- Use `search` tool with query=`ECS_ENABLE_SPOT_INSTANCE_DRAINING|managed.*draining|lifecycle.*hook` to check draining configuration +- Use `cluster_health` tool with clusterName to check if multiple instances are affected + +MAY: +- Use `compare_instances` tool to diff healthy vs draining instances +- Use `search` tool with query=`standalone.*task|PENDING.*stop` to find standalone tasks that won't auto-replace + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of draining/interruption events +- Determine if tasks were successfully rescheduled to other instances +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`replacement.*task|new.*placement|capacity.*provider` to verify replacement task launches +- Check if managed termination protection is enabled for the capacity provider + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the interruption findings +- State root cause: Spot reclamation, manual draining, or ASG lifecycle event +- Recommend enabling ECS_ENABLE_SPOT_INSTANCE_DRAINING if not configured + +SHOULD: +- Include timeline from correlate showing draining sequence +- Recommend using multiple capacity providers with Spot and On-Demand mix +- Recommend enabling managed instance draining on capacity providers + +## Guardrails + +escalation_conditions: + - "Multiple instances draining simultaneously causing capacity shortage" + - "Standalone tasks lost without replacement" + - "Service unable to place replacement tasks" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Capacity provider configuration changes: YELLOW — operator action" + - "ASG/instance type changes: RED — requires approval" + +## Common Issues + +- symptoms: "SpotInterruption: Spot capacity is no longer available" + diagnosis: "EC2 reclaimed Spot capacity in the Availability Zone" + resolution: "Use multiple AZs, diversify instance types, enable Spot Instance draining, consider On-Demand fallback" + +- symptoms: "Container instance set to DRAINING but tasks not replaced" + diagnosis: "No available capacity on other instances or standalone tasks not auto-replaced" + resolution: "Ensure sufficient capacity in other AZs, manually restart standalone tasks, enable managed instance draining" + +- symptoms: "Tasks stuck in STOPPING state during draining" + diagnosis: "stopTimeout too long or application not handling SIGTERM gracefully" + resolution: "Reduce stopTimeout in task definition, implement graceful shutdown handler in application" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K10-api-throttling-service-quotas.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K10-api-throttling-service-quotas.md new file mode 100644 index 0000000..c0d9bb5 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K10-api-throttling-service-quotas.md @@ -0,0 +1,86 @@ +--- +title: "K10 — API Throttling & Service Quota Exceeded" +description: "Diagnose and remediate ECS API throttling, rate limiting, and service quota exhaustion" +status: active +severity: HIGH +triggers: + - "throttl" + - "rate limit" + - "Rate exceeded" + - "TooManyRequestsException" + - "Limit exceeded" + - "service quota" + - "RequestLimitExceeded" + - "Operations are being throttled" +owner: devops-agent +objective: "Identify throttled APIs and quota limits, restore normal API throughput and task scheduling" +context: "ECS integrates with ELB, Cloud Map, EC2, and other services that each have independent API rate limits. Synchronous throttling returns immediate errors. Asynchronous throttling occurs when ECS invokes APIs on behalf of the user (e.g., ENI provisioning, target registration). At scale, RegisterTarget/DeregisterTarget, EC2 ENI APIs, and Cloud Map APIs are common throttle points. Service quotas limit concurrent Fargate tasks, registered instances per cluster (5000), and tasks in PROVISIONING state." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId (any active instance) to gather cluster-level logs +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find throttling errors +- Use `search` tool with instanceId and query=`throttl|rate.*limit|Rate exceeded|TooManyRequests|Limit exceeded|RequestLimitExceeded` to find evidence + +SHOULD: +- Use `search` tool with query=`Operations are being throttled|Will try again later|Cloud Map|RegisterTarget|DeregisterTarget` to identify async throttling +- Use `cluster_health` tool with clusterName to check cluster-wide impact + +MAY: +- Use `search` tool with query=`CloudTrail|ErrorCode.*Throttling|exponential.*backoff` to find CloudTrail throttle evidence +- Use `search` tool with query=`service.*quota|concurrent.*tasks|PROVISIONING.*quota` to check quota limits + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of throttling events +- Determine which API is being throttled: ECS, ELB, EC2, or Cloud Map +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`DescribeTargetHealth|CreateNetworkInterface|RegisterInstance` to identify specific throttled operations +- Use `search` tool with query=`desired.*count|running.*count|pending.*count` to assess scale of deployment + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the throttling findings +- State root cause: which API/service is throttled and current vs allowed rate +- Recommend specific remediation to reduce throttling + +SHOULD: +- Include deployment scaling recommendations (stagger deployments, reduce batch size) +- Recommend quota increase request if hard limits are hit + +## Guardrails + +escalation_conditions: + - "Sustained throttling blocking all deployments" + - "Service quota hard limit reached" + - "Multiple services affected by cascading throttle" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Deployment configuration changes: YELLOW — operator action" + - "Service quota increase request: YELLOW — requires AWS Support" + +## Common Issues + +- symptoms: "Service event: 'Operations are being throttled. Will try again later'" + diagnosis: "Cloud Map API throttling during service discovery registration at scale" + resolution: "Reduce concurrent deployments. Stagger service updates. Contact AWS Support for Cloud Map API limit increase." + +- symptoms: "Slow task launches, tasks stuck in PROVISIONING" + diagnosis: "EC2 ENI API throttling when using awsvpc network mode at scale" + resolution: "Enable ENI trunking to reduce ENI API calls. Use bridge network mode where possible. Stagger deployments." + +- symptoms: "'Limit exceeded' error when creating tasks" + diagnosis: "Fargate concurrent task quota or tasks-in-PROVISIONING-per-cluster quota exceeded" + resolution: "Delete unused task definition revisions. Request Fargate quota increase via Service Quotas console." + +- symptoms: "ELB target registration delays during deployment" + diagnosis: "RegisterTarget/DeregisterTarget API throttled with many services behind load balancers" + resolution: "Reduce number of concurrent service deployments. Use deployment circuit breaker to prevent rapid retry loops." diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K11-fargate-metadata-credential-errors.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K11-fargate-metadata-credential-errors.md new file mode 100644 index 0000000..822e178 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K11-fargate-metadata-credential-errors.md @@ -0,0 +1,87 @@ +--- +title: "K11 — Fargate Task Metadata & Credential Retrieval Errors" +description: "Diagnose and remediate task metadata endpoint failures and credential retrieval errors on Fargate" +status: active +severity: HIGH +triggers: + - "metadata" + - "credential" + - "Missing credentials" + - "could not load credentials" + - "instance metadata" + - "IMDS" + - "ECS_CONTAINER_METADATA" + - "timeout.*metadata" + - "provider chain" +owner: devops-agent +objective: "Restore task metadata endpoint access and credential retrieval for Fargate tasks" +context: "Fargate tasks use the task metadata endpoint (v3/v4) for container metadata, Docker stats, and task-level information. AWS SDK credential retrieval uses the container credential provider (169.254.170.2) injected via AWS_CONTAINER_CREDENTIALS_RELATIVE_URI. Failures occur when the metadata endpoint is unreachable, credentials expire, the task execution role is misconfigured, or network issues block the link-local address range. Intermittent failures may indicate container startup race conditions or SDK version issues." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected task/instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find credential/metadata errors +- Use `search` tool with instanceId and query=`Missing credentials|could not load credentials|metadata.*error|credential.*provider|169\.254\.170` to find evidence + +SHOULD: +- Use `search` tool with query=`ECS_CONTAINER_METADATA_URI|AWS_CONTAINER_CREDENTIALS|timeout.*metadata|IMDS` to identify metadata endpoint issues +- Use `search` tool with query=`AssumeRole|sts:AssumeRole|expired.*token|security.*token` to check credential expiry + +MAY: +- Use `search` tool with query=`SDK.*version|boto3|aws-sdk|retry.*credential` to check SDK-related issues +- Use `network_diagnostics` tool with instanceId to check network path to metadata endpoint + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of credential/metadata failures +- Determine failure type: metadata endpoint unreachable, credentials expired, or role misconfigured +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`task.*execution.*role|taskRoleArn|executionRoleArn` to verify role configuration +- Use `search` tool with query=`platform.*version|1\.4\.0|LATEST` to check platform version compatibility + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the credential/metadata findings +- State root cause: which credential/metadata mechanism failed and why +- Recommend specific remediation + +SHOULD: +- Include timeline showing when credentials started failing +- Recommend SDK upgrade if version-related + +## Guardrails + +escalation_conditions: + - "All tasks in a service unable to retrieve credentials" + - "Credential failures causing cascading application errors" + - "Metadata endpoint completely unreachable" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "IAM role changes: YELLOW — operator action" + - "Network/VPC changes: RED — requires approval" + +## Common Issues + +- symptoms: "'Missing credentials in config, or could not load credentials from any provider'" + diagnosis: "AWS SDK cannot find credentials. Task role not configured or container credential provider not available." + resolution: "Ensure taskRoleArn is set in task definition. Verify AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable is present. Upgrade AWS SDK to latest version." + +- symptoms: "Intermittent metadata errors on Fargate" + diagnosis: "Race condition during container startup — metadata endpoint not ready when application starts" + resolution: "Add retry logic with exponential backoff for metadata/credential calls at application startup. Use SDK built-in retry mechanisms." + +- symptoms: "Timeout errors from instance metadata service on Fargate" + diagnosis: "Network path to 169.254.170.2 blocked or task metadata endpoint overloaded" + resolution: "Verify task is on platform version 1.4.0+. Check that no custom iptables rules block link-local addresses. Reduce metadata polling frequency." + +- symptoms: "'Unable to retrieve instance metadata' in application logs" + diagnosis: "Application using EC2 IMDS (169.254.169.254) instead of ECS container credential provider" + resolution: "Configure application to use ECS container credential provider (AWS_CONTAINER_CREDENTIALS_RELATIVE_URI) instead of EC2 IMDS. Update SDK configuration." diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K12-essential-container-exited-nonzero.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K12-essential-container-exited-nonzero.md new file mode 100644 index 0000000..9be7318 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K12-essential-container-exited-nonzero.md @@ -0,0 +1,96 @@ +--- +title: "K12 — Essential Container Exited with Non-Zero Exit Code" +description: "Diagnose and remediate ECS tasks stopped due to essential container exit with non-zero codes" +status: active +severity: CRITICAL +triggers: + - "EssentialContainerExited" + - "exit code" + - "non-zero" + - "exit 1" + - "exit 137" + - "exit 139" + - "exit 143" + - "exit 255" + - "stopped" + - "essential container" + - "SIGKILL" + - "SIGTERM" + - "segfault" +owner: devops-agent +objective: "Identify why essential containers are exiting and restore task stability" +context: "When an essential container in an ECS task exits, the entire task is stopped. The exit code indicates the failure type: 0=normal, 1=application error, 137=OOM kill or SIGKILL (128+9), 139=segfault SIGSEGV (128+11), 143=SIGTERM (128+15), 255=container runtime error. The DescribeTasks API provides stoppedReason and container exit codes. Common causes include application crashes, OOM kills, signal handling issues, and entrypoint/command errors." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find exit-related errors +- Use `search` tool with instanceId and query=`EssentialContainerExited|exit code|non-zero|stopped.*reason|SIGKILL|SIGTERM|segfault` to find evidence + +SHOULD: +- Use `search` tool with query=`exit.*137|oom.*kill|invoked oom-killer|Memory cgroup` to check for OOM kills +- Use `search` tool with query=`exit.*139|SIGSEGV|segmentation fault|core dump` to check for segfaults +- Use `search` tool with query=`exit.*143|SIGTERM|graceful.*shutdown|signal.*15` to check for termination signals + +MAY: +- Use `search` tool with query=`exit.*1|error|exception|fatal|panic` to check for application errors +- Use `search` tool with query=`entrypoint|CMD|command.*not.*found|exec format error` to check entrypoint issues + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline around the container exit +- Determine exit code and map to failure category (OOM, signal, application error, runtime error) +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`memory.*limit|memory.*reservation|memoryReservation` to check memory configuration +- Use `search` tool with query=`health.*check|UNHEALTHY|health.*status` to check if health check failure triggered stop + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the exit code findings +- State root cause: specific exit code, signal, and triggering condition +- Recommend specific remediation based on exit code category + +SHOULD: +- Include container logs leading up to the exit +- Provide exit code reference table in summary + +## Guardrails + +escalation_conditions: + - "Essential container repeatedly exiting (crash loop)" + - "Exit code 137 (OOM) across multiple tasks" + - "Exit code 139 (segfault) indicating memory corruption" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Task definition memory/CPU changes: YELLOW — operator action" + - "Application code changes: RED — requires development team" + +## Common Issues + +- symptoms: "Task stopped: Essential container exited, exit code 137" + diagnosis: "Container killed by OOM killer — memory usage exceeded container memory limit" + resolution: "Increase memory limit in task definition. Profile application memory usage. Check for memory leaks. Set memoryReservation (soft limit) below memory (hard limit)." + +- symptoms: "Task stopped: Essential container exited, exit code 1" + diagnosis: "Application error — unhandled exception, configuration error, or dependency failure" + resolution: "Check container logs for error messages. Verify environment variables and secrets. Test container locally with same configuration." + +- symptoms: "Task stopped: Essential container exited, exit code 139" + diagnosis: "Segmentation fault (SIGSEGV) — memory corruption, null pointer, or binary incompatibility" + resolution: "Check for architecture mismatch (ARM vs x86). Update application dependencies. Enable core dumps for debugging." + +- symptoms: "Task stopped: Essential container exited, exit code 143" + diagnosis: "Container received SIGTERM — graceful shutdown requested by ECS (deployment, scaling, spot interruption)" + resolution: "Implement SIGTERM handler in application for graceful shutdown. Increase stopTimeout in task definition to allow more time for cleanup." + +- symptoms: "Task stopped: Essential container exited, exit code 255" + diagnosis: "Container runtime error — Docker/containerd failed to start or manage the container" + resolution: "Check container image validity. Verify entrypoint and CMD. Check for exec format errors (wrong architecture). Review container runtime logs." diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K13-windows-container-issues.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K13-windows-container-issues.md new file mode 100644 index 0000000..81dde86 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K13-windows-container-issues.md @@ -0,0 +1,91 @@ +--- +title: "K13 — Windows Container Issues" +description: "Diagnose and remediate Windows-specific ECS container failures including OS mismatch, IAM role bootstrap, and awslogs driver" +status: active +severity: HIGH +triggers: + - "Windows" + - "windows" + - "OS mismatch" + - "operating system does not match" + - "EnableTaskIAMRole" + - "ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE" + - "No valid providers in chain" + - "Unable to assume the role" + - "Windows Server" +owner: devops-agent +objective: "Resolve Windows-specific ECS task failures and configuration issues" +context: "Windows containers on ECS have unique requirements: the container base image OS version must match the host OS version, IAM roles for tasks require explicit bootstrap configuration (-EnableTaskIAMRole), the awslogs driver needs ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE=true, and several Linux task definition parameters are unsupported (linuxParameters, privileged, readonlyRootFilesystem, user, ulimits). Windows and Linux tasks must run in separate clusters. Windows Server 2016 is deprecated and cannot run the latest Docker version." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected Windows instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find Windows-specific errors +- Use `search` tool with instanceId and query=`operating system does not match|OS mismatch|Windows.*error|EnableTaskIAMRole|AWSLOGS_EXECUTIONROLE` to find evidence + +SHOULD: +- Use `search` tool with query=`No valid providers in chain|Unable to assume.*role|credential.*provider` to check IAM role issues +- Use `search` tool with query=`Windows Server 2016|Windows Server 2019|Windows Server 2022|Windows Server 2025` to identify OS version + +MAY: +- Use `search` tool with query=`user data|bootstrap|Initialize-ECSAgent|Set-Variable` to check instance bootstrap configuration +- Use `cluster_health` tool with clusterName to check cluster-wide Windows instance health + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of Windows-specific failures +- Determine failure category: OS mismatch, IAM bootstrap, awslogs driver, or unsupported parameter +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`container.*image|base.*image|nanoserver|servercore|ltsc` to check image OS version +- Use `search` tool with query=`awslogs|log.*driver|logConfiguration|CreateLogStream` to check logging configuration + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the Windows-specific findings +- State root cause: which Windows-specific configuration is incorrect +- Recommend specific remediation + +SHOULD: +- Include OS version compatibility matrix +- Recommend migration path if on deprecated Windows Server version + +## Guardrails + +escalation_conditions: + - "All Windows tasks failing across the cluster" + - "OS version mismatch requiring AMI update" + - "Windows Server 2016 deprecation blocking updates" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "User data / bootstrap changes: YELLOW — requires instance replacement" + - "AMI update: RED — requires approval and rolling replacement" + +## Common Issues + +- symptoms: "'The container operating system does not match the host operating system'" + diagnosis: "Container base image OS version does not match the EC2 host or Fargate platform OS version" + resolution: "Ensure container image base (e.g., ltsc2022) matches host OS. Use matching ECS-optimized Windows AMI. For Fargate, use compatible Windows platform version." + +- symptoms: "'No valid providers in chain' error on Windows tasks" + diagnosis: "ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE not set on Windows container instance" + resolution: "Add 'ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE=true' to instance user data bootstrap script." + +- symptoms: "'Unable to assume the role' on Windows EC2 tasks" + diagnosis: "IAM roles for tasks not enabled — missing -EnableTaskIAMRole in bootstrap" + resolution: "Add '-EnableTaskIAMRole' flag to Initialize-ECSAgent in instance user data. Ensure Windows instance meets IAM role configuration requirements." + +- symptoms: "Task definition validation fails with unsupported parameters" + diagnosis: "Linux-only parameters used in Windows task definition (linuxParameters, privileged, readonlyRootFilesystem, user, ulimits)" + resolution: "Remove unsupported parameters from task definition. Specify container-level CPU and memory instead of task-level for Windows EC2 tasks." + +- symptoms: "Windows container logs not appearing in CloudWatch" + diagnosis: "awslogs driver requires ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE on Windows instances" + resolution: "Set ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE=true in instance user data. Verify task execution role has logs:CreateLogStream and logs:PutLogEvents permissions." diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K14-task-latency-performance.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K14-task-latency-performance.md new file mode 100644 index 0000000..d6d23f4 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K14-task-latency-performance.md @@ -0,0 +1,94 @@ +--- +title: "K14 — Task Latency & Performance Degradation" +description: "Diagnose and remediate ECS task latency, EBS throttling, network interface throttling, and slow DNS" +status: active +severity: HIGH +triggers: + - "latency" + - "slow" + - "performance" + - "response time" + - "TargetResponseTime" + - "TTFB" + - "throttl" + - "EBS" + - "network.*throughput" + - "DNS.*slow" + - "timeout" +owner: devops-agent +objective: "Identify performance bottlenecks and restore acceptable task latency" +context: "ECS task latency can stem from multiple layers: application code, container resource limits (CPU/memory), EBS volume throttling (IOPS/throughput), network interface throttling (bandwidth/PPS limits), DNS resolution delays, load balancer configuration, or external dependency latency. For EC2 launch type, instance-level metrics (CPU, memory, network, EBS) are critical. For Fargate, container-level CloudWatch Container Insights metrics and network sidecar diagnostics help isolate the bottleneck." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=warning to find performance-related warnings +- Use `search` tool with instanceId and query=`latency|slow|timeout|performance|throttl|response.*time|TTFB` to find evidence + +SHOULD: +- Use `search` tool with query=`CPU.*utilization|memory.*utilization|cpu.*throttl|oom` to check resource saturation +- Use `search` tool with query=`EBS.*throttl|VolumeReadOps|VolumeWriteOps|BurstBalance` to check EBS throttling +- Use `network_diagnostics` tool with instanceId to check network performance + +MAY: +- Use `search` tool with query=`DNS.*resolution|resolve.*time|nslookup|dig` to check DNS latency +- Use `search` tool with query=`TargetResponseTime|HealthyHostCount|RequestCount` to check ALB metrics + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of performance degradation +- Determine bottleneck layer: CPU, memory, EBS, network, DNS, or application +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`instance.*type|vCPU|network.*bandwidth|baseline` to check instance capabilities +- Use `compare_instances` tool to compare performance across healthy vs degraded instances + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the performance findings +- State root cause: which layer is the bottleneck and specific metric evidence +- Recommend specific remediation to restore performance + +SHOULD: +- Include before/after metric comparison if baseline data available +- Recommend right-sizing based on observed resource utilization + +## Guardrails + +escalation_conditions: + - "P99 latency exceeding SLA thresholds" + - "EBS volume consistently throttled" + - "Network bandwidth saturated on instance" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Task definition resource changes: YELLOW — operator action" + - "Instance type change: YELLOW — requires rolling replacement" + +## Common Issues + +- symptoms: "High response times, TargetResponseTime spikes in ALB metrics" + diagnosis: "Application CPU or memory saturation causing slow request processing" + resolution: "Increase task CPU/memory limits. Enable Application Auto Scaling. Profile application for hot paths." + +- symptoms: "Intermittent latency spikes on EC2 launch type" + diagnosis: "EBS volume IOPS or throughput throttling — burst credits exhausted" + resolution: "Upgrade to gp3 volume with provisioned IOPS. Monitor BurstBalance metric. Reduce disk I/O or use instance store." + +- symptoms: "Network throughput degradation on EC2 instances" + diagnosis: "Instance network bandwidth limit reached or network interface PPS throttling" + resolution: "Use larger instance type with higher network bandwidth. Enable enhanced networking (ENA). Distribute traffic across more instances." + +- symptoms: "Slow DNS resolution causing connection timeouts" + diagnosis: "VPC DNS resolver throttled or DNS cache not configured" + resolution: "Enable DNS caching in application or use a local DNS cache sidecar. Check Route 53 Resolver query limits. Reduce DNS TTL churn." + +- symptoms: "Fargate task latency with no obvious resource saturation" + diagnosis: "Network interface throttling on Fargate — each task gets a single ENI with bandwidth limits" + resolution: "Use larger Fargate task size (higher vCPU = higher network bandwidth). Optimize payload sizes. Use connection pooling." diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K15-docker-daemon-agent-errors.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K15-docker-daemon-agent-errors.md new file mode 100644 index 0000000..e32b428 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K15-docker-daemon-agent-errors.md @@ -0,0 +1,88 @@ +--- +title: "K15 — Docker Daemon & Container Runtime Errors" +description: "Diagnose and remediate Docker API 500 errors, Docker daemon issues, containerd failures, and ECS agent runtime problems" +status: active +severity: CRITICAL +triggers: + - "Docker" + - "docker" + - "containerd" + - "API error 500" + - "devmapper" + - "daemon" + - "thin pool" + - "storage driver" + - "docker.sock" + - "container runtime" +owner: devops-agent +objective: "Restore Docker daemon and container runtime health on ECS container instances" +context: "Docker API 500 errors typically indicate the thin pool storage on a container instance is full, preventing new container creation. The default ECS-optimized AMI provides 8 GiB for OS and 22 GiB for images/metadata. When storage fills up, the Docker daemon cannot create containers. Other runtime issues include containerd failures, docker.sock permission errors, and stale container cleanup. The ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION agent variable controls how long stopped containers remain." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find Docker/runtime errors +- Use `search` tool with instanceId and query=`API error.*500|devmapper|thin pool|docker.*daemon|containerd.*error|docker\.sock` to find evidence + +SHOULD: +- Use `search` tool with query=`disk.*full|no space left|storage.*driver|overlay2|devicemapper` to check storage driver issues +- Use `search` tool with query=`ECS_ENGINE_TASK_CLEANUP|cleanup.*wait|stopped.*container|dead.*container` to check container cleanup + +MAY: +- Use `search` tool with query=`docker.*version|containerd.*version|runc.*version` to check runtime versions +- Use `cluster_health` tool with clusterName to check if multiple instances are affected + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of Docker/runtime failures +- Determine failure type: storage exhaustion, daemon crash, containerd failure, or permission error +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`docker.*info|Storage Driver|Data Space|Metadata Space|Thin Pool` to check Docker storage info +- Use `search` tool with query=`systemctl.*docker|service.*docker|restart.*docker|docker.*start` to check daemon restart attempts + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the Docker/runtime findings +- State root cause: storage exhaustion, daemon failure, or configuration issue +- Recommend specific remediation to restore container runtime + +SHOULD: +- Include storage utilization data +- Recommend preventive measures (cleanup duration, larger volumes, monitoring) + +## Guardrails + +escalation_conditions: + - "Docker daemon completely unresponsive" + - "All container launches failing on instance" + - "Multiple instances with same Docker error" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Docker daemon restart: YELLOW — causes brief task disruption" + - "Instance replacement: RED — requires draining and replacement" + +## Common Issues + +- symptoms: "Docker API error (500): devmapper — thin pool full" + diagnosis: "Container instance thin pool storage exhausted by accumulated images and stopped containers" + resolution: "Terminate instance and launch new one with larger data volume. Reduce ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION (default 3h). Remove unused images with 'docker image prune'. Use fstrim to reclaim space." + +- symptoms: "Cannot connect to Docker daemon at unix:///var/run/docker.sock" + diagnosis: "Docker daemon not running or crashed" + resolution: "Check Docker daemon status: systemctl status docker. Review /var/log/docker for crash logs. Restart Docker: systemctl restart docker. If persistent, replace instance." + +- symptoms: "containerd: runtime error during container creation" + diagnosis: "containerd runtime failure — may be caused by corrupted container state or resource exhaustion" + resolution: "Restart containerd service. Check system memory and disk. If persistent, drain and replace the instance." + +- symptoms: "Stale containers consuming disk space" + diagnosis: "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION too long, stopped containers accumulating" + resolution: "Set ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION to a shorter value (e.g., 15m). Manually clean: docker container prune. Monitor disk usage with CloudWatch agent." diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K2-task-placement-failures.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K2-task-placement-failures.md new file mode 100644 index 0000000..3e5c62b --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K2-task-placement-failures.md @@ -0,0 +1,90 @@ +--- +title: "K2 — Task Placement Failures" +description: "Diagnose and remediate ECS tasks failing to place due to insufficient resources, missing attributes, or constraint violations" +status: active +severity: CRITICAL +triggers: + - "no container instance met all of its requirements" + - "unable to place a task" + - "insufficient CPU" + - "insufficient memory" + - "AGENT" + - "MemberOf placement constraint unsatisfied" + - "SERVICE_TASK_PLACEMENT_FAILURE" + - "PROVISIONING" +owner: devops-agent +objective: "Identify why tasks cannot be placed and restore task scheduling" +context: "Task placement failures occur when no container instance in the cluster meets the task's CPU, memory, port, attribute, or constraint requirements. Tasks remain in PROVISIONING/PENDING state." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId (any active instance) to gather cluster-level logs +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find placement-related errors +- Use `search` tool with instanceId and query=`no container instance|unable to place|insufficient.*CPU|insufficient.*memory|placement.*constraint` to find placement failure evidence + +SHOULD: +- Use `cluster_health` tool with clusterName to check overall cluster capacity and instance count +- Use `search` tool with query=`AGENT|agent.*disconnected|agent.*not.*connected` to check for disconnected agents + +MAY: +- Use `compare_instances` tool to diff instances that can vs cannot accept tasks +- Use `search` tool with query=`port.*already.*use|required.*port` to check port conflicts + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of placement failures +- Determine the specific constraint that failed: CPU, memory, port, attribute, or placement constraint +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`attribute.*missing|ecs.capability|ecs.instance-type` to check missing attributes +- Use `search` tool with query=`desired.*count|running.*count|capacity.*provider` to check capacity vs demand + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the placement failure findings +- State root cause: which requirement could not be satisfied +- Recommend specific remediation based on failure type + +SHOULD: +- Include cluster capacity analysis showing available vs required resources +- Recommend right-sizing task definitions or adding capacity + +## Guardrails + +escalation_conditions: + - "All tasks in a service stuck in PENDING/PROVISIONING" + - "Cluster has zero registered container instances" + - "Capacity provider unable to scale out" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Task definition changes: YELLOW — operator action" + - "Instance type or ASG changes: RED — requires approval" + +## Common Issues + +- symptoms: "no container instance met all of its requirements — insufficient CPU units" + diagnosis: "Task requires more CPU than any instance has available" + resolution: "Reduce task CPU, use larger instance types, or add more instances" + +- symptoms: "no container instance met all of its requirements — insufficient memory" + diagnosis: "Task requires more memory than any instance has available" + resolution: "Reduce task memory, use larger instance types, or terminate unused tasks" + +- symptoms: "closest matching container-instance encountered error AGENT" + diagnosis: "ECS agent on the instance is disconnected" + resolution: "SSH to instance and restart ecs agent: sudo systemctl restart ecs" + +- symptoms: "MemberOf placement constraint unsatisfied" + diagnosis: "No instances match the placement constraint expression" + resolution: "Add custom attributes to instances or relax placement constraints" + +- symptoms: "closest matching container instance already uses a required port" + diagnosis: "Host port conflict — another task already bound to the required port" + resolution: "Use dynamic port mapping with ALB, or add more container instances" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K3-service-steady-state-failures.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K3-service-steady-state-failures.md new file mode 100644 index 0000000..fd1a2ec --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K3-service-steady-state-failures.md @@ -0,0 +1,87 @@ +--- +title: "K3 — Service Steady State Failures" +description: "Diagnose and remediate ECS services that fail to reach or maintain steady state" +status: active +severity: CRITICAL +triggers: + - "service.*unable to reach steady state" + - "SERVICE_TASK_START_IMPAIRED" + - "tasks failed to start" + - "deployment failed" + - "SERVICE_DEPLOYMENT_FAILED" + - "rolling back" + - "not healthy in target-group" +owner: devops-agent +objective: "Identify why a service cannot reach steady state and restore service stability" +context: "A service fails to reach steady state when tasks repeatedly fail health checks, exit with errors, cannot be placed, or fail to start. The service scheduler continuously tries to replace failed tasks, creating a restart loop." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from an affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find service-level errors +- Use `search` tool with instanceId and query=`steady state|deployment failed|tasks failed to start|SERVICE_TASK_START_IMPAIRED|rolling back` to find steady state failure evidence + +SHOULD: +- Use `search` tool with query=`health check|unhealthy|target.*not.*found|deregistered` to check health check failures +- Use `search` tool with query=`exit code|non-zero|EssentialContainerExited|OOMKilled` to check task exit reasons +- Use `cluster_health` tool with clusterName to check overall service health + +MAY: +- Use `compare_instances` tool to diff instances running healthy vs failing tasks +- Use `network_diagnostics` tool with instanceId to check network-related causes + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of service events +- Determine the specific failure mode: health check, task crash, placement, or configuration +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`HealthCheckGracePeriod|healthCheckPath|deregistration.*delay` to check health check configuration +- Use `search` tool with query=`minimumHealthyPercent|maximumPercent|desiredCount` to check deployment configuration + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the steady state failure findings +- State root cause: which component is preventing steady state +- Recommend specific remediation based on failure mode + +SHOULD: +- Include deployment timeline showing task start/stop cycles +- Recommend adjusting HealthCheckGracePeriodSeconds if tasks need more startup time + +## Guardrails + +escalation_conditions: + - "Service in continuous restart loop for more than 30 minutes" + - "All tasks in service failing simultaneously" + - "Deployment circuit breaker triggered" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Health check parameter changes: YELLOW — operator action" + - "Service/task definition changes: YELLOW — operator action" + - "Rollback to previous task definition: RED — requires approval" + +## Common Issues + +- symptoms: "service unable to reach steady state — tasks failing health checks" + diagnosis: "Application not responding on health check path within timeout" + resolution: "Increase HealthCheckGracePeriodSeconds, verify health check path returns 200, check application startup time" + +- symptoms: "service unable to reach steady state — ELB health checks failing" + diagnosis: "Security group blocking health check traffic or wrong port/path" + resolution: "Verify security group allows ALB to reach container port, confirm health check path and expected response code" + +- symptoms: "tasks exiting with non-zero exit code" + diagnosis: "Application crashing due to configuration error, missing env vars, or dependency failure" + resolution: "Check CloudWatch Logs for application errors, verify environment variables and secrets, test container locally" + +- symptoms: "SERVICE_DEPLOYMENT_FAILED with circuit breaker" + diagnosis: "Too many consecutive task failures triggered the deployment circuit breaker" + resolution: "Fix the underlying task failure, then create a new deployment with the corrected task definition" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K4-service-auto-scaling-issues.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K4-service-auto-scaling-issues.md new file mode 100644 index 0000000..60cef0f --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K4-service-auto-scaling-issues.md @@ -0,0 +1,87 @@ +--- +title: "K4 — Service Auto Scaling / Capacity Provider Issues" +description: "Diagnose and remediate ECS service auto scaling failures and capacity provider scaling problems" +status: active +severity: HIGH +triggers: + - "CapacityProviderReservation" + - "scaling policy" + - "desired count" + - "instance count discrepancy" + - "Limit exceeded" + - "InsufficientCapacity" + - "VcpuLimitExceeded" + - "managed scaling" +owner: devops-agent +objective: "Identify auto scaling or capacity provider issues and restore proper scaling behavior" +context: "ECS uses Application Auto Scaling for service task count and cluster auto scaling (capacity providers) for EC2 instance count. Failures can occur at either level — tasks not scaling, or instances not launching to support task demand." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from a container instance in the cluster +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=high to find scaling-related errors +- Use `search` tool with instanceId and query=`scaling|capacity.*provider|desired.*count|Limit exceeded|InsufficientCapacity|VcpuLimitExceeded` to find scaling failure evidence + +SHOULD: +- Use `cluster_health` tool with clusterName to check cluster capacity metrics +- Use `search` tool with query=`CapacityProviderReservation|targetCapacity|minimumScalingStepSize|maximumScalingStepSize` to check capacity provider configuration + +MAY: +- Use `search` tool with query=`CloudWatch.*alarm|target.*tracking|scaling.*policy` to check scaling policy triggers +- Use `compare_instances` tool to check instance distribution across AZs + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of scaling events +- Determine if the issue is at service level (task count) or cluster level (instance count) +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`Auto Scaling group|ASG|launch.*template|instance.*type` to check ASG configuration +- Use `search` tool with query=`service quota|rate limit|throttl` to check for API throttling + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the scaling failure findings +- State root cause: service scaling policy, capacity provider, ASG limits, or service quotas +- Recommend specific remediation + +SHOULD: +- Include scaling timeline showing demand vs capacity +- Recommend capacity provider configuration adjustments + +## Guardrails + +escalation_conditions: + - "Tasks stuck in PENDING due to no available capacity" + - "Service quota limits reached" + - "Capacity provider unable to launch instances in any AZ" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Scaling policy adjustments: YELLOW — operator action" + - "Service quota increase requests: YELLOW — operator action" + - "ASG/capacity provider changes: RED — requires approval" + +## Common Issues + +- symptoms: "Tasks stuck in PENDING, no new instances launching" + diagnosis: "Capacity provider managed scaling not enabled or ASG at MaxSize" + resolution: "Enable managed scaling on capacity provider, increase ASG MaxSize, or add instances manually" + +- symptoms: "Instance count discrepancy between ASG and ECS cluster" + diagnosis: "Instances launched but not registering with ECS cluster" + resolution: "Check ECS agent connectivity, verify instance user data sets correct cluster name, check security groups allow ECS agent traffic" + +- symptoms: "VcpuLimitExceeded error" + diagnosis: "EC2 vCPU service quota reached for the instance type family" + resolution: "Request service quota increase, use different instance types, or terminate unused instances" + +- symptoms: "Service not scaling despite high CPU/memory" + diagnosis: "Auto scaling policy not configured or CloudWatch alarm not triggering" + resolution: "Verify target tracking policy exists, check CloudWatch metrics are being published, ensure Container Insights is enabled" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K5-ecs-exec-failures.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K5-ecs-exec-failures.md new file mode 100644 index 0000000..2a80ae7 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K5-ecs-exec-failures.md @@ -0,0 +1,83 @@ +--- +title: "K5 — ECS Exec Failures" +description: "Diagnose and remediate failures when using ECS Exec to run commands in containers" +status: active +severity: MEDIUM +triggers: + - "execute command failed" + - "TargetNotConnectedException" + - "ExecuteCommandAgent" + - "SSM agent" + - "session manager" + - "ecs exec" +owner: devops-agent +objective: "Identify why ECS Exec cannot connect to a container and restore interactive access" +context: "ECS Exec uses AWS Systems Manager (SSM) Session Manager to establish connections to containers. Failures occur due to missing IAM permissions, SSM agent issues, VPC endpoint gaps, or the feature not being enabled on the service." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=medium to find exec-related errors +- Use `search` tool with instanceId and query=`execute command|ExecuteCommandAgent|TargetNotConnected|SSM.*agent|session.*manager` to find exec failure evidence + +SHOULD: +- Use `search` tool with query=`enableExecuteCommand|executeCommandConfiguration|task.*role|ssmmessages` to check ECS Exec configuration +- Use `network_diagnostics` tool with instanceId to check VPC endpoint connectivity + +MAY: +- Use `search` tool with query=`vpc.*endpoint|com.amazonaws.*ssmmessages|com.amazonaws.*ssm` to check SSM VPC endpoints +- Use `cluster_health` tool with clusterName to check if exec works on other tasks + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline around exec failures +- Determine the specific failure: IAM permissions, SSM agent, VPC endpoints, or feature not enabled +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`task.*IAM.*role|iam:PassRole|ssm:StartSession` to check IAM role configuration +- Use `search` tool with query=`managed.*agent|RUNNING|STOPPED` to check ExecuteCommandAgent status + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the exec failure findings +- State root cause: which component is preventing ECS Exec +- Recommend specific remediation steps + +SHOULD: +- Recommend running the ECS Exec Checker script for comprehensive validation +- Include IAM policy requirements for task role + +## Guardrails + +escalation_conditions: + - "ECS Exec needed for production incident debugging but unavailable" + - "SSM agent not running on any container instances" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "IAM role policy changes: YELLOW — operator action" + - "VPC endpoint creation: RED — requires approval" + +## Common Issues + +- symptoms: "The execute command failed because execute command was not enabled" + diagnosis: "ECS Exec not enabled on the service or task" + resolution: "Update service with --enable-execute-command flag, then force new deployment" + +- symptoms: "TargetNotConnectedException" + diagnosis: "SSM agent in the container cannot reach SSM endpoints" + resolution: "Create VPC endpoints for ssmmessages, ssm, and ec2messages, or ensure NAT gateway for internet access" + +- symptoms: "The execute command failed — missing permissions" + diagnosis: "Task IAM role lacks SSM permissions" + resolution: "Add ssmmessages:CreateControlChannel, ssmmessages:CreateDataChannel, ssmmessages:OpenControlChannel, ssmmessages:OpenDataChannel to task role" + +- symptoms: "ExecuteCommandAgent status is STOPPED" + diagnosis: "SSM agent crashed or container restarted" + resolution: "Force new deployment to restart tasks with fresh SSM agent, check container has enough memory for SSM agent overhead" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K6-service-connect-discovery-failures.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K6-service-connect-discovery-failures.md new file mode 100644 index 0000000..95b167e --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K6-service-connect-discovery-failures.md @@ -0,0 +1,90 @@ +--- +title: "K6 — Service Connect / Service Discovery Failures" +description: "Diagnose and remediate ECS Service Connect and AWS Cloud Map service discovery issues" +status: active +severity: HIGH +triggers: + - "service connect" + - "service discovery" + - "Cloud Map" + - "namespace" + - "SERVICE_DISCOVERY_INSTANCE_UNHEALTHY" + - "SERVICE_DISCOVERY_OPERATION_THROTTLED" + - "envoy" + - "proxy" +owner: devops-agent +objective: "Identify service-to-service communication failures and restore service discovery" +context: "ECS Service Connect uses Envoy proxy sidecars for service mesh. Service Discovery uses AWS Cloud Map DNS. Failures include namespace mismatches, proxy crashes, DNS resolution failures, and Cloud Map API throttling." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from an affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=high to find service connect/discovery errors +- Use `search` tool with instanceId and query=`service connect|service discovery|Cloud Map|namespace|envoy|proxy.*crash|UNHEALTHY` to find evidence + +SHOULD: +- Use `search` tool with query=`SERVICE_DISCOVERY_INSTANCE_UNHEALTHY|SERVICE_DISCOVERY_OPERATION_THROTTLED|port.*mapping|ingressPortOverride` to check specific failure types +- Use `network_diagnostics` tool with instanceId to check inter-service connectivity + +MAY: +- Use `search` tool with query=`DNS.*resolution|NXDOMAIN|SERVFAIL|resolve.*fail` to check DNS issues +- Use `cluster_health` tool with clusterName to check if discovery works for other services + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of discovery failures +- Determine if the issue is Service Connect (proxy), Cloud Map (DNS), or namespace configuration +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`security group|network ACL|port.*block|connection.*refused` to check network access +- Use `search` tool with query=`sidecar|ecs-service-connect|container.*definition` to check proxy container status + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the discovery failure findings +- State root cause: namespace mismatch, proxy failure, DNS issue, or permissions +- Recommend specific remediation + +SHOULD: +- Include service mesh topology showing which services can/cannot communicate +- Recommend verifying namespace configuration across all services + +## Guardrails + +escalation_conditions: + - "All inter-service communication broken" + - "Cloud Map API throttling affecting multiple services" + - "Envoy proxy crashing in a loop" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Namespace/service configuration changes: YELLOW — operator action" + - "Security group/network ACL changes: RED — requires approval" + +## Common Issues + +- symptoms: "Services cannot discover each other" + diagnosis: "Services not in the same Cloud Map namespace" + resolution: "Verify all services use the same namespace, update service configuration if needed" + +- symptoms: "SERVICE_DISCOVERY_INSTANCE_UNHEALTHY" + diagnosis: "Container health check failing, causing Cloud Map to mark instance unhealthy" + resolution: "Fix container health check, verify application responds correctly" + +- symptoms: "Service Connect proxy container crashing" + diagnosis: "Insufficient CPU/memory allocated for the Envoy sidecar" + resolution: "Add 256 CPU units and 64+ MiB memory to task definition for the proxy container" + +- symptoms: "SERVICE_DISCOVERY_OPERATION_THROTTLED" + diagnosis: "Too many Cloud Map API calls from rapid task churn" + resolution: "Reduce deployment frequency, increase task stability, contact AWS Support if persistent" + +- symptoms: "Connection refused between services using Service Connect" + diagnosis: "Security group not allowing traffic on containerPort or ingressPortOverride" + resolution: "Update security groups to allow inbound traffic from client service subnets on the Service Connect port" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K7-ebs-efs-volume-mount-failures.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K7-ebs-efs-volume-mount-failures.md new file mode 100644 index 0000000..079b85f --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K7-ebs-efs-volume-mount-failures.md @@ -0,0 +1,86 @@ +--- +title: "K7 — EBS/EFS Volume Mount Failures" +description: "Diagnose and remediate ECS task failures related to EBS volume attachment or EFS file system mounting" +status: active +severity: HIGH +triggers: + - "CannotCreateVolumeError" + - "volume mount" + - "EFS" + - "EBS" + - "mount failed" + - "file system" + - "access point" + - "transit encryption" +owner: devops-agent +objective: "Identify volume mount failures and restore persistent storage access for ECS tasks" +context: "ECS tasks can use EBS volumes (configuredAtLaunch), EFS file systems, bind mounts, and Docker volumes. Failures include IAM permission issues, security group blocking NFS traffic, EFS access point misconfiguration, and EBS attachment limits." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=high to find volume-related errors +- Use `search` tool with instanceId and query=`CannotCreateVolume|volume.*mount|EFS|EBS|mount.*fail|file.*system|access.*point` to find volume failure evidence + +SHOULD: +- Use `search` tool with query=`nfs|port 2049|security group|transit.*encryption|authorization` to check EFS connectivity +- Use `search` tool with query=`configuredAtLaunch|infrastructure.*role|ebs.*attach|volume.*type` to check EBS configuration + +MAY: +- Use `network_diagnostics` tool with instanceId to check NFS port connectivity +- Use `search` tool with query=`disk.*space|inode|throughput|burst.*credit` to check EFS performance issues + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of volume mount failures +- Determine if the issue is EBS attachment, EFS mount, permissions, or network +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`iam.*role|elasticfilesystem|ebs:CreateVolume|ebs:AttachVolume` to check IAM permissions +- Use `search` tool with query=`subnet|availability.*zone|mount.*target` to check EFS mount target availability + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the volume failure findings +- State root cause: IAM, security group, mount target, or configuration issue +- Recommend specific remediation + +SHOULD: +- Include storage architecture showing volume configuration +- Recommend security group rules for NFS (port 2049) if EFS-related + +## Guardrails + +escalation_conditions: + - "All tasks failing to mount shared EFS volume" + - "EBS volume attachment limit reached on instance" + - "Data loss risk from volume configuration changes" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Security group rule changes: YELLOW — operator action" + - "EFS/EBS configuration changes: RED — requires approval" + +## Common Issues + +- symptoms: "CannotCreateVolumeError: failed to create EBS volume" + diagnosis: "ECS infrastructure IAM role lacks ebs:CreateVolume permission or volume quota exceeded" + resolution: "Add EBS permissions to infrastructure role, check EBS volume limits in the region" + +- symptoms: "ResourceInitializationError: failed to invoke EFS utils commands" + diagnosis: "EFS mount target not available in the task's AZ or security group blocking port 2049" + resolution: "Create EFS mount target in the task's subnet AZ, allow inbound TCP 2049 from task security group" + +- symptoms: "EFS mount timeout" + diagnosis: "Security group or network ACL blocking NFS traffic between task ENI and EFS mount target" + resolution: "Allow TCP port 2049 inbound on EFS security group from task security group" + +- symptoms: "EFS access denied" + diagnosis: "EFS access point IAM authorization failing or task role missing elasticfilesystem:ClientMount" + resolution: "Add elasticfilesystem:ClientMount and ClientWrite to task role, verify access point configuration" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K8-task-stuck-pending.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K8-task-stuck-pending.md new file mode 100644 index 0000000..da395ea --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K8-task-stuck-pending.md @@ -0,0 +1,84 @@ +--- +title: "K8 — Task Stuck in PENDING State" +description: "Diagnose and remediate ECS tasks that remain in PENDING or PROVISIONING state" +status: active +severity: CRITICAL +triggers: + - "PENDING" + - "PROVISIONING" + - "stuck" + - "task not starting" + - "waiting for capacity" + - "timed out waiting" +owner: devops-agent +objective: "Identify why tasks are stuck in PENDING and restore task scheduling" +context: "Tasks stuck in PENDING/PROVISIONING indicate the scheduler cannot find suitable placement. Causes include no available instances, insufficient resources, ENI limits, subnet IP exhaustion, or capacity provider scaling delays." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId (any active instance) to gather cluster-level logs +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find pending-related errors +- Use `search` tool with instanceId and query=`PENDING|PROVISIONING|stuck|waiting.*capacity|timed out|unable to place` to find evidence + +SHOULD: +- Use `cluster_health` tool with clusterName to check cluster capacity and registered instances +- Use `search` tool with query=`ENI.*limit|network.*interface|subnet.*IP|InsufficientFreeAddresses` to check ENI/IP exhaustion + +MAY: +- Use `search` tool with query=`capacity.*provider|managed.*scaling|Auto Scaling|launch.*template` to check scaling pipeline +- Use `compare_instances` tool to check resource availability across instances + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of pending events +- Determine the bottleneck: CPU, memory, ENI, IP addresses, or instance availability +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`desired.*capacity|running.*count|pending.*count` to check demand vs supply +- Use `search` tool with query=`awsvpc|bridge|host|network.*mode` to check if awsvpc mode is causing ENI limits + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the pending task findings +- State root cause: which resource is the bottleneck +- Recommend specific remediation to unblock task scheduling + +SHOULD: +- Include capacity analysis showing available vs required resources +- Recommend ENI trunking if awsvpc mode is hitting ENI limits + +## Guardrails + +escalation_conditions: + - "Tasks stuck in PENDING for more than 15 minutes" + - "All new deployments blocked" + - "Capacity provider unable to scale out" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Instance type or count changes: YELLOW — operator action" + - "Subnet/VPC changes: RED — requires approval" + +## Common Issues + +- symptoms: "Tasks stuck in PROVISIONING — Fargate" + diagnosis: "Fargate capacity not available in the selected AZ or subnet has no IPs" + resolution: "Add subnets in multiple AZs, ensure subnets have available IP addresses" + +- symptoms: "Tasks stuck in PENDING — EC2 launch type" + diagnosis: "No container instances with sufficient CPU/memory" + resolution: "Add instances, use larger instance types, or enable capacity provider managed scaling" + +- symptoms: "Tasks stuck due to ENI limit on EC2 instance" + diagnosis: "awsvpc network mode requires one ENI per task, instance ENI limit reached" + resolution: "Enable ENI trunking (ECS_AWSVPC_TRUNKING=true), use instances with higher ENI limits, or switch to bridge network mode" + +- symptoms: "Task timed out waiting for capacity" + diagnosis: "Capacity provider scaling too slow or instances failing to register" + resolution: "Reduce instanceWarmupPeriod, check instance user data for correct cluster name, verify ECS agent connectivity" diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K9-fargate-platform-ephemeral-storage.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K9-fargate-platform-ephemeral-storage.md new file mode 100644 index 0000000..71c3c95 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/K9-fargate-platform-ephemeral-storage.md @@ -0,0 +1,88 @@ +--- +title: "K9 — Fargate Platform Version & Ephemeral Storage Issues" +description: "Diagnose and remediate Fargate platform version incompatibilities and ephemeral storage exhaustion" +status: active +severity: HIGH +triggers: + - "platform version" + - "ephemeral storage" + - "disk space" + - "no space left" + - "platform 1.3" + - "platform 1.4" + - "LATEST" + - "storage exceeded" + - "EFS" + - "volume mount" +owner: devops-agent +objective: "Resolve Fargate platform version mismatches and ephemeral storage exhaustion" +context: "Fargate tasks on platform version 1.4.0+ receive 20 GiB ephemeral storage (expandable to 200 GiB). Older platform versions (1.3.0 and earlier) only get 10 GiB for Docker layers plus 4 GiB for volume mounts. Features like EFS, Secrets Manager injection, task metadata v4, and containerd runtime require platform 1.4.0+. Using LATEST resolves to the newest version but explicit pinning can cause drift." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected task/instance +- Use `status` tool with executionId to poll until collection completes +- Use `errors` tool with instanceId and severity=critical to find storage or platform errors +- Use `search` tool with instanceId and query=`no space left|disk.*full|ephemeral.*storage|platform.*version|LATEST` to find evidence + +SHOULD: +- Use `search` tool with query=`EFS|volume.*mount|bind.*mount|storage.*exceeded` to check volume-related failures +- Use `search` tool with query=`containerd|docker|Fargate.*agent|platform.*1\.[0-3]` to detect old platform version symptoms + +MAY: +- Use `cluster_health` tool with clusterName to check if multiple tasks are affected +- Use `search` tool with query=`encryption|KMS|AES-256` to verify ephemeral storage encryption status + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build timeline of storage/platform events +- Determine whether the issue is ephemeral storage exhaustion or platform version incompatibility +- Use `validate` tool with instanceId to confirm log bundle completeness + +SHOULD: +- Use `search` tool with query=`ephemeralStorage|sizeInGiB|20.*GiB|200.*GiB` to check configured storage +- Use `search` tool with query=`task metadata|ECS_CONTAINER_METADATA_URI` to verify metadata endpoint availability + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the storage/platform findings +- State root cause: ephemeral storage limit hit or platform version too old for required features +- Recommend specific remediation + +SHOULD: +- Include storage utilization data if available +- Recommend platform version upgrade path if on older version + +## Guardrails + +escalation_conditions: + - "All Fargate tasks failing due to storage exhaustion" + - "Platform version change required across multiple services" + - "EFS mount failures blocking critical workloads" + +safety_ratings: + - "Log collection, search, errors, correlate: GREEN (read-only)" + - "Task definition ephemeralStorage update: YELLOW — operator action" + - "Platform version change: YELLOW — requires deployment" + +## Common Issues + +- symptoms: "Task fails with 'no space left on device' on Fargate" + diagnosis: "Default 20 GiB ephemeral storage exhausted by large container images or runtime data" + resolution: "Increase ephemeralStorage in task definition (up to 200 GiB). Reduce container image size. Clean temp files in entrypoint." + +- symptoms: "EFS volume mount fails on Fargate" + diagnosis: "Task using platform version older than 1.4.0 which does not support EFS" + resolution: "Update service to use platform version 1.4.0 or LATEST. Verify EFS security group allows NFS (port 2049) from task security group." + +- symptoms: "Secrets Manager injection fails on Fargate" + diagnosis: "Platform version 1.3.0 or earlier does not support Secrets Manager environment variable injection" + resolution: "Upgrade to platform version 1.4.0 or LATEST. Ensure task execution role has secretsmanager:GetSecretValue permission." + +- symptoms: "Task metadata endpoint v4 not available" + diagnosis: "Platform version older than 1.4.0 only supports metadata endpoint v3" + resolution: "Upgrade to platform version 1.4.0 or LATEST for ECS_CONTAINER_METADATA_URI_V4 support." diff --git a/mcp/aws-ecs-instance-log-mcp/sops/runbooks/Z1-general-troubleshooting.md b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/Z1-general-troubleshooting.md new file mode 100644 index 0000000..dc89c53 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/sops/runbooks/Z1-general-troubleshooting.md @@ -0,0 +1,55 @@ +--- +title: "Z1 — General ECS Troubleshooting" +description: "General troubleshooting guide for ECS issues that don't match specific SOPs" +status: active +severity: INFO +triggers: + - "general" + - "unknown" +owner: devops-agent +objective: "Provide a systematic approach to diagnosing unclassified ECS issues" +context: "Use this SOP when the issue doesn't match any specific category. Follow the systematic approach to narrow down the problem." +--- + +## Phase 1 — Triage + +MUST: +- Use `collect` tool with instanceId to gather logs from the affected container instance +- Use `status` tool with executionId to poll until collection completes +- Use `validate` tool with instanceId to verify log bundle completeness +- Use `errors` tool with instanceId and severity=all to get full error summary +- Use `cluster_health` tool with clusterName to check overall cluster state + +SHOULD: +- Use `search` tool with instanceId and query=`error|fail|denied|timeout` for broad error search +- Use `network_diagnostics` tool with instanceId and sections=all for network overview + +## Phase 2 — Enrich + +MUST: +- Use `correlate` tool with instanceId to build event timeline +- Review findings by component to identify the affected subsystem +- Use `compare_instances` tool if healthy instances are available for comparison + +SHOULD: +- Use `search` tool with targeted queries based on findings from Phase 1 + +## Phase 3 — Report + +MUST: +- Use `summarize` tool with instanceId and finding_ids from the most relevant findings +- Document what was found and what was ruled out +- Recommend next steps for further investigation + +## Systematic Approach + +1. Check ECS agent connectivity (G1, G2 SOPs) +2. Check task startup (A1, A2 SOPs) +3. Check image pulls (B1, B2, B3 SOPs) +4. Check IAM/secrets (C1, C2 SOPs) +5. Check resources (D1, D2, D3 SOPs) +6. Check networking (E1, E2, E3 SOPs) +7. Check health checks (F1, F2 SOPs) +8. Check logging (H1 SOP) +9. Check deployments (I1 SOP) +10. Check container runtime (J1 SOP) diff --git a/mcp/aws-ecs-instance-log-mcp/src/ecs-log-gateway-construct-v2.ts b/mcp/aws-ecs-instance-log-mcp/src/ecs-log-gateway-construct-v2.ts new file mode 100644 index 0000000..913b522 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/src/ecs-log-gateway-construct-v2.ts @@ -0,0 +1,1450 @@ +import * as cdk from 'aws-cdk-lib'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as cognito from 'aws-cdk-lib/aws-cognito'; +import * as kms from 'aws-cdk-lib/aws-kms'; +import * as s3n from 'aws-cdk-lib/aws-s3-notifications'; +import * as cr from 'aws-cdk-lib/custom-resources'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import * as s3deploy from 'aws-cdk-lib/aws-s3-deployment'; +import * as path from 'path'; +import { Construct } from 'constructs'; + +export interface EcsLogGatewayV2Props { + readonly gatewayName?: string; + readonly cognitoUserPoolName?: string; + readonly resourceServerName?: string; + readonly logRetentionDays?: number; + readonly enableKmsEncryption?: boolean; + readonly sopBucketName?: string; + + /** + * Explicit list of AWS regions permitted for cross-region SSM execution. + * Used for IAM aws:RequestedRegion conditions and Lambda ALLOWED_REGIONS env var. + * @default [Stack region] + */ + readonly allowedRegions?: string[]; + + /** + * IAM role ARNs for ECS container instances that need KMS encrypt and S3 PutObject access. + * When provided, replaces AnyPrincipal/AccountRootPrincipal with these specific roles. + * When omitted, backward-compatible permissive policies are retained. + * @default undefined (backward-compatible mode) + */ + readonly ecsInstanceRoleArns?: string[]; + + /** + * Presigned URL expiration in seconds for all S3 presigned URLs generated by the Lambda. + * @default 900 + */ + readonly presignedUrlExpirationSeconds?: number; + + /** + * IAM role ARN for the SSM Default Host Management Configuration role. + * This role needs S3 PutObject and KMS Encrypt to upload collected logs. + * @default undefined + */ + readonly ssmDefaultHostRoleArn?: string; +} + +export class EcsLogGatewayConstructV2 extends Construct { + public readonly logsBucket: s3.Bucket; + public readonly kmsKey: kms.Key | undefined; + public readonly ssmAutomationFunction: lambda.Function; + public readonly unzipFunction: lambda.Function; + public readonly findingsIndexerFunction: lambda.Function; + public readonly userPool: cognito.UserPool; + public readonly userPoolClient: cognito.UserPoolClient; + public readonly ssmAutomationRole: iam.Role; + public readonly gatewayExecutionRole: iam.Role; + public readonly sopBucket: s3.Bucket; + + constructor(scope: Construct, id: string, props: EcsLogGatewayV2Props = {}) { + super(scope, id); + + const gatewayName = props.gatewayName ?? 'EcsInstanceLogMcpGW'; + const cognitoUserPoolName = props.cognitoUserPoolName ?? 'ecs-log-gateway-pool'; + const resourceServerName = props.resourceServerName ?? 'ecs-log-gateway-id'; + const logRetentionDays = props.logRetentionDays ?? 1; + const enableKmsEncryption = props.enableKmsEncryption ?? true; + + // Default allowedRegions to the stack's region if not provided or empty + const allowedRegions = (props.allowedRegions && props.allowedRegions.length > 0) + ? props.allowedRegions + : [cdk.Stack.of(this).region]; + + // ======================================================================== + // KMS KEY (Optional) + // ======================================================================== + if (enableKmsEncryption) { + this.kmsKey = new kms.Key(this, 'LogsEncryptionKey', { + alias: `${cdk.Stack.of(this).stackName}-logs-key`, + description: 'KMS key for ECS log encryption', + enableKeyRotation: true, + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + + this.kmsKey.addToResourcePolicy(new iam.PolicyStatement({ + sid: 'AllowEC2InstancesEncrypt', + effect: iam.Effect.ALLOW, + principals: [new iam.AccountRootPrincipal()], + actions: ['kms:Encrypt', 'kms:GenerateDataKey*'], + resources: ['*'], + conditions: { + StringEquals: { 'aws:PrincipalAccount': cdk.Stack.of(this).account }, + }, + })); + + // KMS key policy for ECS container instance uploads + const ecsInstanceRoleArns = props.ecsInstanceRoleArns; + if (ecsInstanceRoleArns && ecsInstanceRoleArns.length > 0) { + // Tight policy: grant encrypt-only to specific ECS instance role ARNs + this.kmsKey.addToResourcePolicy(new iam.PolicyStatement({ + sid: 'AllowECSInstanceRolesEncrypt', + effect: iam.Effect.ALLOW, + principals: ecsInstanceRoleArns.map(arn => new iam.ArnPrincipal(arn)), + actions: ['kms:GenerateDataKey', 'kms:Encrypt'], + resources: ['*'], + })); + } else { + // Backward-compatible: AnyPrincipal with account condition, reduced to encrypt-only actions + this.kmsKey.addToResourcePolicy(new iam.PolicyStatement({ + sid: 'AllowAccountPrincipalsEncrypt', + effect: iam.Effect.ALLOW, + principals: [new iam.AnyPrincipal()], + actions: ['kms:GenerateDataKey', 'kms:GenerateDataKey*', 'kms:Encrypt'], + resources: ['*'], + conditions: { + StringEquals: { + 'aws:PrincipalAccount': cdk.Stack.of(this).account, + }, + }, + })); + } + + // Grant SSM Default Host Management role KMS encrypt access + if (props.ssmDefaultHostRoleArn) { + this.kmsKey.addToResourcePolicy(new iam.PolicyStatement({ + sid: 'AllowSSMDefaultHostRoleEncrypt', + effect: iam.Effect.ALLOW, + principals: [new iam.ArnPrincipal(props.ssmDefaultHostRoleArn)], + actions: ['kms:GenerateDataKey', 'kms:Encrypt'], + resources: ['*'], + })); + } + } + + // ======================================================================== + // IAM ROLES + // ======================================================================== + this.ssmAutomationRole = new iam.Role(this, 'SSMAutomationRole', { + roleName: `${cdk.Stack.of(this).stackName}-ssm-automation-role`, + assumedBy: new iam.ServicePrincipal('ssm.amazonaws.com'), + }); + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'ssm:StartAutomationExecution', 'ssm:StopAutomationExecution', + 'ssm:GetAutomationExecution', 'ssm:DescribeAutomationExecutions', + 'ssm:SendCommand', 'ssm:GetCommandInvocation', + 'ssm:ListCommandInvocations', 'ssm:ListCommands', + 'ssm:CancelCommand', 'ssm:GetDocument', 'ssm:DescribeDocument', + 'ssm:DescribeInstanceInformation', + ], + resources: ['*'], + })); + + this.ssmAutomationRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'ec2:DescribeInstances', 'ec2:DescribeTags', 'ec2:DescribeInstanceStatus', + 'ecs:DescribeClusters', 'ecs:ListClusters', + 'ecs:DescribeContainerInstances', 'ecs:ListContainerInstances', + ], + resources: ['*'], + })); + + // ======================================================================== + // S3 BUCKET + // ======================================================================== + const bucketProps: s3.BucketProps = { + bucketName: `${cdk.Stack.of(this).stackName.toLowerCase()}-logs-${cdk.Stack.of(this).account}`, + versioned: true, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + enforceSSL: true, + lifecycleRules: [ + { id: 'DeleteOldLogs', enabled: true, expiration: cdk.Duration.days(logRetentionDays) }, + { id: 'DeleteOldIdempotencyMappings', enabled: true, prefix: 'idempotency/', expiration: cdk.Duration.days(7) }, + { id: 'DeleteOldExecutionRegionMappings', enabled: true, prefix: 'execution-regions/', expiration: cdk.Duration.days(7) }, + { id: 'ExpireOldBaselines', enabled: true, prefix: 'baselines/', expiration: cdk.Duration.days(90) }, + ], + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }; + + if (this.kmsKey) { + Object.assign(bucketProps, { encryption: s3.BucketEncryption.KMS, encryptionKey: this.kmsKey, bucketKeyEnabled: true }); + } else { + Object.assign(bucketProps, { encryption: s3.BucketEncryption.S3_MANAGED }); + } + + this.logsBucket = new s3.Bucket(this, 'LogsBucket', bucketProps); + this.logsBucket.grantReadWrite(this.ssmAutomationRole); + if (this.kmsKey) { + this.kmsKey.grantEncryptDecrypt(this.ssmAutomationRole); + } + + // S3 bucket policy: scope to specific instance roles if provided + const uploadPrincipals: iam.IPrincipal[] = (props.ecsInstanceRoleArns && props.ecsInstanceRoleArns.length > 0) + ? props.ecsInstanceRoleArns.map(arn => new iam.ArnPrincipal(arn)) + : [new iam.AccountRootPrincipal()]; + + // Add SSM Default Host Management role if provided + if (props.ssmDefaultHostRoleArn) { + uploadPrincipals.push(new iam.ArnPrincipal(props.ssmDefaultHostRoleArn)); + } + + this.logsBucket.addToResourcePolicy(new iam.PolicyStatement({ + sid: 'AllowEC2InstancesUpload', + effect: iam.Effect.ALLOW, + principals: uploadPrincipals, + actions: ['s3:PutObject', 's3:GetBucketPolicyStatus', 's3:GetBucketAcl'], + resources: [this.logsBucket.bucketArn, `${this.logsBucket.bucketArn}/*`], + conditions: { StringEquals: { 'aws:PrincipalAccount': cdk.Stack.of(this).account } }, + })); + + // ======================================================================== + // FINDINGS INDEXER LAMBDA (triggered by Unzip after extraction) + // ======================================================================== + const findingsIndexerRole = new iam.Role(this, 'FindingsIndexerRole', { + roleName: `${cdk.Stack.of(this).stackName}-findings-indexer-role`, + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')], + }); + this.logsBucket.grantReadWrite(findingsIndexerRole); + if (this.kmsKey) { this.kmsKey.grantEncryptDecrypt(findingsIndexerRole); } + + this.findingsIndexerFunction = new lambda.Function(this, 'FindingsIndexerFunction', { + functionName: `${cdk.Stack.of(this).stackName}-findings-indexer`, + runtime: lambda.Runtime.PYTHON_3_11, + handler: 'index.lambda_handler', + role: findingsIndexerRole, + timeout: cdk.Duration.minutes(5), + memorySize: 1024, + environment: { LOGS_BUCKET_NAME: this.logsBucket.bucketName }, + code: lambda.Code.fromInline(this.getFindingsIndexerCode()), + logRetention: logs.RetentionDays.TWO_WEEKS, + }); + + // ======================================================================== + // UNZIP LAMBDA (triggers Findings Indexer after extraction) + // ======================================================================== + const unzipLambdaRole = new iam.Role(this, 'UnzipLambdaRole', { + roleName: `${cdk.Stack.of(this).stackName}-unzip-lambda-role`, + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')], + }); + this.logsBucket.grantReadWrite(unzipLambdaRole); + if (this.kmsKey) { this.kmsKey.grantEncryptDecrypt(unzipLambdaRole); } + this.findingsIndexerFunction.grantInvoke(unzipLambdaRole); + + this.unzipFunction = new lambda.Function(this, 'UnzipFunction', { + functionName: `${cdk.Stack.of(this).stackName}-unzip-function`, + runtime: lambda.Runtime.PYTHON_3_11, + handler: 'index.lambda_handler', + role: unzipLambdaRole, + timeout: cdk.Duration.minutes(5), + memorySize: 1024, + environment: { FINDINGS_INDEXER_FUNCTION: this.findingsIndexerFunction.functionName }, + code: lambda.Code.fromInline(this.getUnzipLambdaCode()), + logRetention: logs.RetentionDays.TWO_WEEKS, + }); + + this.logsBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.LambdaDestination(this.unzipFunction), { suffix: '.zip' }); + this.logsBucket.addEventNotification(s3.EventType.OBJECT_CREATED, new s3n.LambdaDestination(this.unzipFunction), { suffix: '.tar.gz' }); + + // ======================================================================== + // SSM AUTOMATION LAMBDA + // ======================================================================== + const lambdaExecutionRole = new iam.Role(this, 'LambdaExecutionRole', { + roleName: `${cdk.Stack.of(this).stackName}-lambda-execution-role`, + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')], + }); + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'ssm:StartAutomationExecution', 'ssm:GetAutomationExecution', + 'ssm:DescribeAutomationExecutions', 'ssm:StopAutomationExecution', + 'ssm:DescribeInstanceInformation', 'ssm:ListCommands', 'ssm:ListCommandInvocations', + ], + resources: ['*'], + })); + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + sid: 'SSMDocumentAccess', + effect: iam.Effect.ALLOW, + actions: ['ssm:GetDocument', 'ssm:DescribeDocument'], + resources: [ + `arn:aws:ssm:*::document/AWSSupport-CollectECSInstanceLogs`, + `arn:aws:ssm:*:${cdk.Stack.of(this).account}:document/*`, + ], + })); + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: [ + 'ec2:DescribeInstances', 'ec2:DescribeRegions', 'ec2:DescribeTags', + 'ec2:DescribeInstanceStatus', 'ec2:DescribeNetworkInterfaces', + 'ec2:DescribeSubnets', 'ec2:DescribeSecurityGroups', 'ec2:DescribeRouteTables', + 'ecs:DescribeClusters', 'ecs:ListClusters', + 'ecs:DescribeContainerInstances', 'ecs:ListContainerInstances', + 'ecs:DescribeServices', 'ecs:ListServices', + 'ecs:DescribeTasks', 'ecs:ListTasks', + 'ssm:DescribeInstanceInformation', + 'autoscaling:DescribeAutoScalingGroups', + ], + resources: ['*'], + })); + + this.logsBucket.grantReadWrite(lambdaExecutionRole); + if (this.kmsKey) { this.kmsKey.grantEncryptDecrypt(lambdaExecutionRole); } + + lambdaExecutionRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['iam:PassRole'], + resources: [this.ssmAutomationRole.roleArn], + conditions: { StringEquals: { 'iam:PassedToService': 'ssm.amazonaws.com' } }, + })); + + const lambdaEnv: { [key: string]: string } = { + LOGS_BUCKET_NAME: this.logsBucket.bucketName, + SSM_AUTOMATION_ROLE_ARN: this.ssmAutomationRole.roleArn, + ALLOWED_REGIONS: allowedRegions.join(','), + PRESIGNED_URL_EXPIRATION_SECONDS: String(props.presignedUrlExpirationSeconds ?? 900), + }; + if (this.kmsKey) { lambdaEnv['KMS_KEY_ARN'] = this.kmsKey.keyArn; } + + // SOP Bucket + this.sopBucket = new s3.Bucket(this, 'SOPBucket', { + bucketName: props.sopBucketName ?? `${cdk.Stack.of(this).stackName.toLowerCase()}-sops-${cdk.Stack.of(this).account}`, + versioned: true, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + enforceSSL: true, + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + lambdaEnv['SOP_BUCKET_NAME'] = this.sopBucket.bucketName; + this.sopBucket.grantRead(lambdaExecutionRole); + + // Auto-deploy SOP runbooks from local sops/ directory to the SOP bucket + new s3deploy.BucketDeployment(this, 'SOPDeployment', { + sources: [s3deploy.Source.asset(path.join(__dirname, '../', 'sops'))], + destinationBucket: this.sopBucket, + prune: true, + }); + + this.ssmAutomationFunction = new lambda.Function(this, 'SSMAutomationFunction', { + functionName: `${cdk.Stack.of(this).stackName}-ssm-automation`, + runtime: lambda.Runtime.PYTHON_3_11, + handler: 'ecs-log-automation.lambda_handler', + role: lambdaExecutionRole, + timeout: cdk.Duration.minutes(5), + memorySize: 1024, + environment: lambdaEnv, + code: lambda.Code.fromAsset(path.join(__dirname, 'lambda')), + logRetention: logs.RetentionDays.TWO_WEEKS, + }); + + // ======================================================================== + // GATEWAY EXECUTION ROLE + // ======================================================================== + this.gatewayExecutionRole = new iam.Role(this, 'GatewayExecutionRole', { + roleName: `${cdk.Stack.of(this).stackName}-gateway-execution-role`, + assumedBy: new iam.ServicePrincipal('bedrock-agentcore.amazonaws.com'), + }); + + this.gatewayExecutionRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['lambda:InvokeFunction'], + resources: [this.ssmAutomationFunction.functionArn], + })); + + this.gatewayExecutionRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], + resources: ['*'], + })); + + this.ssmAutomationFunction.addPermission('AllowAgentCoreInvoke', { + principal: new iam.ServicePrincipal('bedrock-agentcore.amazonaws.com'), + action: 'lambda:InvokeFunction', + sourceAccount: cdk.Stack.of(this).account, + }); + + // ======================================================================== + // COGNITO - Authentication + // ======================================================================== + this.userPool = new cognito.UserPool(this, 'UserPool', { + userPoolName: cognitoUserPoolName, + passwordPolicy: { minLength: 8, requireUppercase: false, requireLowercase: false, requireDigits: false, requireSymbols: false }, + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + + const resourceServer = new cognito.UserPoolResourceServer(this, 'ResourceServer', { + userPool: this.userPool, + identifier: resourceServerName, + scopes: [ + { scopeName: 'gateway:read', scopeDescription: 'Read access to gateway' }, + { scopeName: 'gateway:write', scopeDescription: 'Write access to gateway' }, + ], + }); + + this.userPoolClient = new cognito.UserPoolClient(this, 'UserPoolClient', { + userPool: this.userPool, + userPoolClientName: 'ecs-log-gateway-client', + generateSecret: true, + oAuth: { + flows: { clientCredentials: true }, + scopes: [ + cognito.OAuthScope.custom(`${resourceServerName}/gateway:read`), + cognito.OAuthScope.custom(`${resourceServerName}/gateway:write`), + ], + }, + supportedIdentityProviders: [cognito.UserPoolClientIdentityProvider.COGNITO], + }); + this.userPoolClient.node.addDependency(resourceServer); + + const userPoolDomain = new cognito.UserPoolDomain(this, 'UserPoolDomain', { + userPool: this.userPool, + cognitoDomain: { domainPrefix: `${cdk.Stack.of(this).stackName.toLowerCase()}-${cdk.Stack.of(this).account}` }, + }); + + // ======================================================================== + // CLIENT SECRET RETRIEVAL + // ======================================================================== + const clientSecretRetrieverRole = new iam.Role(this, 'ClientSecretRetrieverRole', { + roleName: `${cdk.Stack.of(this).stackName}-secret-retriever-role`, + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')], + }); + clientSecretRetrieverRole.addToPolicy(new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + actions: ['cognito-idp:DescribeUserPoolClient'], + resources: [this.userPool.userPoolArn], + })); + + const clientSecretRetrieverFunction = new lambda.Function(this, 'ClientSecretRetrieverFunction', { + functionName: `${cdk.Stack.of(this).stackName}-secret-retriever`, + runtime: lambda.Runtime.PYTHON_3_11, + handler: 'index.handler', + role: clientSecretRetrieverRole, + timeout: cdk.Duration.seconds(30), + code: lambda.Code.fromInline(this.getClientSecretRetrieverCode()), + }); + + const clientSecretRetriever = new cr.AwsCustomResource(this, 'ClientSecretRetriever', { + onCreate: { + service: 'Lambda', action: 'invoke', + parameters: { + FunctionName: clientSecretRetrieverFunction.functionName, + Payload: JSON.stringify({ + RequestType: 'Create', + ResourceProperties: { UserPoolId: this.userPool.userPoolId, ClientId: this.userPoolClient.userPoolClientId }, + }), + }, + physicalResourceId: cr.PhysicalResourceId.of('ClientSecretRetriever'), + }, + policy: cr.AwsCustomResourcePolicy.fromStatements([ + new iam.PolicyStatement({ actions: ['lambda:InvokeFunction'], resources: [clientSecretRetrieverFunction.functionArn] }), + ]), + }); + clientSecretRetriever.node.addDependency(this.userPoolClient); + + // ======================================================================== + // AGENTCORE GATEWAY + // ======================================================================== + const gateway = new cdk.CfnResource(this, 'AgentCoreGateway', { + type: 'AWS::BedrockAgentCore::Gateway', + properties: { + Name: gatewayName, + Description: 'ECS Instance Log Collection MCP Server with pre-indexed findings, byte-range streaming, and incident analysis', + ProtocolType: 'MCP', + AuthorizerType: 'CUSTOM_JWT', + AuthorizerConfiguration: { + CustomJWTAuthorizer: { + AllowedClients: [this.userPoolClient.userPoolClientId], + DiscoveryUrl: `https://cognito-idp.${cdk.Stack.of(this).region}.amazonaws.com/${this.userPool.userPoolId}/.well-known/openid-configuration`, + }, + }, + RoleArn: this.gatewayExecutionRole.roleArn, + }, + }); + gateway.node.addDependency(this.userPoolClient); + gateway.node.addDependency(userPoolDomain); + + const gatewayTarget = new cdk.CfnResource(this, 'LambdaGatewayTarget', { + type: 'AWS::BedrockAgentCore::GatewayTarget', + properties: { + GatewayIdentifier: gateway.ref, + Name: 'ECSInstanceLogTarget', + Description: 'ECS Instance Log Collection Target', + TargetConfiguration: { + Mcp: { + Lambda: { + LambdaArn: this.ssmAutomationFunction.functionArn, + ToolSchema: { InlinePayload: this.getToolSchemaDefinitions() }, + }, + }, + }, + CredentialProviderConfigurations: [{ CredentialProviderType: 'GATEWAY_IAM_ROLE' }], + }, + }); + gatewayTarget.node.addDependency(this.ssmAutomationFunction); + gatewayTarget.node.addDependency(this.gatewayExecutionRole); + + // ======================================================================== + // OUTPUTS + // ======================================================================== + new cdk.CfnOutput(this, 'GatewayId', { description: 'ID of the AgentCore Gateway', value: gateway.ref, exportName: `${cdk.Stack.of(this).stackName}-GatewayId` }); + new cdk.CfnOutput(this, 'GatewayUrl', { description: 'MCP Server URL', value: gateway.getAtt('GatewayUrl').toString(), exportName: `${cdk.Stack.of(this).stackName}-GatewayUrl` }); + new cdk.CfnOutput(this, 'CognitoUserPoolId', { description: 'Cognito User Pool ID', value: this.userPool.userPoolId, exportName: `${cdk.Stack.of(this).stackName}-CognitoUserPoolId` }); + new cdk.CfnOutput(this, 'CognitoClientId', { description: 'OAuth Client ID', value: this.userPoolClient.userPoolClientId, exportName: `${cdk.Stack.of(this).stackName}-CognitoClientId` }); + new cdk.CfnOutput(this, 'OAuthExchangeUrl', { description: 'OAuth Token URL', value: `https://${cdk.Stack.of(this).stackName.toLowerCase()}-${cdk.Stack.of(this).account}.auth.${cdk.Stack.of(this).region}.amazoncognito.com/oauth2/token`, exportName: `${cdk.Stack.of(this).stackName}-OAuthExchangeUrl` }); + new cdk.CfnOutput(this, 'OAuthScope', { description: 'OAuth Scope', value: `${resourceServerName}/gateway:read`, exportName: `${cdk.Stack.of(this).stackName}-OAuthScope` }); + new cdk.CfnOutput(this, 'LogsBucketName', { description: 'S3 bucket for collected logs', value: this.logsBucket.bucketName, exportName: `${cdk.Stack.of(this).stackName}-LogsBucketName` }); + new cdk.CfnOutput(this, 'SSMAutomationRoleArn', { description: 'SSM Automation Role ARN', value: this.ssmAutomationRole.roleArn, exportName: `${cdk.Stack.of(this).stackName}-SSMAutomationRoleArn` }); + if (this.kmsKey) { new cdk.CfnOutput(this, 'EncryptionKeyArn', { description: 'KMS Encryption Key ARN', value: this.kmsKey.keyArn, exportName: `${cdk.Stack.of(this).stackName}-EncryptionKeyArn` }); } + new cdk.CfnOutput(this, 'SOPBucketName', { description: 'S3 bucket for Standard Operating Procedures', value: this.sopBucket.bucketName, exportName: `${cdk.Stack.of(this).stackName}-SOPBucketName` }); + } + + private getToolSchemaDefinitions(): object[] { + return [ + // ===================================================================== + // TIER 1: CORE OPERATIONS + // ===================================================================== + { + Name: 'collect', + Description: 'Start ECS log collection from a container instance. Returns immediately with executionId for async polling. Supports idempotency tokens to prevent duplicate executions. Supports cross-region: auto-detects instance region or accepts explicit region parameter. CITATION: When presenting results, always cite the executionId and region returned.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID of the ECS container instance (e.g., i-0123456789abcdef0)' }, + idempotencyToken: { Type: 'string', Description: 'Optional token to prevent duplicate executions. If provided and a matching execution exists, returns the existing executionId.' }, + region: { Type: 'string', Description: 'AWS region where the instance runs (e.g., us-west-2). Optional: auto-detected from instance if omitted.' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + executionId: { Type: 'string', Description: 'SSM Automation execution ID for polling' }, + instanceId: { Type: 'string' }, + region: { Type: 'string' }, + status: { Type: 'string' }, + estimatedCompletionTime: { Type: 'string' }, + idempotent: { Type: 'boolean', Description: 'True if returning existing execution' }, + task: { Type: 'object', Description: 'Async task envelope for polling', Properties: { taskId: { Type: 'string' }, state: { Type: 'string', Description: 'running|completed|failed|cancelled' }, message: { Type: 'string' }, progress: { Type: 'integer', Description: '0-100 percent' } } }, + }, + }, + }, + { + Name: 'status', + Description: 'Get detailed status of a log collection execution including progress percentage, step details, and failure reasons. Automatically resolves the region where the execution was started. CITATION: Always cite executionId and status in your response.', + InputSchema: { + Type: 'object', + Properties: { + executionId: { Type: 'string', Description: 'The SSM Automation execution ID returned from collect' }, + includeStepDetails: { Type: 'boolean', Description: 'Include individual step status (default: true)' }, + region: { Type: 'string', Description: 'AWS region of the execution. Optional: auto-resolved from stored execution metadata if omitted.' }, + }, + Required: ['executionId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + automation: { Type: 'object', Properties: { executionId: { Type: 'string' }, status: { Type: 'string', Description: 'InProgress|Success|Failed|Cancelled' }, progress: { Type: 'integer' }, failureReason: { Type: 'string' }, stepDetails: { Type: 'array' } } }, + task: { Type: 'object', Properties: { taskId: { Type: 'string' }, state: { Type: 'string' }, message: { Type: 'string' }, progress: { Type: 'integer' } } }, + }, + }, + }, + { + Name: 'validate', + Description: 'Verify all expected files were extracted from the log bundle. Returns manifest with file counts, sizes, and missing patterns. Uses manifest.json when available for authoritative file inventory. CITATION: Cite fileCount, totalSizeHuman, and any missingPatterns.', + InputSchema: { + Type: 'object', + Properties: { + executionId: { Type: 'string', Description: 'SSM execution ID to validate' }, + instanceId: { Type: 'string', Description: 'Alternative: Instance ID to locate bundle' }, + }, + }, + OutputSchema: { + Type: 'object', + Properties: { + complete: { Type: 'boolean' }, fileCount: { Type: 'integer' }, totalSize: { Type: 'integer' }, totalSizeHuman: { Type: 'string' }, + missingPatterns: { Type: 'array' }, foundPatterns: { Type: 'array' }, hasFindingsIndex: { Type: 'boolean' }, + manifest: { Type: 'array', Description: 'File list with key, fullKey, size, sizeHuman' }, + }, + }, + }, + { + Name: 'errors', + Description: 'Get pre-indexed error findings (fast path). Returns categorized errors by severity with finding_ids for citation. Each finding has a stable finding_id (F-001 format). Automatically recommends relevant SOPs based on detected error patterns (recommendedSOPs field). CITATION: Always cite finding_id and severity when referencing findings. Use finding_ids with summarize tool.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to get error summary for' }, + severity: { Type: 'string', Description: 'Filter by severity: critical, high, medium, low, info, or all (default: all). Legacy "warning" maps to "high".' }, + response_format: { Type: 'string', Description: '"concise" (default) returns finding_id/severity/pattern/file/count only. "detailed" includes full sample text and line numbers.' }, + pageSize: { Type: 'integer', Description: 'Number of findings per page (default: 50, max: 200)' }, + pageToken: { Type: 'string', Description: 'Opaque token for next page (from previous response nextPageToken)' }, + clusterContext: { Type: 'string', Description: 'ECS cluster name for baseline subtraction. When provided, findings seen 10+ times are annotated as baseline (normal operation).' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, + findings: { Type: 'array', Description: 'Array of findings with finding_id, severity, pattern, file, count' }, + totalFindings: { Type: 'integer' }, + summary: { Type: 'object', Description: 'Counts by severity: critical, high, medium, low, info' }, + hasMore: { Type: 'boolean' }, nextPageToken: { Type: 'string' }, + coverage_report: { Type: 'object', Description: 'files_scanned, files_skipped, scan_complete' }, + recommendedSOPs: { Type: 'array', Description: 'Auto-matched SOP runbooks based on detected error patterns. Each entry has sopName, relevanceScore, matchedKeywords, reason. Use get_sop to retrieve full SOP content.' }, + }, + }, + }, + { + Name: 'read', + Description: 'Read a chunk of a log file using byte-range streaming. NO TRUNCATION. Supports both byte-range and line-based reading for multi-GB files. Line-aligned: byte reads snap to newline boundaries. CITATION: Cite logKey, startByte, endByte, and totalSize.', + InputSchema: { + Type: 'object', + Properties: { + logKey: { Type: 'string', Description: 'The S3 key of the log file (from validate manifest)' }, + startByte: { Type: 'integer', Description: 'Starting byte offset (default: 0). Snaps forward to next newline.' }, + endByte: { Type: 'integer', Description: 'Ending byte offset (default: startByte + 1MB). Snaps forward to next newline.' }, + startLine: { Type: 'integer', Description: 'Alternative: Starting line number (1-based)' }, + lineCount: { Type: 'integer', Description: 'Number of lines to return when using startLine (default: 1000)' }, + }, + Required: ['logKey'], + }, + OutputSchema: { + Type: 'object', + Properties: { + logKey: { Type: 'string' }, content: { Type: 'string' }, + startByte: { Type: 'integer' }, endByte: { Type: 'integer' }, totalSize: { Type: 'integer' }, + hasMore: { Type: 'boolean' }, nextChunkToken: { Type: 'string' }, + truncated: { Type: 'boolean', Description: 'Always false — never truncates' }, + lineAligned: { Type: 'boolean' }, + }, + }, + }, + // ===================================================================== + // TIER 2: ADVANCED ANALYSIS + // ===================================================================== + { + Name: 'search', + Description: 'Full-text regex search across all logs without truncation. Use for detailed investigation after reviewing error summary. CITATION: Cite finding_id (S-NNN format), file name, and match count for each result group.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to search logs for' }, + query: { Type: 'string', Description: 'Regex pattern to search for (e.g., "CannotPullContainerError|ResourceInitializationError")' }, + logTypes: { Type: 'string', Description: 'Comma-separated log types to search. Available types: ecs-agent, docker, containers, system, dmesg, networking, cgroups, metadata, gpu (default: all)' }, + maxResults: { Type: 'integer', Description: 'Maximum results per file (default: 100, max: 500)' }, + response_format: { Type: 'string', Description: '"concise" (default) or "detailed"' }, + }, + Required: ['instanceId', 'query'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, query: { Type: 'string' }, + filesSearched: { Type: 'integer' }, filesWithMatches: { Type: 'integer' }, totalMatches: { Type: 'integer' }, + results: { Type: 'array', Description: 'Array of {finding_id, file, fullKey, matchCount, matches[]}' }, + coverage_report: { Type: 'object' }, + }, + }, + }, + { + Name: 'correlate', + Description: 'Cross-file timeline correlation for incident analysis. Groups events by component, builds temporal clusters, and identifies potential root cause chains. Automatically recommends relevant SOPs based on correlated findings (recommendedSOPs field). CITATION: Cite finding_ids, confidence level, and any gaps reported.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to correlate events for' }, + timeWindow: { Type: 'integer', Description: 'Seconds around pivot event (default: 60)' }, + pivotEvent: { Type: 'string', Description: 'Event to correlate around (optional)' }, + components: { Type: 'array', Description: 'Components to include (optional)' }, + response_format: { Type: 'string', Description: '"concise" (default) or "detailed"' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, timeline: { Type: 'array' }, byComponent: { Type: 'object' }, + correlations: { Type: 'array' }, temporal_clusters: { Type: 'array' }, + potential_root_cause_chain: { Type: 'array' }, + confidence: { Type: 'string', Description: 'high|medium|low|none' }, + gaps: { Type: 'array' }, coverage_report: { Type: 'object' }, + recommendedSOPs: { Type: 'array', Description: 'Auto-matched SOP runbooks based on correlated findings. Each entry has sopName, relevanceScore, matchedKeywords, reason.' }, + }, + }, + }, + { + Name: 'artifact', + Description: 'Get secure presigned URL for large artifacts. Use for files too large to return directly. CITATION: Cite the logKey and expiresAt timestamp.', + InputSchema: { + Type: 'object', + Properties: { + logKey: { Type: 'string', Description: 'The S3 key of the artifact' }, + expirationMinutes: { Type: 'integer', Description: 'URL expiration in minutes (default: 15, max: 60)' }, + }, + Required: ['logKey'], + }, + OutputSchema: { + Type: 'object', + Properties: { + logKey: { Type: 'string' }, presignedUrl: { Type: 'string' }, expiresAt: { Type: 'string' }, sizeHuman: { Type: 'string' }, + }, + }, + }, + { + Name: 'summarize', + Description: 'Generate structured incident summary grounded in indexed findings. Pass finding_ids from errors tool to constrain summary to specific findings. Output includes grounded flag (true = all claims backed by finding_ids). Automatically recommends relevant SOPs based on findings and triage category (recommendedSOPs field). CITATION: Always cite the finding_ids that support each claim.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to summarize' }, + includeRecommendations: { Type: 'boolean', Description: 'Include remediation suggestions (default: true)' }, + finding_ids: { Type: 'array', Description: 'Optional list of finding_ids (F-001 format) from errors tool. When provided, summary is constrained to only these findings.' }, + }, + Required: ['instanceId', 'finding_ids'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, grounded: { Type: 'boolean' }, + unresolvedFindingIds: { Type: 'array' }, criticalFindings: { Type: 'array' }, + highFindings: { Type: 'array' }, affectedComponents: { Type: 'array' }, + recommendations: { Type: 'array' }, confidence: { Type: 'string' }, gaps: { Type: 'array' }, + recommendedSOPs: { Type: 'array', Description: 'Auto-matched SOP runbooks based on findings and triage category. Each entry has sopName, relevanceScore, matchedKeywords, reason. Use get_sop to retrieve full SOP content.' }, + }, + }, + }, + { + Name: 'history', + Description: 'List historical log collections for audit and comparison. Supports cross-region listing. CITATION: Cite executionId and status for each entry.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'Filter by instance ID (optional)' }, + maxResults: { Type: 'integer', Description: 'Maximum results (default: 20, max: 50)' }, + status: { Type: 'string', Description: 'Filter by status: Success, Failed, InProgress (optional)' }, + region: { Type: 'string', Description: 'AWS region to list executions from (default: Lambda region).' }, + }, + }, + OutputSchema: { + Type: 'object', + Properties: { executions: { Type: 'array' }, totalCount: { Type: 'integer' } }, + }, + }, + // ===================================================================== + // TIER 3: CLUSTER-LEVEL INTELLIGENCE + // ===================================================================== + { + Name: 'cluster_health', + Description: 'Get a comprehensive health overview of an ECS cluster. Enumerates all container instances, checks SSM agent status, instance metadata (type, AZ, AMI, launch time), ECS agent connectivity, and flags unhealthy instances. The entry point before diving into individual instance investigation. CITATION: Cite clusterName, instance counts, and any unhealthy instanceIds.', + InputSchema: { + Type: 'object', + Properties: { + clusterName: { Type: 'string', Description: 'Name of the ECS cluster to inspect' }, + region: { Type: 'string', Description: 'AWS region of the cluster (auto-detected if omitted)' }, + includeSSMStatus: { Type: 'boolean', Description: 'Check SSM agent connectivity for each instance (default: true)' }, + }, + Required: ['clusterName'], + }, + OutputSchema: { + Type: 'object', + Properties: { + clusterName: { Type: 'string' }, totalInstances: { Type: 'integer' }, + healthyInstances: { Type: 'integer' }, unhealthyInstances: { Type: 'integer' }, + instances: { Type: 'array' }, + }, + }, + }, + { + Name: 'compare_instances', + Description: 'Diff error findings and health status between two or more container instances. Surfaces what is unique to a failing instance vs. common across all instances. CITATION: Cite instanceIds compared and unique findings per instance.', + InputSchema: { + Type: 'object', + Properties: { + instanceIds: { Type: 'array', Description: 'List of 2+ EC2 instance IDs to compare (e.g., ["i-aaa", "i-bbb"])' }, + compareFields: { Type: 'string', Description: 'What to compare: "errors", "config", "all" (default: "all")' }, + }, + Required: ['instanceIds'], + }, + OutputSchema: { + Type: 'object', + Properties: { + commonFindings: { Type: 'array' }, uniqueFindings: { Type: 'object' }, insight: { Type: 'string' }, + }, + }, + }, + { + Name: 'batch_collect', + Description: 'Smart batch log collection with statistical sampling. Triages all container instances in a cluster, groups unhealthy instances into buckets by failure signature, and collects from representative samples. Use dryRun to preview before collecting. CITATION: Cite batchId, instance count, and sampling strategy used.', + InputSchema: { + Type: 'object', + Properties: { + clusterName: { Type: 'string', Description: 'Name of the ECS cluster' }, + region: { Type: 'string', Description: 'AWS region of the cluster (auto-detected if omitted)' }, + filter: { Type: 'string', Description: 'Instance filter: "all", "unhealthy", "disconnected" (default: "unhealthy")' }, + strategy: { Type: 'string', Description: '"sample" for smart sampling or "all" to collect from every filtered instance (default: "sample")' }, + samplesPerBucket: { Type: 'integer', Description: 'Instances to sample per failure bucket (default: 3, max: 5)' }, + maxTotalCollections: { Type: 'integer', Description: 'Hard cap on total collections (default: 15, max: 15)' }, + groupBy: { Type: 'string', Description: 'Grouping strategy: "auto", "az", "instance-type", "ami" (default: "auto")' }, + dryRun: { Type: 'boolean', Description: 'Preview which instances would be collected without starting (default: false)' }, + }, + Required: ['clusterName'], + }, + OutputSchema: { + Type: 'object', + Properties: { + batchId: { Type: 'string' }, executions: { Type: 'array' }, + totalInstances: { Type: 'integer' }, sampledInstances: { Type: 'integer' }, strategy: { Type: 'string' }, + task: { Type: 'object', Properties: { taskId: { Type: 'string' }, state: { Type: 'string' }, message: { Type: 'string' }, progress: { Type: 'integer' } } }, + }, + }, + }, + { + Name: 'batch_status', + Description: 'Poll status of multiple log collections at once. Returns consolidated view with allComplete boolean. Use after batch_collect to wait for all collections to finish. CITATION: Cite allComplete status and any failed executionIds.', + InputSchema: { + Type: 'object', + Properties: { + executionIds: { Type: 'array', Description: 'List of SSM execution IDs to poll (from batch_collect response)' }, + batchId: { Type: 'string', Description: 'Batch ID from batch_collect (alternative to executionIds)' }, + }, + }, + OutputSchema: { + Type: 'object', + Properties: { + allComplete: { Type: 'boolean' }, executions: { Type: 'array' }, + successCount: { Type: 'integer' }, failedCount: { Type: 'integer' }, inProgressCount: { Type: 'integer' }, + }, + }, + }, + { + Name: 'network_diagnostics', + Description: 'Extract and structure networking info from collected log bundles. Parses iptables rules, Docker networking, route tables, DNS resolution, ENI attachment status, and security group config. Returns structured data instead of raw text. Automatically recommends relevant SOPs based on detected networking issues (recommendedSOPs field). CITATION: Cite instanceId and each section analyzed.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'The EC2 instance ID to analyze networking for' }, + sections: { Type: 'string', Description: 'Comma-separated sections: "iptables,docker,routes,dns,eni,security-groups" or "all" (default: "all")' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, sections: { Type: 'object' }, + assessment: { Type: 'string' }, issues: { Type: 'array' }, + confidence: { Type: 'string' }, gaps: { Type: 'array' }, + recommendedSOPs: { Type: 'array', Description: 'Auto-matched SOP runbooks based on detected networking issues. Each entry has sopName, relevanceScore, matchedKeywords, reason.' }, + }, + }, + }, + // ===================================================================== + // TIER 4: LIVE PACKET CAPTURE + // ===================================================================== + { + Name: 'tcpdump_capture', + Description: 'Run tcpdump on an ECS container instance via SSM Run Command. Captures packets for a specified duration, generates protocol stats and decoded text, and uploads pcap/stats/text to S3. Supports capturing inside a specific ECS task network namespace via taskId. Returns commandId for async polling. CITATION: Cite commandId and instanceId.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'EC2 instance ID of the ECS container instance (required)' }, + durationSeconds: { Type: 'integer', Description: 'Capture duration in seconds (default: 120, max: 300)' }, + interface: { Type: 'string', Description: 'Network interface to capture on (default: "any")' }, + filter: { Type: 'string', Description: 'BPF filter expression (e.g., "port 443", "host 10.0.0.1")' }, + taskId: { Type: 'string', Description: 'ECS task ID to capture traffic for a specific container network namespace (optional)' }, + containerName: { Type: 'string', Description: 'Container name within the task (optional, uses first container if omitted)' }, + commandId: { Type: 'string', Description: 'If provided, polls status of an existing capture instead of starting a new one' }, + region: { Type: 'string', Description: 'AWS region where the instance runs (optional, auto-detected)' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + commandId: { Type: 'string', Description: 'SSM Command ID for polling' }, + instanceId: { Type: 'string' }, + status: { Type: 'string' }, + s3Key: { Type: 'string' }, + s3KeyTxt: { Type: 'string' }, + s3KeyStats: { Type: 'string' }, + s3Bucket: { Type: 'string' }, + task: { Type: 'object', Properties: { taskId: { Type: 'string' }, state: { Type: 'string' }, message: { Type: 'string' }, progress: { Type: 'integer' } } }, + }, + }, + }, + { + Name: 'tcpdump_analyze', + Description: 'Read and analyze a completed tcpdump capture from S3. Returns decoded packet text, protocol statistics (TCP/UDP/ICMP breakdown, port distribution, TCP flags), top talkers (source and destination IPs), and anomaly detection (high RST rates, retransmissions, SYN/RST ratios). CITATION: Cite commandId and any anomalies detected.', + InputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string', Description: 'EC2 instance ID (required)' }, + commandId: { Type: 'string', Description: 'SSM Command ID from tcpdump_capture (optional — finds latest capture if omitted)' }, + section: { Type: 'string', Description: '"summary" (decoded packets), "stats" (protocol breakdown), "all" (default: "all")' }, + maxPackets: { Type: 'integer', Description: 'Max decoded packet lines to return (default: 500, max: 3000)' }, + filter: { Type: 'string', Description: 'Text filter on decoded lines (e.g., "SYN", "RST", "10.0.0.5")' }, + }, + Required: ['instanceId'], + }, + OutputSchema: { + Type: 'object', + Properties: { + instanceId: { Type: 'string' }, + commandId: { Type: 'string' }, + captureInfo: { Type: 'object' }, + statistics: { Type: 'object', Description: 'Protocol breakdown, port distribution, TCP flags, top talkers' }, + anomalies: { Type: 'array', Description: 'Detected anomalies with type, severity, and message' }, + decodedPackets: { Type: 'object', Description: 'Decoded packet lines with totalPackets, returnedPackets, truncated flag' }, + pcapDownloadUrl: { Type: 'string', Description: 'Presigned URL to download the raw pcap file (1 hour expiry)' }, + }, + }, + }, + // ===================================================================== + // SOP MANAGEMENT TOOLS + // ===================================================================== + { + Name: 'list_sops', + Description: 'List all available Standard Operating Procedures (SOPs) in the S3 bucket. Returns name, size, and last modified date for each SOP.', + InputSchema: { + Type: 'object', + Properties: {}, + }, + OutputSchema: { + Type: 'object', + Properties: { + sops: { Type: 'array', Description: 'Array of {name, size, lastModified}' }, + count: { Type: 'integer' }, + bucket: { Type: 'string' }, + }, + }, + }, + { + Name: 'get_sop', + Description: 'Get a specific Standard Operating Procedure (SOP) by name. Returns the full content of the SOP file. Use list_sops first to discover available SOPs.', + InputSchema: { + Type: 'object', + Properties: { + sopName: { + Type: 'string', + Description: 'The name/key of the SOP file to retrieve (e.g., "runbooks/pod-crashloop.md")', + }, + }, + Required: ['sopName'], + }, + OutputSchema: { + Type: 'object', + Properties: { + sop: { Type: 'object', Description: '{name, content, size, lastModified, contentType}' }, + }, + }, + }, + ]; + } + + private getUnzipLambdaCode(): string { + return ` +import json +import boto3 +import zipfile +import tarfile +import io +import os +from datetime import datetime + +s3_client = boto3.client('s3') +lambda_client = boto3.client('lambda') + +FINDINGS_INDEXER_FUNCTION = os.environ.get('FINDINGS_INDEXER_FUNCTION', '') + +def get_content_type(file_name): + if file_name.endswith('.log') or file_name.endswith('.txt'): + return 'text/plain' + elif file_name.endswith('.json'): + return 'application/json' + elif file_name.endswith('.yaml') or file_name.endswith('.yml'): + return 'text/yaml' + return 'application/octet-stream' + +def sanitize_archive_path(file_name): + normalized = os.path.normpath(file_name) + if normalized.startswith('..') or '/../' in normalized or normalized.startswith('/'): + print(f"SECURITY: Skipping suspicious archive path: {file_name}") + return None + return normalized.lstrip('./') + +def extract_zip(bucket, key, content): + base_path = key[:-4] + extract_prefix = f"{base_path}/extracted/" + extracted_files = [] + with zipfile.ZipFile(io.BytesIO(content), 'r') as zip_ref: + for file_info in zip_ref.infolist(): + if file_info.is_dir(): + continue + file_name = sanitize_archive_path(file_info.filename) + if file_name is None: + continue + file_content = zip_ref.read(file_info.filename) + extract_key = f"{extract_prefix}{file_name}" + s3_client.put_object(Bucket=bucket, Key=extract_key, Body=file_content, ContentType=get_content_type(file_name)) + extracted_files.append(extract_key) + print(f"Extracted: {extract_key}") + return extracted_files, extract_prefix + +def extract_targz(bucket, key, content): + if key.endswith('.tar.gz'): + base_path = key[:-7] + else: + base_path = key[:-4] + extract_prefix = f"{base_path}/extracted/" + extracted_files = [] + with tarfile.open(fileobj=io.BytesIO(content), mode='r:gz') as tar_ref: + for member in tar_ref.getmembers(): + if not member.isfile(): + continue + file_name = sanitize_archive_path(member.name) + if file_name is None: + continue + file_obj = tar_ref.extractfile(member) + if file_obj is None: + continue + file_content = file_obj.read() + extract_key = f"{extract_prefix}{file_name}" + s3_client.put_object(Bucket=bucket, Key=extract_key, Body=file_content, ContentType=get_content_type(file_name)) + extracted_files.append(extract_key) + print(f"Extracted: {extract_key}") + return extracted_files, extract_prefix + +def trigger_findings_indexer(bucket, prefix, file_count): + if not FINDINGS_INDEXER_FUNCTION: + print("No findings indexer function configured, skipping") + return + try: + lambda_client.invoke(FunctionName=FINDINGS_INDEXER_FUNCTION, InvocationType='Event', Payload=json.dumps({'bucket': bucket, 'prefix': prefix, 'fileCount': file_count})) + print(f"Triggered findings indexer for prefix: {prefix}") + except Exception as e: + print(f"Failed to trigger findings indexer: {str(e)}") + +def generate_manifest(bucket, prefix, extracted_files, archive_key, archive_size): + try: + file_details = [] + for file_key in extracted_files: + try: + head = s3_client.head_object(Bucket=bucket, Key=file_key) + relative_path = file_key.split('/extracted/')[-1] if '/extracted/' in file_key else file_key + file_details.append({'key': relative_path, 'fullKey': file_key, 'size': head['ContentLength'], 'contentType': head.get('ContentType', 'application/octet-stream')}) + except Exception: + file_details.append({'key': file_key.split('/extracted/')[-1] if '/extracted/' in file_key else file_key, 'fullKey': file_key, 'size': 0, 'contentType': 'unknown'}) + manifest = {'version': 2, 'generatedAt': datetime.utcnow().isoformat() + 'Z', 'archiveKey': archive_key, 'archiveSize': archive_size, 'totalFiles': len(extracted_files), 'totalSize': sum(f['size'] for f in file_details), 'files': file_details} + manifest_key = f"{prefix}manifest.json" + s3_client.put_object(Bucket=bucket, Key=manifest_key, Body=json.dumps(manifest, default=str), ContentType='application/json') + print(f"Wrote manifest.json to {manifest_key} ({len(extracted_files)} files)") + except Exception as e: + print(f"Warning: Failed to generate manifest.json: {str(e)}") + +def cleanup_old_bundles(bucket, current_archive_key, max_bundles_to_keep=2): + """Delete old bundles for the same instance, keeping only the N most recent.""" + try: + parts = current_archive_key.split('_') + if len(parts) < 3 or not parts[0].startswith('e'): + print(f"Cannot parse instance from key: {current_archive_key}, skipping cleanup") + return + prefix_type = parts[0] + instance_id = parts[1] + bundle_prefix = f"{prefix_type}_{instance_id}_" + paginator = s3_client.get_paginator('list_objects_v2') + archives = [] + for page in paginator.paginate(Bucket=bucket, Prefix=bundle_prefix): + for obj in page.get('Contents', []): + k = obj['Key'] + if k.endswith('.tar.gz') or k.endswith('.zip'): + archives.append({'key': k, 'last_modified': obj['LastModified']}) + if len(archives) <= max_bundles_to_keep: + print(f"Only {len(archives)} bundles for {instance_id}, nothing to clean up") + return + archives.sort(key=lambda a: a['last_modified'], reverse=True) + to_delete = archives[max_bundles_to_keep:] + for old in to_delete: + old_key = old['key'] + if old_key.endswith('.tar.gz'): + extracted_prefix = old_key[:-7] + '/extracted/' + elif old_key.endswith('.zip'): + extracted_prefix = old_key[:-4] + '/extracted/' + else: + continue + del_keys = [] + for page in paginator.paginate(Bucket=bucket, Prefix=extracted_prefix): + for obj in page.get('Contents', []): + del_keys.append({'Key': obj['Key']}) + bundle_base = old_key.rsplit('.', 1)[0] if '.tar.' in old_key else old_key[:-4] + for page in paginator.paginate(Bucket=bucket, Prefix=bundle_base): + for obj in page.get('Contents', []): + del_keys.append({'Key': obj['Key']}) + del_keys.append({'Key': old_key}) + seen = set() + unique_keys = [] + for dk in del_keys: + if dk['Key'] not in seen: + seen.add(dk['Key']) + unique_keys.append(dk) + if unique_keys: + for i in range(0, len(unique_keys), 1000): + batch = unique_keys[i:i+1000] + s3_client.delete_objects(Bucket=bucket, Delete={'Objects': batch, 'Quiet': True}) + print(f"Cleaned up old bundle: {old_key} ({len(unique_keys)} objects deleted)") + print(f"Bundle cleanup complete for {instance_id}: kept {max_bundles_to_keep}, deleted {len(to_delete)} old bundles") + except Exception as e: + print(f"Warning: Bundle cleanup failed (non-fatal): {str(e)}") + +def lambda_handler(event, context): + print(f"Received event: {json.dumps(event)}") + for record in event.get('Records', []): + bucket = record['s3']['bucket']['name'] + key = record['s3']['object']['key'] + if '/extracted/' in key: + print(f"Skipping already extracted file: {key}") + continue + is_zip = key.lower().endswith('.zip') + is_targz = key.lower().endswith('.tar.gz') or key.lower().endswith('.tgz') + if not is_zip and not is_targz: + print(f"Skipping unsupported file type: {key}") + continue + print(f"Processing archive: s3://{bucket}/{key}") + try: + response = s3_client.get_object(Bucket=bucket, Key=key) + content = response['Body'].read() + if is_zip: + extracted_files, prefix = extract_zip(bucket, key, content) + else: + extracted_files, prefix = extract_targz(bucket, key, content) + print(f"Successfully extracted {len(extracted_files)} files from {key}") + generate_manifest(bucket, prefix, extracted_files, key, len(content)) + trigger_findings_indexer(bucket, prefix, len(extracted_files)) + cleanup_old_bundles(bucket, key, max_bundles_to_keep=2) + except (zipfile.BadZipFile, tarfile.TarError) as e: + print(f"Error: {key} is not a valid archive: {str(e)}") + except Exception as e: + print(f"Error processing {key}: {str(e)}") + raise e + return {'statusCode': 200, 'body': json.dumps({'message': 'Archive extraction complete'})} +`; + } + + private getFindingsIndexerCode(): string { + return ` +import json +import boto3 +import os +import re +from datetime import datetime + +s3_client = boto3.client('s3') +LOGS_BUCKET = os.environ['LOGS_BUCKET_NAME'] + +# ECS-specific scannable file patterns +SCANNABLE_FILE_PATTERNS = [ + r'ecs[-_]agent', + r'ecs[-_]init', + r'docker[/.]', + r'containerd[/-]log', + r'dmesg', + r'messages', + r'secure', + r'syslog', + r'journal', + r'containers/', + r'container-logs', + r'var_log/.*\\.log', + r'networking', +] +SCANNABLE_REGEXES = [re.compile(p, re.IGNORECASE) for p in SCANNABLE_FILE_PATTERNS] + +SKIP_FILE_PATTERNS = [ + r'sysctl', r'ethtool', r'ifconfig', r'conntrack\\.txt', + r'pkglist', r'ps\\.txt', r'ps-threads', r'top\\.txt', + r'allprocstat', r'mounts\\.txt', r'modinfo', r'iptables', + r'metrics\\.json', r'\\.json$', r'\\.yaml$', r'\\.yml$', + r'\\.conf$', r'\\.toml$', r'\\.service$', + r'docker-info', r'docker-ps', r'docker-images', + r'instance-', r'metadata', +] +SKIP_REGEXES = [re.compile(p, re.IGNORECASE) for p in SKIP_FILE_PATTERNS] + +# ECS-specific error patterns for findings indexer +ERROR_PATTERNS = { + 'critical': [ + (r'(?i)agent.*connected.*false', 'ECS Agent disconnected'), + (r'(?i)AGENT_DISCONNECTED', 'ECS Agent disconnected from cluster'), + (r'(?i)failed.*register.*container.*instance', 'Container instance registration failed'), + (r'(?i)CannotPullContainerError', 'Cannot pull container image'), + (r'(?i)CannotPullECRContainerError', 'Cannot pull ECR container image'), + (r'(?i)ResourceInitializationError', 'Resource initialization failed'), + (r'(?i)TaskFailedToStart', 'Task failed to start'), + (r'(?i)CannotStartContainerError', 'Cannot start container'), + (r'(?i)CannotCreateContainerError', 'Cannot create container'), + (r'(?i)ContainerRuntimeError', 'Container runtime error'), + (r'(?i)pull.*access.*denied', 'Image pull access denied'), + (r'(?i)repository.*does.*not.*exist', 'Repository does not exist'), + (r'(?i)unauthorized.*authentication.*required', 'Authentication required'), + (r'(?i)toomanyrequests.*Too Many Requests', 'Docker Hub rate limit'), + (r'(?i)unable to pull secrets or registry auth', 'Secrets/registry auth failed'), + (r'(?i)docker.*daemon.*not.*running', 'Docker daemon not running'), + (r'(?i)OCI runtime create failed', 'OCI runtime create failed'), + (r'(?i)no.*space.*left.*device', 'No space left on device'), + (r'(?i)invoked oom-killer', 'OOM killer invoked'), + (r'(?i)Out of memory: Kill', 'OOM kill'), + (r'(?i)Memory cgroup out of memory', 'cgroup OOM'), + (r'(?i)kernel.*panic', 'kernel panic'), + (r'(?i)segfault', 'segfault'), + (r'(?i)ENI.*allocation.*failed', 'ENI allocation failed'), + (r'(?i)InsufficientFreeAddressesInSubnet', 'IP exhaustion'), + (r'(?i)deployment circuit breaker.*triggered', 'Circuit breaker triggered'), + (r'(?i)AccessDeniedException', 'Access denied'), + (r'(?i)exit code 137', 'Container killed (OOM)'), + (r'(?i)exit code 139', 'Container segfault'), + (r'(?i)SpotInterruptionError', 'Spot interruption'), + ], + 'warning': [ + (r'(?i)health.*check.*failed', 'Health check failed'), + (r'(?i)UNHEALTHY', 'Container unhealthy'), + (r'(?i)task.*stopped', 'Task stopped'), + (r'(?i)Essential container.*exited', 'Essential container exited'), + (r'(?i)service.*was unable to place a task', 'Task placement failure'), + (r'(?i)connection.*refused', 'Connection refused'), + (r'(?i)connection.*timeout', 'Connection timeout'), + (r'(?i)TLS.*handshake.*timeout', 'TLS timeout'), + (r'(?i)DNS.*failed', 'DNS resolution failed'), + (r'(?i)memory.*pressure', 'Memory pressure'), + (r'(?i)cpu.*throttl', 'CPU throttling'), + (r'(?i)log.*driver.*error', 'Log driver error'), + (r'(?i)awslogs.*error', 'awslogs driver error'), + (r'(?i)docker.*restart', 'Docker restarted'), + (r'(?i)is not authorized to perform', 'IAM permission denied'), + (r'(?i)OOMKilled', 'Container OOMKilled'), + (r'(?i)conntrack.*table.*full', 'Conntrack table full'), + ], + 'info': [ + (r'(?i)level=(?:warn|warning)', 'log-level warning'), + (r'(?i)DEPRECATION:', 'deprecation notice'), + (r'(?i)context deadline exceeded', 'context deadline'), + (r'(?i)retrying', 'retrying operation'), + ], +} + +COMPILED_PATTERNS = {} +for _sev, _pats in ERROR_PATTERNS.items(): + COMPILED_PATTERNS[_sev] = [] + for _pat, _desc in _pats: + try: + COMPILED_PATTERNS[_sev].append((re.compile(_pat), _desc)) + except re.error: + pass + +def is_scannable(filename): + for skip_re in SKIP_REGEXES: + if skip_re.search(filename): + return False + for scan_re in SCANNABLE_REGEXES: + if scan_re.search(filename): + return True + if filename.endswith('.log') or filename.endswith('.txt'): + log_dirs = ['ecs-agent', 'docker', 'var_log', 'kernel', 'system/messages'] + return any(d in filename for d in log_dirs) + return False + +def lambda_handler(event, context): + print(f"Received event: {json.dumps(event)}") + bucket = event.get('bucket', LOGS_BUCKET) + prefix = event.get('prefix', '') + if not prefix: + return {'statusCode': 400, 'body': 'No prefix provided'} + try: + files_to_scan = [] + files_skipped = 0 + paginator = s3_client.get_paginator('list_objects_v2') + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get('Contents', []): + key = obj['Key'] + if any(key.endswith(ext) for ext in ['.tar.gz', '.zip', '.gz', '.bin', '.so']): + continue + if obj['Size'] > 5242880: + continue + filename = key.split('/extracted/')[-1] if '/extracted/' in key else key + if not is_scannable(filename): + files_skipped += 1 + continue + files_to_scan.append({'key': key, 'size': obj['Size']}) + print(f"Scanning {len(files_to_scan)} log files ({files_skipped} non-log files skipped)") + all_findings = [] + for file_info in files_to_scan[:200]: + findings = scan_file(bucket, file_info['key']) + all_findings.extend(findings) + deduplicated = deduplicate_findings(all_findings) + + # False positive suppression + FALSE_POSITIVE_SUPPRESSIONS = [ + ('DNS.*failed', r'health[-.]?check|readiness|liveness', 'Health check DNS'), + ('OOMKilled', r'stress[-.]?test|load[-.]?test|chaos', 'Stress test'), + ('connection refused', r'127\\.0\\.0\\.1.*healthz', 'Local healthz'), + ('TLS handshake', r'health[-.]?check', 'Probe TLS'), + ] + suppressed_count = 0 + filtered = [] + for f in deduplicated: + suppressed = False + pat = f.get('pattern', '') + ctx = f.get('sample', '') + for err_pat, fp_regex, reason in FALSE_POSITIVE_SUPPRESSIONS: + if re.search(err_pat, pat, re.IGNORECASE): + if re.search(fp_regex, ctx, re.IGNORECASE): + suppressed = True + suppressed_count += 1 + break + if not suppressed: + filtered.append(f) + deduplicated = filtered + + for idx, finding in enumerate(deduplicated): + finding['finding_id'] = f"F-{idx + 1:03d}" + finding['evidence'] = {'source_file': finding.get('file', ''), 'full_key': finding.get('fullKey', ''), 'excerpt': finding.get('sample', '')[:500], 'line_range': {'start': finding.get('line', 0), 'end': finding.get('line', 0)}} + + # Multi-signal confirmation for critical findings + pattern_files = {} + for f in deduplicated: + p = f.get('pattern', '') + if p not in pattern_files: + pattern_files[p] = set() + pattern_files[p].add(f.get('file', '')) + for f in deduplicated: + if f.get('severity') == 'critical': + sources = list(pattern_files.get(f.get('pattern', ''), set())) + f['confirmation'] = {'signals': len(sources), 'confirmed': len(sources) >= 2, 'sources': sources[:5]} + if len(sources) < 2: + f['severity_note'] = 'Single-source critical finding. Verify with additional log sources.' + + summary = { + 'critical': len([f for f in deduplicated if f.get('severity') == 'critical']), + 'high': len([f for f in deduplicated if f.get('severity') in ('warning', 'high')]), + 'medium': len([f for f in deduplicated if f.get('severity') == 'medium']), + 'low': len([f for f in deduplicated if f.get('severity') == 'low']), + 'info': len([f for f in deduplicated if f.get('severity') == 'info']), + } + total_files_in_bundle = len(files_to_scan) + files_skipped + coverage = {'files_scanned': len(files_to_scan), 'total_files': total_files_in_bundle, 'coverage_pct': round(len(files_to_scan) / max(total_files_in_bundle, 1) * 100, 1), 'files_skipped': files_skipped, 'suppressed_false_positives': suppressed_count} + index_data = {'index_version': 'v2', 'indexedAt': datetime.utcnow().isoformat(), 'prefix': prefix, 'coverage': coverage, 'filesScanned': len(files_to_scan), 'filesSkipped': files_skipped, 'findings': deduplicated[:500], 'summary': summary} + index_key = f"{prefix}findings_index.json" + s3_client.put_object(Bucket=bucket, Key=index_key, Body=json.dumps(index_data, default=str), ContentType='application/json') + print(f"Wrote findings index to {index_key}") + print(f"Summary: {summary}") + return {'statusCode': 200, 'body': json.dumps({'message': 'Findings indexed', 'indexKey': index_key, 'summary': summary})} + except Exception as e: + print(f"Error indexing findings: {str(e)}") + return {'statusCode': 500, 'body': str(e)} + +def scan_file(bucket, key): + findings = [] + try: + response = s3_client.get_object(Bucket=bucket, Key=key, Range='bytes=0-1048575') + content = response['Body'].read() + try: + content_str = content.decode('utf-8') + except: + content_str = content.decode('latin-1', errors='ignore') + filename = key.split('/extracted/')[-1] if '/extracted/' in key else key + lines = content_str.split('\\n')[:5000] + for severity, compiled_list in COMPILED_PATTERNS.items(): + for regex, description in compiled_list: + match_count = 0 + for i, line in enumerate(lines): + if regex.search(line): + findings.append({'file': filename, 'fullKey': key, 'severity': severity, 'pattern': description, 'line': i + 1, 'sample': line.strip()[:300]}) + match_count += 1 + if match_count > 50 or len(findings) > 200: + break + if len(findings) > 200: + return findings + except Exception as e: + print(f"Error scanning {key}: {str(e)}") + return findings + +def deduplicate_findings(findings): + seen = {} + for finding in findings: + dedup_key = f"{finding.get('file')}:{finding.get('pattern')}" + if dedup_key not in seen: + seen[dedup_key] = {**finding, 'count': 1, 'lines': [finding.get('line')], 'first_seen': finding.get('sample', '')[:50], 'last_seen': finding.get('sample', '')[:50]} + else: + seen[dedup_key]['count'] += 1 + if len(seen[dedup_key]['lines']) < 10: + seen[dedup_key]['lines'].append(finding.get('line')) + seen[dedup_key]['last_seen'] = finding.get('sample', '')[:50] + severity_map = {'warning': 'high'} + for entry in seen.values(): + old_sev = entry.get('severity', 'info') + if old_sev in severity_map: + entry['severity'] = severity_map[old_sev] + severity_order = {'critical': 0, 'high': 1, 'medium': 2, 'low': 3, 'info': 4} + return sorted(seen.values(), key=lambda x: (severity_order.get(x.get('severity'), 4), -x.get('count', 0))) +`; + } + + /** + * Returns the Client Secret Retriever Lambda function code + */ + private getClientSecretRetrieverCode(): string { + return ` +import boto3 +import json + +def handler(event, context): + try: + if isinstance(event, str): + event = json.loads(event) + if 'Payload' in event: + event = json.loads(event['Payload']) + if event.get('RequestType') == 'Delete': + return {'statusCode': 200, 'body': json.dumps({'ClientSecret': ''})} + + props = event.get('ResourceProperties', event) + user_pool_id = props['UserPoolId'] + client_id = props['ClientId'] + + cognito = boto3.client('cognito-idp') + response = cognito.describe_user_pool_client( + UserPoolId=user_pool_id, + ClientId=client_id + ) + client_secret = response['UserPoolClient'].get('ClientSecret', '') + + return { + 'statusCode': 200, + 'body': json.dumps({'ClientSecret': client_secret}) + } + except Exception as e: + print(f"Error: {str(e)}") + return {'statusCode': 500, 'body': json.dumps({'Error': str(e)})} +`; + } +} diff --git a/mcp/aws-ecs-instance-log-mcp/src/ecs-log-gateway-stack-v2.ts b/mcp/aws-ecs-instance-log-mcp/src/ecs-log-gateway-stack-v2.ts new file mode 100644 index 0000000..b568cb2 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/src/ecs-log-gateway-stack-v2.ts @@ -0,0 +1,31 @@ +import * as cdk from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import { EcsLogGatewayConstructV2, EcsLogGatewayV2Props } from './ecs-log-gateway-construct-v2'; + +export interface EcsLogGatewayStackV2Props extends cdk.StackProps { + /** + * Properties for the ECS Log Gateway construct + */ + readonly gatewayProps?: EcsLogGatewayV2Props; +} + +/** + * Production-grade ECS Instance Log Collection MCP Server Stack + * + * Features: + * - Async task pattern with SSM Run Command + * - Byte-range streaming for large files + * - Manifest validation and completeness verification + * - AI-ready incident summary generation + * - KMS encryption at rest + * - Comprehensive ECS/Docker log collection + */ +export class EcsLogGatewayStackV2 extends cdk.Stack { + public readonly gateway: EcsLogGatewayConstructV2; + + constructor(scope: Construct, id: string, props?: EcsLogGatewayStackV2Props) { + super(scope, id, props); + + this.gateway = new EcsLogGatewayConstructV2(this, 'EcsLogGateway', props?.gatewayProps); + } +} diff --git a/mcp/aws-ecs-instance-log-mcp/src/index.ts b/mcp/aws-ecs-instance-log-mcp/src/index.ts new file mode 100644 index 0000000..8959fab --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/src/index.ts @@ -0,0 +1,2 @@ +export * from './ecs-log-gateway-construct-v2'; +export * from './ecs-log-gateway-stack-v2'; diff --git a/mcp/aws-ecs-instance-log-mcp/src/lambda/ecs-log-automation.py b/mcp/aws-ecs-instance-log-mcp/src/lambda/ecs-log-automation.py new file mode 100644 index 0000000..8dddfd5 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/src/lambda/ecs-log-automation.py @@ -0,0 +1,4837 @@ +""" +ECS Instance Log Collection MCP Server - Enhanced Lambda Handler +Provides comprehensive ECS container instance log collection and analysis via SSM. +Similar to EKS Node Log MCP but tailored for ECS EC2 instances. +""" + +import json +import boto3 +import os +import re +import time +import hashlib +from datetime import datetime, timezone, timedelta +from typing import Dict, Any, List, Optional, Tuple +from concurrent.futures import ThreadPoolExecutor, as_completed +from enum import Enum +from botocore.exceptions import ClientError + +# AWS Clients - default region (where Lambda runs) +# S3 client uses SigV4 explicitly — required for presigned URLs on KMS-encrypted buckets +from botocore.config import Config as BotoConfig +ssm_client = boto3.client('ssm') +s3_client = boto3.client('s3', config=BotoConfig(signature_version='s3v4')) +ec2_client = boto3.client('ec2') +ecs_client = boto3.client('ecs') + +# Regional client cache +_regional_clients: Dict[str, Any] = {} + +# Environment variables +LOGS_BUCKET = os.environ.get('LOGS_BUCKET_NAME', '') +SSM_AUTOMATION_ROLE_ARN = os.environ.get('SSM_AUTOMATION_ROLE_ARN', '') +KMS_KEY_ARN = os.environ.get('KMS_KEY_ARN', '') +DEFAULT_REGION = os.environ.get('AWS_REGION', 'us-east-1') + +# Constants +MAX_CONCURRENT_READS = 10 +DEFAULT_MAX_BYTES = 100000 +DEFAULT_CHUNK_SIZE = 1048576 # 1MB +MAX_CHUNK_SIZE = 5242880 # 5MB +DEFAULT_LINE_COUNT = 1000 +MAX_LINE_COUNT = 10000 +FINDINGS_INDEX_FILE = 'findings_index.json' + + +# ============================================================================= +# PRESIGNED URL EXPIRATION — configurable via env var (T5 mitigation) +# ============================================================================= + +def _parse_presigned_url_expiration() -> int: + """Parse PRESIGNED_URL_EXPIRATION_SECONDS env var, default to 900.""" + raw = os.environ.get('PRESIGNED_URL_EXPIRATION_SECONDS', '') + try: + val = int(raw) + if val > 0: + return val + except (ValueError, TypeError): + pass + return 900 + +PRESIGNED_URL_EXPIRATION = _parse_presigned_url_expiration() + + +# ============================================================================= +# ALLOWED REGIONS — configurable via env var (T9, T11 mitigation) +# ============================================================================= + +ALLOWED_REGIONS = set( + r.strip() for r in os.environ.get('ALLOWED_REGIONS', '').split(',') + if r.strip() +) or {os.environ.get('AWS_REGION', DEFAULT_REGION)} + + +def validate_region(region: str) -> Optional[Dict]: + """ + Validate that a region is in the allowed set. + Returns None if valid, or an error response dict if invalid. + """ + if region not in ALLOWED_REGIONS: + return error_response( + 403, + f"Region '{region}' is not permitted. Allowed regions: {', '.join(sorted(ALLOWED_REGIONS))}" + ) + return None + + +def resolve_and_validate_region(arguments: Dict, instance_id: str = None) -> tuple: + """ + Resolve and validate region. Returns (region, error_response). + If error_response is not None, caller should return it immediately. + """ + region = resolve_region(arguments, instance_id) + error = validate_region(region) + return region, error + + +# ============================================================================= +# ECS INSTANCE VALIDATION — verify target is an ECS container instance (T4, T13 mitigation) +# ============================================================================= + +def validate_ecs_instance(instance_id: str, region: str) -> Optional[Dict]: + """ + Validate that an instance belongs to an ECS cluster by checking for + the ECS agent tag (aws:ecs:clusterName) or by querying ECS + ListContainerInstances across clusters. + Returns None if valid, or an error response dict if invalid. + """ + try: + regional_ec2 = get_regional_client('ec2', region) + resp = regional_ec2.describe_instances(InstanceIds=[instance_id]) + for reservation in resp.get('Reservations', []): + for instance in reservation.get('Instances', []): + tags = instance.get('Tags', []) + for tag in tags: + # ECS-managed instances have aws:ecs:clusterName tag + if tag['Key'] == 'aws:ecs:clusterName': + return None # Valid ECS instance + # Also accept ecs:cluster tag (set by some ECS AMIs) + if tag['Key'].startswith('ecs:cluster'): + return None + # Accept instances with ECS-related names + if tag['Key'] == 'Name' and 'ecs' in tag.get('Value', '').lower(): + return None + + # Fallback: try to find the instance via ECS API + try: + regional_ecs = get_regional_client('ecs', region) + clusters_resp = regional_ecs.list_clusters() + for cluster_arn in clusters_resp.get('clusterArns', [])[:10]: + ci_resp = regional_ecs.list_container_instances( + cluster=cluster_arn, + filter=f'ec2InstanceId == {instance_id}', + ) + if ci_resp.get('containerInstanceArns'): + return None # Found in an ECS cluster + except Exception as e: + print(f"Warning: ECS API fallback check failed: {str(e)}") + # Non-fatal: if ECS API fails, fall through to tag-based rejection + + return error_response( + 403, + f"Instance {instance_id} does not appear to be part of an ECS cluster " + f"(no aws:ecs:clusterName tag found and not found via ECS API). " + f"This tool is designed for ECS container instances only." + ) + except ClientError as e: + if e.response['Error']['Code'] == 'InvalidInstanceID.NotFound': + return error_response(404, f"Instance {instance_id} not found in region {region}") + return error_response(500, f"Failed to validate instance {instance_id}: {str(e)}") + + +# ============================================================================= +# TIME WINDOW RESOLVER — enforces time-bounded log analysis +# ============================================================================= + +class TimeWindowResolver: + """ + Resolves an analysis time window from user-provided incident time parameters. + + Rules: + 1. If start_time AND end_time provided: use exactly. + 2. If a single incident_time provided: window = [incident_time - 5min, incident_time + 5min]. + 3. If nothing provided: window = [now_utc - 10min, now_utc]. + + All outputs are UTC datetime objects. + """ + + DEFAULT_WINDOW_MINUTES = 10 + INCIDENT_PADDING_MINUTES = 5 + MAX_WINDOW_HOURS = 24 # safety cap + + @staticmethod + def resolve(arguments: Dict) -> Dict: + now_utc = datetime.utcnow() + incident_time_str = arguments.get('incident_time') + start_time_str = arguments.get('start_time') + end_time_str = arguments.get('end_time') + + window_start = None + window_end = None + reason = '' + + if start_time_str and end_time_str: + window_start = TimeWindowResolver._parse_timestamp(start_time_str) + window_end = TimeWindowResolver._parse_timestamp(end_time_str) + if window_start and window_end: + reason = 'explicit incident window provided' + else: + reason = 'failed to parse explicit window; default last 10 minutes' + window_start = None + window_end = None + + if window_start is None and incident_time_str: + incident_dt = TimeWindowResolver._parse_timestamp(incident_time_str) + if incident_dt: + pad = timedelta(minutes=TimeWindowResolver.INCIDENT_PADDING_MINUTES) + window_start = incident_dt - pad + window_end = incident_dt + pad + reason = f'incident time provided; applied +/- {TimeWindowResolver.INCIDENT_PADDING_MINUTES} minute padding' + else: + reason = 'failed to parse incident_time; default last 10 minutes' + + if window_start is None: + window_end = now_utc + window_start = now_utc - timedelta(minutes=TimeWindowResolver.DEFAULT_WINDOW_MINUTES) + if not reason: + reason = f'no incident time; default last {TimeWindowResolver.DEFAULT_WINDOW_MINUTES} minutes' + + # Safety cap + max_delta = timedelta(hours=TimeWindowResolver.MAX_WINDOW_HOURS) + if (window_end - window_start) > max_delta: + window_start = window_end - max_delta + reason += f' (clamped to max {TimeWindowResolver.MAX_WINDOW_HOURS}h window)' + + if window_end < window_start: + window_start, window_end = window_end, window_start + reason += ' (swapped start/end)' + + jctl_fmt = '%Y-%m-%d %H:%M:%S' + return { + 'window_start_utc': window_start, + 'window_end_utc': window_end, + 'window_start_iso': window_start.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'window_end_iso': window_end.strftime('%Y-%m-%dT%H:%M:%SZ'), + 'resolution_reason': reason, + 'journalctl_since': window_start.strftime(jctl_fmt), + 'journalctl_until': window_end.strftime(jctl_fmt), + } + + @staticmethod + def _parse_timestamp(ts_str: str) -> Optional[datetime]: + if not ts_str or not isinstance(ts_str, str): + return None + ts_str = ts_str.strip() + for fmt in [ + '%Y-%m-%dT%H:%M:%SZ', + '%Y-%m-%dT%H:%M:%S', + '%Y-%m-%dT%H:%M:%S.%fZ', + '%Y-%m-%dT%H:%M:%S.%f', + '%Y-%m-%dT%H:%M:%S%z', + '%Y-%m-%d %H:%M:%S UTC', + '%Y-%m-%d %H:%M:%S', + '%Y-%m-%d %H:%M', + ]: + try: + dt = datetime.strptime(ts_str, fmt) + if dt.tzinfo: + dt = dt.astimezone(timezone.utc).replace(tzinfo=None) + return dt + except ValueError: + continue + try: + ts_float = float(ts_str) + if 1_000_000_000 < ts_float < 2_000_000_000: + return datetime.utcfromtimestamp(ts_float) + if 1_000_000_000_000 < ts_float < 2_000_000_000_000: + return datetime.utcfromtimestamp(ts_float / 1000) + except (ValueError, OSError): + pass + return None + + @staticmethod + def is_within_window(timestamp_str: str, window: Dict) -> bool: + dt = TimeWindowResolver._parse_timestamp(timestamp_str) + if dt is None: + return True # Conservative: include if unparseable + return window['window_start_utc'] <= dt <= window['window_end_utc'] + + @staticmethod + def filter_findings_by_window(findings: List[Dict], window: Dict) -> Dict: + included = [] + excluded_count = 0 + unparseable_count = 0 + for f in findings: + sample = f.get('sample', '') or f.get('line', '') + ts_str = extract_timestamp(sample) if sample else None + if ts_str is None: + unparseable_count += 1 + included.append(f) + continue + if TimeWindowResolver.is_within_window(ts_str, window): + included.append(f) + else: + excluded_count += 1 + return { + 'findings': included, + 'excluded_outside_window': excluded_count, + 'unparseable_timestamps': unparseable_count, + 'total_before_filter': len(findings), + } + + @staticmethod + def window_metadata(window: Dict) -> Dict: + return { + 'window_start_utc': window['window_start_iso'], + 'window_end_utc': window['window_end_iso'], + 'resolution_reason': window['resolution_reason'], + } + +# ============================================================================ +# ECS ERROR PATTERNS +# ============================================================================ + +ECS_ERROR_PATTERNS = { + 'critical': [ + (r'(?i)agent.*connected.*false', 'ECS Agent disconnected'), + (r'(?i)agent.*not.*connected', 'ECS Agent not connected'), + (r'(?i)failed.*register.*container.*instance', 'Container instance registration failed'), + (r'(?i)unable.*connect.*ecs', 'Unable to connect to ECS service'), + (r'(?i)No container instances were found', 'No container instances available'), + (r'(?i)AGENT_DISCONNECTED', 'ECS Agent disconnected from cluster'), + (r'(?i)ECS Agent failed to start', 'ECS Agent startup failure'), + (r'(?i)websocket.*unable to dial', 'Agent websocket connection failed'), + (r'(?i)Error getting ECS instance credentials', 'Agent credential retrieval failed'), + (r'(?i)client version.*is too old', 'Docker API version mismatch'), + (r'(?i)CannotPullContainerError', 'Cannot pull container image'), + (r'(?i)CannotPullECRContainerError', 'Cannot pull ECR container image'), + (r'(?i)ResourceInitializationError', 'Resource initialization failed'), + (r'(?i)TaskFailedToStart', 'Task failed to start'), + (r'(?i)CannotStartContainerError', 'Cannot start container'), + (r'(?i)CannotCreateContainerError', 'Cannot create container'), + (r'(?i)ContainerRuntimeError', 'Container runtime error'), + (r'(?i)ContainerRuntimeTimeoutError', 'Container runtime timeout'), + (r'(?i)CannotCreateVolumeError', 'Cannot create volume mount'), + (r'(?i)CannotInspectContainerError', 'Cannot inspect container'), + (r'(?i)CannotStopContainerError', 'Cannot stop container'), + (r'(?i)SpotInterruptionError', 'Spot capacity interruption'), + (r'(?i)InternalError', 'AWS internal error'), + (r'(?i)pull.*access.*denied', 'Image pull access denied'), + (r'(?i)repository.*does.*not.*exist', 'Repository does not exist'), + (r'(?i)manifest.*not.*found', 'Image manifest not found'), + (r'(?i)image.*not.*found', 'Image not found'), + (r'(?i)unauthorized.*authentication.*required', 'Authentication required for image pull'), + (r'(?i)no.*basic.*auth.*credentials', 'Missing authentication credentials'), + (r'(?i)denied.*requested.*access.*resource', 'Access denied to resource'), + (r'(?i)toomanyrequests.*Too Many Requests', 'Docker Hub rate limit exceeded'), + (r'(?i)failed to resolve ref.*not found', 'Image reference not found'), + (r'(?i)net/http.*request canceled while waiting', 'Image pull network timeout'), + (r'(?i)API error \(500\).*Get https://.*ecr', 'ECR API error 500'), + (r'(?i)ecr:BatchGetImage.*not authorized', 'Cross-account ECR access denied'), + (r'(?i)inspect image has been retried', 'Image inspection retry failure'), + (r'(?i)unable to pull secrets or registry auth', 'Secrets/registry auth pull failed'), + (r'(?i)unable to retrieve secret from asm', 'Secrets Manager retrieval failed'), + (r'(?i)unable to retrieve ecr registry auth', 'ECR registry auth retrieval failed'), + (r'(?i)failed to validate logger args', 'Logger args validation failed'), + (r'(?i)execution resource retrieval failed', 'Execution resource retrieval failed'), + (r'(?i)unable to get registry auth from asm', 'Private registry auth from ASM failed'), + (r'(?i)failed to initialize logging driver', 'Logging driver initialization failed'), + (r'(?i)service call has been retried.*times', 'Service call retry exhausted'), + (r'(?i)insufficient.*cpu.*units', 'Insufficient CPU units'), + (r'(?i)insufficient.*memory', 'Insufficient memory'), + (r'(?i)insufficient.*GPU.*units', 'Insufficient GPU units'), + (r'(?i)OutOfMemoryError', 'Out of memory error'), + (r'(?i)oom.*kill', 'OOM kill detected'), + (r'(?i)Memory cgroup out of memory', 'Memory cgroup OOM'), + (r'(?i)invoked oom-killer', 'OOM killer invoked'), + (r'(?i)Killed process.*total-vm', 'Process killed by OOM'), + (r'(?i)exit code 137', 'Container killed (exit 137 - OOM)'), + (r'(?i)exit code 139', 'Container segfault (exit 139 - SIGSEGV)'), + (r'(?i)exit code 255', 'Container ENTRYPOINT/CMD failed (exit 255)'), + (r'(?i)No valid providers in chain', 'IAM credential chain error'), + (r'(?i)unable.*assume.*role', 'Unable to assume IAM role'), + (r'(?i)AccessDeniedException', 'Access denied exception'), + (r'(?i)UnauthorizedOperation', 'Unauthorized operation'), + (r'(?i)is not authorized to perform', 'IAM permission denied'), + (r'(?i)execution role.*does not have', 'Task execution role missing permissions'), + (r'(?i)task role.*does not have', 'Task role missing permissions'), + (r'(?i)AssumeRoleUnauthorizedAccess', 'Cannot assume IAM role'), + (r'(?i)ecr:GetAuthorizationToken.*denied', 'ECR GetAuthorizationToken denied'), + (r'(?i)failed.*retrieve.*secrets', 'Failed to retrieve secrets'), + (r'(?i)SecretNotFound', 'Secret not found'), + (r'(?i)ParameterNotFound', 'SSM parameter not found'), + (r'(?i)AccessDenied.*secretsmanager', 'Secrets Manager access denied'), + (r'(?i)AccessDenied.*ssm', 'SSM access denied'), + (r'(?i)ResourceNotFoundException.*secret', 'Secret resource not found'), + (r'(?i)InvalidRequestException.*secret', 'Invalid secret request'), + (r'(?i)secretsmanager:GetSecretValue.*denied', 'GetSecretValue permission denied'), + (r'(?i)ssm:GetParameters.*denied', 'SSM GetParameters permission denied'), + (r'(?i)docker.*daemon.*not.*running', 'Docker daemon not running'), + (r'(?i)cannot.*connect.*docker', 'Cannot connect to Docker'), + (r'(?i)OCI runtime create failed', 'OCI runtime create failed'), + (r'(?i)containerd.*not.*running', 'containerd not running'), + (r'(?i)no.*space.*left.*device', 'No space left on device'), + (r'(?i)disk.*full', 'Disk full'), + (r'(?i)exec format error', 'Wrong image architecture'), + (r'(?i)container_linux.go.*starting container process', 'Container process start failed'), + (r'(?i)no such file or directory.*entrypoint', 'Entrypoint not found'), + (r'(?i)permission denied.*entrypoint', 'Entrypoint permission denied'), + (r'(?i)network.*unreachable', 'Network unreachable'), + (r'(?i)ENI.*allocation.*failed', 'ENI allocation failed'), + (r'(?i)failed.*create.*network.*interface', 'Failed to create network interface'), + (r'(?i)InsufficientFreeAddressesInSubnet', 'Insufficient IP addresses in subnet'), + (r'(?i)no.*available.*IP.*addresses', 'No available IP addresses'), + (r'(?i)Timeout waiting for network interface', 'ENI provisioning timeout'), + (r'(?i)deployment circuit breaker.*triggered', 'Deployment circuit breaker triggered'), + (r'(?i)ECS Deployment Circuit Breaker was triggered', 'Circuit breaker deployment failed'), + (r'(?i)kernel.*panic', 'Kernel panic'), + (r'(?i)BUG:.*', 'Kernel bug detected'), + (r'(?i)segfault', 'Segmentation fault'), + (r'(?i)watchdog.*soft.*lockup', 'Soft lockup detected'), + ], + 'warning': [ + (r'(?i)health.*check.*failed', 'Health check failed'), + (r'(?i)UNHEALTHY', 'Container unhealthy'), + (r'(?i)health.*status.*unhealthy', 'Health status unhealthy'), + (r'(?i)target.*unhealthy', 'Target unhealthy'), + (r'(?i)failed.*health.*check', 'Failed health check'), + (r'(?i)failed container health checks', 'Container health check failure'), + (r'(?i)failed ELB health checks', 'ELB health check failure'), + (r'(?i)Instance.*port.*is unhealthy', 'Instance port unhealthy'), + (r'(?i)task.*stopped', 'Task stopped'), + (r'(?i)container.*stopped.*unexpectedly', 'Container stopped unexpectedly'), + (r'(?i)container.*exited.*non-zero', 'Container exited with non-zero code'), + (r'(?i)Essential container.*exited', 'Essential container exited'), + (r'(?i)STOPPED', 'Task/container stopped'), + (r'(?i)DEPROVISIONING', 'Task deprovisioning'), + (r'(?i)exit code 143', 'Container graceful shutdown (SIGTERM)'), + (r'(?i)exit code 1', 'Container general error'), + (r'(?i)task.*stuck.*PROVISIONING', 'Task stuck in provisioning'), + (r'(?i)task.*stuck.*PENDING', 'Task stuck in pending'), + (r'(?i)service.*was unable to place a task', 'Service placement failure'), + (r'(?i)service.*has stopped.*running tasks', 'Service stopped tasks'), + (r'(?i)deployment circuit breaker.*rolling back', 'Deployment rolling back'), + (r'(?i)service.*unable to place a task', 'Task placement failure'), + (r'(?i)service.*discovery.*failed', 'Service discovery failed'), + (r'(?i)failed.*register.*service', 'Failed to register service'), + (r'(?i)DNS.*registration.*failed', 'DNS registration failed'), + (r'(?i)target.*draining', 'Target draining'), + (r'(?i)deregistering.*target', 'Deregistering target'), + (r'(?i)failed.*register.*target', 'Failed to register target'), + (r'(?i)target-group.*is unhealthy', 'Target group unhealthy'), + (r'(?i)scaling.*activity.*failed', 'Scaling activity failed'), + (r'(?i)unable.*scale', 'Unable to scale'), + (r'(?i)capacity.*provider.*error', 'Capacity provider error'), + (r'(?i)service.*began draining connections', 'Service draining connections'), + (r'(?i)connection.*refused', 'Connection refused'), + (r'(?i)connection.*timeout', 'Connection timeout'), + (r'(?i)dial.*tcp.*timeout', 'TCP dial timeout'), + (r'(?i)i/o timeout', 'I/O timeout'), + (r'(?i)TLS.*handshake.*timeout', 'TLS handshake timeout'), + (r'(?i)DNS.*failed', 'DNS resolution failed'), + (r'(?i)no.*route.*host', 'No route to host'), + (r'(?i)packet.*dropped', 'Packets dropped'), + (r'(?i)conntrack.*table.*full', 'Conntrack table full'), + (r'(?i)Post.*dial tcp.*timeout', 'HTTP POST timeout'), + (r'(?i)memory.*pressure', 'Memory pressure'), + (r'(?i)cpu.*throttl', 'CPU throttling'), + (r'(?i)disk.*pressure', 'Disk pressure'), + (r'(?i)inode.*exhausted', 'Inodes exhausted'), + (r'(?i)image.*pull.*slow', 'Slow image pull'), + (r'(?i)layer.*already.*exists', 'Layer already exists (potential issue)'), + (r'(?i)docker.*restart', 'Docker restarted'), + (r'(?i)docker.*timeout', 'Docker timeout'), + (r'(?i)log.*driver.*error', 'Log driver error'), + (r'(?i)failed.*send.*logs', 'Failed to send logs'), + (r'(?i)CloudWatch.*error', 'CloudWatch logging error'), + (r'(?i)awslogs.*error', 'awslogs driver error'), + (r'(?i)logs:CreateLogStream.*denied', 'CreateLogStream permission denied'), + (r'(?i)performing maintenance on.*infrastructure', 'AWS infrastructure maintenance'), + (r'(?i)task retirement', 'Task retirement notice'), + (r'(?i)error', 'Error detected'), + (r'(?i)fail', 'Failure detected'), + (r'(?i)denied', 'Access denied'), + (r'(?i)refused', 'Connection refused'), + (r'(?i)timeout', 'Timeout detected'), + (r'(?i)unauthorized', 'Unauthorized'), + (r'(?i)forbidden', 'Forbidden'), + (r'(?i)backoff', 'Backoff detected'), + ], + 'info': [ + (r'(?i)warn', 'Warning'), + (r'(?i)warning', 'Warning'), + (r'(?i)unable', 'Unable to perform operation'), + (r'(?i)cannot', 'Cannot perform operation'), + (r'(?i)invalid', 'Invalid configuration'), + (r'(?i)deprecated', 'Deprecated feature'), + (r'(?i)missing', 'Missing resource'), + (r'(?i)not found', 'Resource not found'), + (r'(?i)retrying', 'Retrying operation'), + (r'(?i)slow', 'Slow operation'), + (r'(?i)delayed', 'Delayed operation'), + (r'(?i)waiting', 'Waiting for resource'), + (r'(?i)pending', 'Pending operation'), + ] +} + +ECS_TRIAGE_CATEGORIES = { + 'A': {'name': 'Task Startup Failures', 'patterns': [(r'CannotPullContainerError', 'high'), (r'CannotPullECRContainerError', 'high'), (r'ResourceInitializationError', 'high'), (r'TaskFailedToStart', 'high'), (r'CannotStartContainerError', 'high'), (r'CannotCreateContainerError', 'high'), (r'ContainerRuntimeError', 'high'), (r'ContainerRuntimeTimeoutError', 'high'), (r'CannotCreateVolumeError', 'high'), (r'CannotInspectContainerError', 'high'), (r'CannotStopContainerError', 'medium'), (r'SpotInterruptionError', 'high'), (r'InternalError', 'high')], 'log_sources': ['ecs-agent', 'docker', 'containerd'], 'description': 'Task fails to start due to image pull, resource init, or container runtime issues', 'runbook': 'AWSSupport-TroubleshootECSTaskFailedToStart'}, + 'B': {'name': 'Image Pull Issues', 'patterns': [(r'pull.*access.*denied', 'high'), (r'repository.*does.*not.*exist', 'high'), (r'manifest.*not.*found', 'high'), (r'unauthorized.*authentication', 'high'), (r'no.*basic.*auth.*credentials', 'high'), (r'ECR.*token.*expired', 'high'), (r'toomanyrequests.*Too Many Requests', 'high'), (r'failed to resolve ref.*not found', 'high'), (r'net/http.*request canceled', 'high'), (r'API error \(500\)', 'high'), (r'ecr:BatchGetImage.*not authorized', 'high'), (r'inspect image has been retried', 'high')], 'log_sources': ['ecs-agent', 'docker'], 'description': 'ECR/Docker Hub authentication, image not found, registry connectivity, rate limits', 'docs': 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_cannot_pull_image.html'}, + 'C': {'name': 'IAM/Secrets Issues', 'patterns': [(r'No valid providers in chain', 'high'), (r'AccessDeniedException', 'high'), (r'is not authorized to perform', 'high'), (r'failed.*retrieve.*secrets', 'high'), (r'SecretNotFound', 'high'), (r'ParameterNotFound', 'high'), (r'AssumeRoleUnauthorizedAccess', 'high'), (r'ecr:GetAuthorizationToken.*denied', 'high'), (r'secretsmanager:GetSecretValue.*denied', 'high'), (r'ssm:GetParameters.*denied', 'high'), (r'unable to pull secrets or registry auth', 'high'), (r'unable to retrieve secret from asm', 'high'), (r'unable to retrieve ecr registry auth', 'high')], 'log_sources': ['ecs-agent', 'messages', 'secure'], 'description': 'Task execution role, task role, secrets manager, SSM parameter store', 'docs': 'https://repost.aws/knowledge-center/ecs-unable-to-pull-secrets'}, + 'D': {'name': 'Resource Exhaustion', 'patterns': [(r'insufficient.*cpu', 'high'), (r'insufficient.*memory', 'high'), (r'OutOfMemoryError', 'high'), (r'oom.*kill', 'high'), (r'exit code 137', 'high'), (r'exit code 139', 'high'), (r'no.*space.*left', 'high'), (r'Memory cgroup out of memory', 'high'), (r'invoked oom-killer', 'high'), (r'Killed process.*total-vm', 'high')], 'log_sources': ['dmesg', 'messages', 'cgroups', 'docker'], 'description': 'CPU/memory limits, OOM kills, disk space', 'docs': 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/out-of-memory.html'}, + 'E': {'name': 'Networking Issues', 'patterns': [(r'ENI.*allocation.*failed', 'high'), (r'InsufficientFreeAddressesInSubnet', 'high'), (r'network.*unreachable', 'high'), (r'connection.*refused', 'medium'), (r'connection.*timeout', 'medium'), (r'DNS.*failed', 'medium'), (r'Timeout waiting for network interface', 'high'), (r'failed.*create.*network.*interface', 'high'), (r'i/o timeout', 'medium'), (r'dial.*tcp.*timeout', 'medium')], 'log_sources': ['networking', 'ecs-agent', 'docker'], 'description': 'ENI allocation, subnet IP exhaustion, security groups, DNS, VPC endpoints'}, + 'F': {'name': 'Health Check Failures', 'patterns': [(r'health.*check.*failed', 'high'), (r'UNHEALTHY', 'high'), (r'target.*unhealthy', 'high'), (r'Essential container.*exited', 'high'), (r'failed container health checks', 'high'), (r'failed ELB health checks', 'high'), (r'Instance.*port.*is unhealthy', 'high')], 'log_sources': ['ecs-agent', 'docker', 'containers'], 'description': 'Container health checks, ALB/NLB target health'}, + 'G': {'name': 'ECS Agent Issues', 'patterns': [(r'agent.*connected.*false', 'high'), (r'AGENT_DISCONNECTED', 'high'), (r'failed.*register.*container.*instance', 'high'), (r'No container instances were found', 'high'), (r'ECS Agent failed to start', 'high'), (r'websocket.*unable to dial', 'high'), (r'Error getting ECS instance credentials', 'high'), (r'client version.*is too old', 'high')], 'log_sources': ['ecs-agent', 'messages'], 'description': 'Agent connectivity, instance registration, cluster communication'}, + 'H': {'name': 'Logging/Monitoring Issues', 'patterns': [(r'log.*driver.*error', 'medium'), (r'failed.*send.*logs', 'medium'), (r'CloudWatch.*error', 'medium'), (r'awslogs.*error', 'medium'), (r'failed to validate logger args', 'high'), (r'failed to initialize logging driver', 'high'), (r'logs:CreateLogStream.*denied', 'high')], 'log_sources': ['docker', 'ecs-agent'], 'description': 'CloudWatch logs, FireLens, log driver configuration'}, + 'I': {'name': 'Deployment/Circuit Breaker', 'patterns': [(r'deployment circuit breaker.*triggered', 'high'), (r'ECS Deployment Circuit Breaker was triggered', 'high'), (r'deployment circuit breaker.*rolling back', 'high'), (r'service.*was unable to place a task', 'high'), (r'service.*has stopped.*running tasks', 'medium')], 'log_sources': ['ecs-agent'], 'description': 'Deployment failures, circuit breaker triggers, rollbacks', 'docs': 'https://repost.aws/knowledge-center/ecs-troubleshoot-deployment-failures'}, + 'J': {'name': 'Container Runtime Issues', 'patterns': [(r'OCI runtime create failed', 'high'), (r'exec format error', 'high'), (r'container_linux.go.*starting container process', 'high'), (r'no such file or directory.*entrypoint', 'high'), (r'permission denied.*entrypoint', 'high'), (r'docker.*daemon.*not.*running', 'high'), (r'containerd.*not.*running', 'high')], 'log_sources': ['docker', 'containerd'], 'description': 'Docker/containerd runtime errors, entrypoint issues, architecture mismatch'}, +} + +ECS_LOG_TYPE_PATTERNS = { + 'ecs-agent': ['ecs-agent', 'ecs/', 'ecs_agent', 'ecs-init', 'amazon-ecs-agent', + 'ecs.config', 'ecs_agent_data', 'agent-running-info'], + 'docker': ['docker', 'daemon.json', 'containerd', 'docker-info', 'docker-ps', + 'docker-images', 'docker-version', 'docker-stats', 'docker-not-running', + 'sysconfig-docker', 'docker-storage', 'docker.service', 'containerd.service'], + 'containers': ['containers/', 'container-logs', 'container-'], + 'system': ['messages', 'syslog', 'secure', 'audit', 'journal', 'system.log', + 'services.txt', 'top.txt', 'ps.txt', 'pkglist', 'os-release', + 'uname', 'dmidecode', 'lsmod', 'open-file', 'mounts', + 'lvdisplay', 'vgdisplay', 'pvdisplay', 'selinux'], + 'dmesg': ['dmesg'], + 'networking': ['networking', 'iptables', 'ip-', 'netstat', 'ss-', + 'brctlshow', 'ipaddrshow', 'veth'], + 'cgroups': ['cgroup', 'memory-events', 'memory-stat', 'cgroupv2', + 'system.slice', 'ecstasks.slice'], + 'metadata': ['metadata', 'instance-'], + 'gpu': ['gpu', 'nvidia', 'gpu-list', 'gpu-info', 'gpu-open-module', 'gpu-installed-kmod'], +} + + +# Pre-compile all ECS_ERROR_PATTERNS at module level +COMPILED_ERROR_PATTERNS = {} +for _sev_key, _patterns in ECS_ERROR_PATTERNS.items(): + COMPILED_ERROR_PATTERNS[_sev_key] = [] + for _pat, _desc in _patterns: + try: + COMPILED_ERROR_PATTERNS[_sev_key].append((re.compile(_pat, re.IGNORECASE), _desc)) + except re.error: + pass + +# False positive suppression patterns +FALSE_POSITIVE_PATTERNS = [ + re.compile(r'(?i)error_count["\s]*[:=]\s*0'), + re.compile(r'(?i)no\s+errors?\s+found'), + re.compile(r'(?i)errors?["\s]*[:=]\s*null'), + re.compile(r'(?i)error_rate["\s]*[:=]\s*0'), + re.compile(r'(?i)--error-'), + re.compile(r'(?i)error\.log'), + re.compile(r'(?i)if.*error'), + re.compile(r'(?i)catch.*error'), + re.compile(r'(?i)handle.*error'), +] + +# ============================================================================ +# SEVERITY +# ============================================================================ + +class Severity(Enum): + CRITICAL = 'critical' + HIGH = 'high' + MEDIUM = 'medium' + LOW = 'low' + INFO = 'info' + +SEVERITY_ORDER = {'critical': 0, 'high': 1, 'medium': 2, 'low': 3, 'info': 4} + +# ============================================================================ +# SOP KEYWORD MAP — maps issue keywords to runbook files +# ============================================================================ + +SOP_KEYWORD_MAP = { + # ── Task Startup Failures ── + 'task_startup': [ + {'sop': 'runbooks/A1-task-startup-resource-init.md', 'keywords': ['ResourceInitializationError', 'TaskFailedToStart', 'CannotStartContainerError', 'CannotCreateContainerError', 'resource.*init'], 'relevance': 'primary'}, + {'sop': 'runbooks/A2-task-startup-container-runtime.md', 'keywords': ['ContainerRuntimeError', 'ContainerRuntimeTimeoutError', 'docker.*daemon.*not.*running', 'containerd.*not.*running'], 'relevance': 'primary'}, + ], + # ── Image Pull Issues ── + 'image_pull': [ + {'sop': 'runbooks/B1-ecr-image-pull-auth.md', 'keywords': ['CannotPullECRContainerError', 'ecr:GetAuthorizationToken.*denied', 'ecr:BatchGetImage.*not authorized', 'pull.*access.*denied', 'CannotPullContainerError'], 'relevance': 'primary'}, + {'sop': 'runbooks/B2-image-not-found.md', 'keywords': ['manifest.*not.*found', 'repository.*does.*not.*exist', 'image.*not.*found', 'failed to resolve ref.*not found'], 'relevance': 'primary'}, + {'sop': 'runbooks/B3-docker-hub-rate-limit.md', 'keywords': ['toomanyrequests', 'Too Many Requests', 'rate limit', 'docker.io.*rate'], 'relevance': 'primary'}, + ], + # ── IAM / Secrets Issues ── + 'iam_secrets': [ + {'sop': 'runbooks/C1-task-execution-role-permissions.md', 'keywords': ['AccessDeniedException', 'is not authorized to perform', 'No valid providers in chain', 'AssumeRoleUnauthorizedAccess', 'execution role.*does not have', 'UnauthorizedOperation'], 'relevance': 'primary'}, + {'sop': 'runbooks/C2-secrets-manager-retrieval.md', 'keywords': ['unable to pull secrets', 'unable to retrieve secret', 'SecretNotFound', 'ParameterNotFound', 'secretsmanager:GetSecretValue.*denied', 'ssm:GetParameters.*denied', 'execution resource retrieval failed'], 'relevance': 'primary'}, + ], + # ── Resource Exhaustion ── + 'resource_exhaustion': [ + {'sop': 'runbooks/D1-oom-kill-memory.md', 'keywords': ['OutOfMemoryError', 'oom.*kill', 'Memory cgroup out of memory', 'invoked oom-killer', 'exit code 137', 'Killed process.*total-vm', 'insufficient.*memory'], 'relevance': 'primary'}, + {'sop': 'runbooks/D2-disk-space-exhaustion.md', 'keywords': ['no.*space.*left.*device', 'disk.*full', 'disk.*pressure', 'inode.*exhausted'], 'relevance': 'primary'}, + {'sop': 'runbooks/D3-cpu-throttling.md', 'keywords': ['cpu.*throttl', 'insufficient.*cpu', 'cpu.*limit'], 'relevance': 'primary'}, + ], + # ── Networking Issues ── + 'networking': [ + {'sop': 'runbooks/E1-eni-allocation-subnet-ip.md', 'keywords': ['ENI.*allocation.*failed', 'InsufficientFreeAddressesInSubnet', 'Timeout waiting for network interface', 'failed.*create.*network.*interface', 'no.*available.*IP'], 'relevance': 'primary'}, + {'sop': 'runbooks/E2-dns-resolution-failures.md', 'keywords': ['DNS.*failed', 'resolve.*fail', 'SERVFAIL', 'NXDOMAIN', 'name.*resolution', 'no.*route.*host'], 'relevance': 'primary'}, + {'sop': 'runbooks/E3-connection-timeout.md', 'keywords': ['connection.*timeout', 'network.*unreachable', 'dial.*tcp.*timeout', 'i/o timeout', 'TLS.*handshake.*timeout', 'connection.*refused'], 'relevance': 'primary'}, + ], + # ── Health Check Failures ── + 'health_checks': [ + {'sop': 'runbooks/F1-container-health-check.md', 'keywords': ['health.*check.*failed', 'UNHEALTHY', 'failed container health checks', 'health.*status.*unhealthy'], 'relevance': 'primary'}, + {'sop': 'runbooks/F2-elb-target-health.md', 'keywords': ['target.*unhealthy', 'failed ELB health checks', 'Instance.*port.*is unhealthy', 'deregistering.*target'], 'relevance': 'primary'}, + ], + # ── ECS Agent Issues ── + 'ecs_agent': [ + {'sop': 'runbooks/G1-agent-disconnected.md', 'keywords': ['agent.*connected.*false', 'AGENT_DISCONNECTED', 'websocket.*unable to dial', 'Error getting ECS instance credentials', 'agent.*not.*connected'], 'relevance': 'primary'}, + {'sop': 'runbooks/G2-instance-registration-failure.md', 'keywords': ['failed.*register.*container.*instance', 'No container instances were found', 'ECS Agent failed to start', 'client version.*is too old'], 'relevance': 'primary'}, + ], + # ── Logging/Monitoring Issues ── + 'logging': [ + {'sop': 'runbooks/H1-cloudwatch-log-driver.md', 'keywords': ['log.*driver.*error', 'failed.*send.*logs', 'awslogs.*error', 'failed to initialize logging driver', 'logs:CreateLogStream.*denied', 'CloudWatch.*error', 'failed to validate logger args'], 'relevance': 'primary'}, + ], + # ── Deployment / Circuit Breaker ── + 'deployment': [ + {'sop': 'runbooks/I1-deployment-circuit-breaker.md', 'keywords': ['deployment circuit breaker.*triggered', 'ECS Deployment Circuit Breaker', 'circuit breaker.*rolling back', 'service.*was unable to place a task', 'service.*has stopped.*running tasks'], 'relevance': 'primary'}, + ], + # ── Container Runtime Issues ── + 'container_runtime': [ + {'sop': 'runbooks/J1-oci-runtime-entrypoint.md', 'keywords': ['OCI runtime create failed', 'exec format error', 'container_linux.go.*starting container process', 'no such file or directory.*entrypoint', 'permission denied.*entrypoint'], 'relevance': 'primary'}, + ], + # ── Spot Interruption / Instance Draining ── + 'spot_interruption': [ + {'sop': 'runbooks/K1-spot-interruption-instance-draining.md', 'keywords': ['spot.*interrupt', 'instance.*drain', 'DRAINING', 'Spot Instance interruption', 'rebalance.*recommendation', 'capacity.*rebalance'], 'relevance': 'primary'}, + ], + # ── Task Placement Failures ── + 'task_placement': [ + {'sop': 'runbooks/K2-task-placement-failures.md', 'keywords': ['no container instance.*met.*requirements', 'placement.*constraint', 'placement.*strategy', 'unable to place', 'distinctInstance', 'memberOf', 'attribute:ecs'], 'relevance': 'primary'}, + ], + # ── Service Steady State Failures ── + 'service_stability': [ + {'sop': 'runbooks/K3-service-steady-state-failures.md', 'keywords': ['unable to reach steady state', 'steady state', 'has stopped.*running tasks', 'rolling back', 'service.*unstable', 'deployment.*failed'], 'relevance': 'primary'}, + ], + # ── Auto Scaling / Capacity Provider ── + 'auto_scaling': [ + {'sop': 'runbooks/K4-service-auto-scaling-issues.md', 'keywords': ['auto.*scal', 'capacity.*provider', 'managed.*scaling', 'target.*tracking', 'scaling.*policy', 'desired.*count', 'CapacityProviderReservation'], 'relevance': 'primary'}, + ], + # ── ECS Exec / SSM Failures ── + 'ecs_exec': [ + {'sop': 'runbooks/K5-ecs-exec-failures.md', 'keywords': ['ECS Exec', 'execute-command', 'ExecuteCommandAgent', 'SSM.*session', 'ssmmessages', 'session.*manager', 'exec.*failed'], 'relevance': 'primary'}, + ], + # ── Service Connect / Cloud Map ── + 'service_connect': [ + {'sop': 'runbooks/K6-service-connect-discovery-failures.md', 'keywords': ['Service Connect', 'Cloud Map', 'service.*discovery', 'namespace.*not.*found', 'servicediscovery', 'cloudmap', 'DNS.*SRV'], 'relevance': 'primary'}, + ], + # ── EBS/EFS Volume Mount Failures ── + 'volume_mount': [ + {'sop': 'runbooks/K7-ebs-efs-volume-mount-failures.md', 'keywords': ['volume.*mount.*fail', 'EBS.*attach', 'EFS.*mount', 'nfs.*timeout', 'mount.*target', 'volume.*not.*found', 'bind.*mount'], 'relevance': 'primary'}, + ], + # ── Task Stuck in PENDING ── + 'task_pending': [ + {'sop': 'runbooks/K8-task-stuck-pending.md', 'keywords': ['PENDING', 'PROVISIONING', 'stuck', 'task not starting', 'waiting for capacity', 'timed out waiting'], 'relevance': 'primary'}, + ], + # ── Fargate Platform / Ephemeral Storage ── + 'fargate_platform': [ + {'sop': 'runbooks/K9-fargate-platform-ephemeral-storage.md', 'keywords': ['platform version', 'ephemeral storage', 'no space left.*fargate', 'platform 1\\.3', 'platform 1\\.4', 'storage exceeded', 'ephemeralStorage'], 'relevance': 'primary'}, + ], + # ── API Throttling / Service Quotas ── + 'api_throttling': [ + {'sop': 'runbooks/K10-api-throttling-service-quotas.md', 'keywords': ['throttl', 'rate.*limit', 'Rate exceeded', 'TooManyRequestsException', 'Limit exceeded', 'service.*quota', 'RequestLimitExceeded', 'Operations are being throttled'], 'relevance': 'primary'}, + ], + # ── Fargate Metadata / Credential Errors ── + 'metadata_credentials': [ + {'sop': 'runbooks/K11-fargate-metadata-credential-errors.md', 'keywords': ['Missing credentials', 'could not load credentials', 'metadata.*error', 'credential.*provider', '169\\.254\\.170', 'ECS_CONTAINER_METADATA', 'IMDS'], 'relevance': 'primary'}, + ], + # ── Essential Container Exited Non-Zero ── + 'container_exit': [ + {'sop': 'runbooks/K12-essential-container-exited-nonzero.md', 'keywords': ['EssentialContainerExited', 'exit code', 'non-zero', 'exit.*137', 'exit.*139', 'exit.*143', 'SIGKILL', 'SIGTERM', 'segfault', 'essential container.*exit'], 'relevance': 'primary'}, + ], + # ── Windows Container Issues ── + 'windows': [ + {'sop': 'runbooks/K13-windows-container-issues.md', 'keywords': ['Windows', 'OS mismatch', 'operating system does not match', 'EnableTaskIAMRole', 'ECS_ENABLE_AWSLOGS_EXECUTIONROLE_OVERRIDE', 'No valid providers in chain.*windows', 'Unable to assume the role.*windows', 'Windows Server'], 'relevance': 'primary'}, + ], + # ── Task Latency / Performance ── + 'performance': [ + {'sop': 'runbooks/K14-task-latency-performance.md', 'keywords': ['latency', 'slow', 'performance', 'response time', 'TargetResponseTime', 'TTFB', 'EBS.*throttl', 'network.*throughput', 'DNS.*slow'], 'relevance': 'primary'}, + ], + # ── Docker Daemon / Agent Errors ── + 'docker_agent': [ + {'sop': 'runbooks/K15-docker-daemon-agent-errors.md', 'keywords': ['Docker.*API.*500', 'devmapper', 'thin pool', 'docker.*daemon', 'containerd.*error', 'docker\\.sock', 'storage.*driver', 'container.*runtime.*error'], 'relevance': 'primary'}, + ], +} + +# Map ECS triage categories (A-K) to SOP keyword groups +TRIAGE_CATEGORY_TO_SOP_GROUP = { + 'A': ['task_startup', 'image_pull', 'container_exit', 'fargate_platform'], + 'B': ['image_pull'], + 'C': ['iam_secrets', 'metadata_credentials'], + 'D': ['resource_exhaustion', 'performance'], + 'E': ['networking', 'service_connect'], + 'F': ['health_checks'], + 'G': ['ecs_agent', 'docker_agent'], + 'H': ['logging'], + 'I': ['deployment', 'service_stability', 'auto_scaling'], + 'J': ['container_runtime', 'docker_agent', 'windows'], + 'K': ['spot_interruption', 'task_placement', 'task_pending', 'ecs_exec', 'volume_mount', 'api_throttling'], +} + + +def match_sops_for_issues(issues: List[Dict], findings: List[Dict] = None, + triage_category: str = None, max_sops: int = 5) -> List[Dict]: + """ + Match detected issues/findings against SOP runbooks. + Returns a list of recommended SOPs with relevance and reason. + + Args: + issues: List of issue dicts from diagnostics (each has 'message' and 'section') + findings: Optional list of error findings (each has 'pattern', 'sample') + triage_category: Optional triage category ID (A-J) from ECS triage root cause + max_sops: Maximum SOPs to return + """ + scored_sops = {} # sop_name -> {score, reasons, keywords} + + # Build a combined text corpus from issues and findings for keyword matching + issue_texts = [] + for issue in (issues or []): + issue_texts.append(issue.get('message', '')) + for finding in (findings or []): + issue_texts.append(finding.get('pattern', '')) + issue_texts.append(finding.get('sample', '')[:200]) + corpus = ' '.join(issue_texts).lower() + + # If triage category is known, prioritize SOPs from that category's groups + priority_groups = set() + if triage_category and triage_category in TRIAGE_CATEGORY_TO_SOP_GROUP: + priority_groups = set(TRIAGE_CATEGORY_TO_SOP_GROUP[triage_category]) + + for group_name, sop_entries in SOP_KEYWORD_MAP.items(): + is_priority = group_name in priority_groups + for entry in sop_entries: + sop_name = entry['sop'] + matched_keywords = [] + for kw in entry['keywords']: + try: + if re.search(kw, corpus, re.IGNORECASE): + matched_keywords.append(kw) + except re.error: + if kw.lower() in corpus: + matched_keywords.append(kw) + + if matched_keywords: + if sop_name not in scored_sops: + scored_sops[sop_name] = {'score': 0, 'reasons': [], 'keywords': []} + scored_sops[sop_name]['score'] += len(matched_keywords) * 3 + if is_priority: + scored_sops[sop_name]['score'] += 5 + scored_sops[sop_name]['keywords'].extend(matched_keywords[:3]) + scored_sops[sop_name]['reasons'].append( + f"Matched {len(matched_keywords)} keyword(s) from {group_name}" + ) + + # Always include Z1 general troubleshooting if any issues exist but no specific SOPs matched + if not scored_sops and (issues or findings): + scored_sops['runbooks/Z1-general-troubleshooting.md'] = { + 'score': 1, + 'reasons': ['General troubleshooting guide for unmatched issues'], + 'keywords': [] + } + + # Sort by score descending, take top N + sorted_sops = sorted(scored_sops.items(), key=lambda x: x[1]['score'], reverse=True) + result = [] + for sop_name, info in sorted_sops[:max_sops]: + result.append({ + 'sopName': sop_name, + 'relevanceScore': info['score'], + 'matchedKeywords': list(set(info['keywords']))[:5], + 'reason': '; '.join(info['reasons'][:2]), + }) + return result + + +def normalize_severity_filter(severity_filter: str) -> list: + if severity_filter == 'all': + return ['critical', 'warning', 'info'] + if severity_filter in ('critical', 'warning', 'info'): + return [severity_filter] + return ['critical', 'warning', 'info'] + + +def assign_finding_id(index: int) -> str: + return f"F-{index:03d}" + + +# ============================================================================ +# RESPONSE HELPERS +# ============================================================================ + +def success_response(data: Dict) -> Dict: + MAX_PAYLOAD_BYTES = 5_500_000 + body = json.dumps({'success': True, **data}, default=str) + if len(body.encode('utf-8')) > MAX_PAYLOAD_BYTES: + truncated_data = {k: v for k, v in data.items() if not isinstance(v, list)} + for k, v in data.items(): + if isinstance(v, list): + trimmed = v + while trimmed: + candidate = json.dumps({ + 'success': True, **truncated_data, k: trimmed, + '_payloadTruncated': True, '_originalCount': len(v), '_returnedCount': len(trimmed), + }, default=str) + if len(candidate.encode('utf-8')) <= MAX_PAYLOAD_BYTES: + return {'statusCode': 200, 'body': candidate} + trimmed = trimmed[:len(trimmed) // 2] + truncated_data[k] = [] + body = json.dumps({'success': True, **truncated_data, '_payloadTruncated': True, '_error': 'Response too large'}, default=str) + return {'statusCode': 200, 'body': body} + + +def error_response(code: int, message: str, details: Dict = None) -> Dict: + body = {'success': False, 'error': message} + if details: + body['details'] = details + return {'statusCode': code, 'body': json.dumps(body, default=str)} + + +# ============================================================================ +# S3 SAFE HELPERS +# ============================================================================ + +def safe_s3_read_raw(bucket: str, key: str, byte_range: str = None, s3c=None) -> Optional[bytes]: + """Legacy S3 read — returns raw bytes or None.""" + try: + c = s3c or s3_client + kwargs = {'Bucket': bucket, 'Key': key} + if byte_range: + kwargs['Range'] = byte_range + return c.get_object(**kwargs)['Body'].read() + except Exception: + return None + + +def safe_s3_head_raw(bucket: str, key: str, s3c=None) -> Optional[Dict]: + """Legacy S3 head — returns raw boto3 response or None.""" + try: + c = s3c or s3_client + return c.head_object(Bucket=bucket, Key=key) + except Exception: + return None + + +def safe_s3_read(key: str, range_bytes: str = None, max_size: int = 1048576) -> Dict: + """ + EKS-compatible S3 read — returns dict with 'success', 'content' or 'error'. + NEVER raises exceptions. + """ + try: + params = {'Bucket': LOGS_BUCKET, 'Key': key} + if range_bytes: + params['Range'] = range_bytes + elif max_size: + params['Range'] = f'bytes=0-{max_size - 1}' + response = s3_client.get_object(**params) + content = response['Body'].read() + try: + content_str = content.decode('utf-8') + except UnicodeDecodeError: + content_str = content.decode('latin-1', errors='replace') + return {'success': True, 'content': content_str, 'size': len(content), + 'content_type': response.get('ContentType', 'unknown')} + except Exception as e: + return {'success': False, 'error': f'Failed to read {key}: {str(e)}', + 'error_type': 'read_error', 'content': ''} + + +def safe_s3_head(key: str) -> Dict: + """ + EKS-compatible S3 head — returns dict with 'success', 'size' or 'error'. + NEVER raises exceptions. + """ + try: + response = s3_client.head_object(Bucket=LOGS_BUCKET, Key=key) + return {'success': True, 'size': response['ContentLength'], + 'content_type': response.get('ContentType', 'unknown'), + 'last_modified': response.get('LastModified')} + except Exception as e: + return {'success': False, 'error': f'Failed to get metadata for {key}: {str(e)}', + 'error_type': 'metadata_error'} + + +def safe_s3_list(prefix: str, max_keys: int = 1000) -> Dict: + """ + Safely list S3 objects with graceful error handling. + NEVER raises exceptions - always returns a result dict. + """ + try: + all_objects = [] + paginator = s3_client.get_paginator('list_objects_v2') + for page in paginator.paginate(Bucket=LOGS_BUCKET, Prefix=prefix, PaginationConfig={'MaxItems': max_keys}): + for obj in page.get('Contents', []): + all_objects.append({ + 'key': obj['Key'], + 'size': obj['Size'], + 'last_modified': obj.get('LastModified') + }) + return { + 'success': True, + 'objects': all_objects, + 'count': len(all_objects) + } + except Exception as e: + return { + 'success': False, + 'error': f'Failed to list objects with prefix {prefix}: {str(e)}', + 'error_type': 'list_error', + 'objects': [], + 'count': 0 + } + + +def safe_s3_list_raw(bucket: str, prefix: str, max_keys: int = 1000, s3c=None) -> List[Dict]: + """Legacy list helper that returns raw S3 Contents list.""" + try: + c = s3c or s3_client + resp = c.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=max_keys) + return resp.get('Contents', []) + except Exception: + return [] + + +def find_latest_bundle_files(instance_id: str, prefix_scheme: str = 'ecs') -> Dict: + """ + Shared helper: discover the LATEST extracted bundle for an instance. + Returns only files from the most recent bundle (by last_modified timestamp). + """ + search_result = safe_s3_list(f"{prefix_scheme}_{instance_id}", max_keys=5000) + if not search_result.get('success'): + return {'success': False, 'files': [], 'all_objects': [], 'error': search_result.get('error', 'S3 list failed')} + + all_objects = search_result.get('objects', []) + bundle_files = [] + bundle_timestamps = {} + for obj in all_objects: + key = obj.get('key', '') + if '/extracted/' in key: + bundle_files.append(key) + if obj.get('last_modified'): + bundle_timestamps[key] = obj['last_modified'] + + if not bundle_files: + return {'success': False, 'files': [], 'all_objects': all_objects, 'error': f'No extracted log bundle found for {instance_id}. Run collect first.'} + + from collections import defaultdict + bundles_by_prefix = defaultdict(list) + for f in bundle_files: + prefix_part = f.split('/extracted/')[0] if '/extracted/' in f else f + bundles_by_prefix[prefix_part].append(f) + + latest_prefix = max( + bundles_by_prefix.keys(), + key=lambda p: max( + (bundle_timestamps.get(f, datetime.min.replace(tzinfo=None)) for f in bundles_by_prefix[p]), + default=datetime.min + ) + ) + latest_files = bundles_by_prefix[latest_prefix] + + bundle_age_minutes = None + bundle_collected_at = None + if bundle_timestamps: + ts_values = [ts for f in latest_files for ts in [bundle_timestamps.get(f)] if ts is not None] + if ts_values: + newest_ts = max(ts_values) + now_utc = datetime.now(timezone.utc) + if newest_ts.tzinfo is None: + newest_ts = newest_ts.replace(tzinfo=timezone.utc) + bundle_age_minutes = int((now_utc - newest_ts).total_seconds() / 60) + bundle_collected_at = newest_ts.isoformat() + + return { + 'success': True, + 'files': latest_files, + 'bundle_prefix': latest_prefix, + 'bundle_age_minutes': bundle_age_minutes, + 'bundle_collected_at': bundle_collected_at, + 'all_objects': all_objects, + } + + +# ============================================================================ +# REGIONAL CLIENT HELPERS +# ============================================================================ + +def get_regional_client(service: str, region: str) -> Any: + if region == DEFAULT_REGION: + if service == 'ssm': return ssm_client + if service == 's3': return s3_client + if service == 'ec2': return ec2_client + if service == 'ecs': return ecs_client + cache_key = f'{service}:{region}' + if cache_key not in _regional_clients: + _regional_clients[cache_key] = boto3.client(service, region_name=region) + return _regional_clients[cache_key] + + +def detect_instance_region(instance_id: str) -> Optional[str]: + start = time.time() + try: + resp = ec2_client.describe_instances(InstanceIds=[instance_id]) + if resp['Reservations']: + return DEFAULT_REGION + except Exception: + pass + common_regions = ['us-west-2', 'us-east-2', 'eu-west-1', 'eu-central-1', 'ap-southeast-1', 'ap-northeast-1', 'ap-south-1', 'us-west-1', 'eu-west-2', 'eu-north-1', 'ap-southeast-2', 'ap-northeast-2', 'sa-east-1', 'ca-central-1'] + common_regions = [r for r in common_regions if r != DEFAULT_REGION] + for region in common_regions: + if time.time() - start > 20: + return None + try: + regional_ec2 = get_regional_client('ec2', region) + resp = regional_ec2.describe_instances(InstanceIds=[instance_id]) + if resp['Reservations']: + return region + except Exception: + continue + return None + + +def resolve_region(arguments: Dict, instance_id: str = None) -> str: + explicit = arguments.get('region') + if explicit and re.match(r'^[a-z]{2}(-[a-z]+-\d+)$', explicit): + return explicit + if instance_id: + detected = detect_instance_region(instance_id) + if detected: + return detected + return DEFAULT_REGION + + +# ============================================================================ +# UTILITY HELPERS +# ============================================================================ + +def format_bytes(size: int) -> str: + for unit in ['B', 'KB', 'MB', 'GB']: + if size < 1024: + return f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} TB" + + +def parse_failure_reason(invocation: Dict) -> str: + """Parse failure reason from SSM RunCommand invocation.""" + status = invocation.get('Status', invocation.get('StatusDetails', '')) + stderr = invocation.get('StandardErrorContent', '') + if stderr: + lines = [l.strip() for l in stderr.strip().split('\n') if l.strip()] + if lines: + return f"{status}: {lines[-1][:200]}" + return status + + +def estimate_progress(invocation: Dict) -> Dict: + """Estimate progress from RunCommand invocation.""" + status = invocation.get('Status', '') + status_map = { + 'Pending': {'percent': 5, 'step': 'Queued'}, + 'InProgress': {'percent': 50, 'step': 'Collecting logs'}, + 'Delayed': {'percent': 10, 'step': 'Delayed'}, + 'Success': {'percent': 100, 'step': 'Complete'}, + 'Cancelled': {'percent': 0, 'step': 'Cancelled'}, + 'TimedOut': {'percent': 0, 'step': 'Timed out'}, + 'Failed': {'percent': 0, 'step': 'Failed'}, + } + progress = status_map.get(status, {'percent': 0, 'step': status}) + # Refine InProgress estimate from stdout + if status == 'InProgress': + stdout = invocation.get('StandardOutputContent', '') + if 'Creating Archive' in stdout: + progress = {'percent': 85, 'step': 'Creating archive'} + elif 'Uploading to S3' in stdout: + progress = {'percent': 95, 'step': 'Uploading to S3'} + elif 'Collecting Network' in stdout: + progress = {'percent': 70, 'step': 'Collecting network diagnostics'} + elif 'Collecting ECS Agent' in stdout: + progress = {'percent': 40, 'step': 'Collecting ECS agent info'} + elif 'Collecting Docker' in stdout: + progress = {'percent': 30, 'step': 'Collecting Docker info'} + elif 'Collecting System' in stdout: + progress = {'percent': 20, 'step': 'Collecting system info'} + return progress + + +# ============================================================================ +# IDEMPOTENCY (S3-based) +# ============================================================================ + +def store_execution_region(execution_id: str, region: str): + try: + s3_client.put_object(Bucket=LOGS_BUCKET, Key=f'execution-regions/{execution_id}', Body=region.encode(), ServerSideEncryption='AES256') + except Exception: + pass + + +def get_execution_region(execution_id: str) -> Optional[str]: + data = safe_s3_read_raw(LOGS_BUCKET, f'execution-regions/{execution_id}') + return data.decode('utf-8').strip() if data else None + + +def find_execution_by_idempotency_token(instance_id: str, token: str) -> Optional[Dict]: + """Find existing execution by idempotency token, scoped to instance.""" + key = f'idempotency/{instance_id}/{token}.json' + data = safe_s3_read_raw(LOGS_BUCKET, key) + if data: + try: + return json.loads(data) + except Exception: + pass + return None + + +def store_idempotency_mapping(instance_id: str, token: str, execution_id: str): + """Store idempotency mapping scoped to instance.""" + mapping = { + 'executionId': execution_id, + 'instanceId': instance_id, + 'token': token, + 'status': 'InProgress', + 'createdAt': datetime.utcnow().isoformat(), + } + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, + Key=f'idempotency/{instance_id}/{token}.json', + Body=json.dumps(mapping, default=str).encode(), + ContentType='application/json', + ) + except Exception as e: + print(f"Warning: Failed to store idempotency mapping: {str(e)}") + + +# ============================================================================ +# BASELINE HELPERS +# ============================================================================ + +def load_baselines(instance_id: str) -> Dict: + data = safe_s3_read_raw(LOGS_BUCKET, f'baselines/{instance_id}/baseline.json') + if data: + try: + return json.loads(data) + except Exception: + pass + return {} + + +def update_baselines(instance_id: str, findings: List[Dict]): + baseline = load_baselines(instance_id) + for f in findings: + sig = f'{f.get("description", "")}__{f.get("file", "")}' + key = hashlib.md5(sig.encode()).hexdigest() + if key not in baseline: + baseline[key] = {'firstSeen': datetime.now(timezone.utc).isoformat(), 'count': 0, 'description': f.get('description', '')} + baseline[key]['count'] = baseline[key].get('count', 0) + 1 + baseline[key]['lastSeen'] = datetime.now(timezone.utc).isoformat() + try: + s3_client.put_object(Bucket=LOGS_BUCKET, Key=f'baselines/{instance_id}/baseline.json', Body=json.dumps(baseline, default=str).encode(), ServerSideEncryption='AES256') + except Exception: + pass + + +def annotate_findings_with_baselines(findings: List[Dict], baselines: Dict) -> List[Dict]: + for f in findings: + sig = f'{f.get("description", "")}__{f.get("file", "")}' + key = hashlib.md5(sig.encode()).hexdigest() + if key in baselines: + f['isBaseline'] = True + f['baselineFirstSeen'] = baselines[key].get('firstSeen') + f['baselineCount'] = baselines[key].get('count', 0) + else: + f['isBaseline'] = False + return findings + + +# ============================================================================ +# SCAN HELPERS +# ============================================================================ + +def find_findings_index(prefix: str, s3c=None) -> Optional[str]: + """ + Find the findings index file for a log collection. + Returns the S3 key string of the findings_index.json in the LATEST bundle, or None. + """ + parts = prefix.split('_', 1) + instance_id = parts[1] if len(parts) > 1 else prefix + scheme = parts[0] if len(parts) > 1 else 'ecs' + + bundle_info = find_latest_bundle_files(instance_id, prefix_scheme=scheme) + if not bundle_info['success']: + return None + + index_files = [f for f in bundle_info['files'] if FINDINGS_INDEX_FILE in f] + return index_files[0] if index_files else None + + +def scan_file_for_errors(content: str, filename: str) -> List[Dict]: + findings = [] + lines = content.split('\n') + for line_num, line in enumerate(lines, 1): + if not line.strip(): + continue + # False positive suppression + if any(fp.search(line) for fp in FALSE_POSITIVE_PATTERNS): + continue + for sev_key, compiled_patterns in COMPILED_ERROR_PATTERNS.items(): + for regex, description in compiled_patterns: + if regex.search(line): + findings.append({ + 'severity': sev_key, + 'description': description, + 'file': filename, + 'lineNumber': line_num, + 'line': line[:500], + 'pattern': regex.pattern, + }) + break # One match per line per severity + return findings + + +def scan_and_index_errors(prefix: str, s3c=None) -> Dict: + """Scan all files in a bundle and build findings index.""" + c = s3c or s3_client + files = safe_s3_list_raw(LOGS_BUCKET, prefix, s3c=c) + all_findings = [] + files_scanned = 0 + for obj in files: + key = obj['Key'] + size = obj.get('Size', 0) + if any(key.endswith(ext) for ext in ['.tar.gz', '.zip', '.gz', '.tar', '.bin', '.so', '.png', '.jpg']): + continue + if key.endswith(FINDINGS_INDEX_FILE) or key.endswith('manifest.json'): + continue + if size > MAX_CHUNK_SIZE or size == 0: + continue + data = safe_s3_read_raw(LOGS_BUCKET, key, s3c=c) + if not data: + continue + try: + content = data.decode('utf-8', errors='ignore') + except Exception: + continue + files_scanned += 1 + filename = key[len(prefix):] if key.startswith(prefix) else key.split('/')[-1] + file_findings = scan_file_for_errors(content, filename) + all_findings.extend(file_findings) + # Assign finding IDs and sort by severity + all_findings.sort(key=lambda f: SEVERITY_ORDER.get(f.get('severity', 'info'), 4)) + for i, f in enumerate(all_findings): + f['finding_id'] = assign_finding_id(i + 1) + index = { + 'generatedAt': datetime.now(timezone.utc).isoformat(), + 'filesScanned': files_scanned, + 'totalFindings': len(all_findings), + 'findings': all_findings, + 'summary': {}, + } + for sev in ['critical', 'warning', 'info']: + index['summary'][sev] = len([f for f in all_findings if f['severity'] == sev]) + # Store index + try: + c.put_object(Bucket=LOGS_BUCKET, Key=f'{prefix}{FINDINGS_INDEX_FILE}', Body=json.dumps(index, default=str).encode(), ServerSideEncryption='AES256') + except Exception: + pass + return index + + +# ============================================================================ +# READ / SEARCH HELPERS +# ============================================================================ + +def read_by_lines(bucket: str, key: str, start_line: int = 1, max_lines: int = DEFAULT_LINE_COUNT, s3c=None) -> Dict: + head = safe_s3_head_raw(bucket, key, s3c=s3c) + if not head: + return {'error': f'File not found: {key}'} + total_size = head['ContentLength'] + data = safe_s3_read_raw(bucket, key, s3c=s3c) + if not data: + return {'error': f'Cannot read: {key}'} + try: + content = data.decode('utf-8', errors='ignore') + except Exception: + content = data.decode('latin-1', errors='ignore') + lines = content.split('\n') + total_lines = len(lines) + end_line = min(start_line + max_lines - 1, total_lines) + selected = lines[start_line - 1:end_line] + return { + 'lines': selected, 'startLine': start_line, 'endLine': end_line, + 'totalLines': total_lines, 'totalSize': total_size, 'hasMore': end_line < total_lines, + } + + +def search_file_for_pattern(bucket: str, key: str, pattern: str, max_matches: int = 50, s3c=None) -> List[Dict]: + """Search a file for regex pattern with chunked reading for large files.""" + head = safe_s3_head_raw(bucket, key, s3c=s3c) + if not head: + return [] + file_size = head['ContentLength'] + if file_size > 10 * 1024 * 1024: # Skip files > 10MB + return [] + data = safe_s3_read_raw(bucket, key, s3c=s3c) + if not data: + return [] + try: + content = data.decode('utf-8', errors='ignore') + except Exception: + return [] + try: + regex = re.compile(pattern, re.IGNORECASE) + except re.error: + return [] + matches = [] + filename = key.split('/')[-1] + for i, line in enumerate(content.split('\n'), 1): + if regex.search(line): + matches.append({ + 'file': filename, 'fullKey': key, 'lineNumber': i, + 'line': line[:500], 'pattern': pattern, + }) + if len(matches) >= max_matches: + break + return matches + + +def get_line_context(content: str, line_num: int, context: int = 3) -> List[str]: + lines = content.split('\n') + start = max(0, line_num - context - 1) + end = min(len(lines), line_num + context) + return lines[start:end] + + +def extract_timestamp(line: str) -> Optional[str]: + patterns = [ + r'(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2})', + r'(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})', + r'(\[\d+\.\d+\])', + ] + for p in patterns: + m = re.search(p, line) + if m: + return m.group(1) + return None + + +def categorize_log_source(filename: str) -> str: + """Categorize a log file into ECS-specific component.""" + fl = filename.lower() + for category, patterns in ECS_LOG_TYPE_PATTERNS.items(): + if any(p in fl for p in patterns): + return category + if 'kernel' in fl or 'dmesg' in fl: + return 'kernel' + return 'system' + + +def find_correlations(findings: List[Dict]) -> List[Dict]: + """Find ECS-specific cross-component correlations.""" + correlations = [] + # Group findings by component + by_component = {} + for f in findings: + comp = categorize_log_source(f.get('file', '')) + by_component.setdefault(comp, []).append(f) + # Kernel <-> ECS Agent correlation + kernel_findings = by_component.get('kernel', []) + by_component.get('dmesg', []) + ecs_findings = by_component.get('ecs-agent', []) + if kernel_findings and ecs_findings: + correlations.append({ + 'type': 'kernel-ecs-agent', + 'description': 'Kernel issues detected alongside ECS agent problems - kernel instability may cause agent disconnection', + 'components': ['kernel', 'ecs-agent'], + 'findingIds': [f.get('finding_id') for f in (kernel_findings[:3] + ecs_findings[:3]) if f.get('finding_id')], + 'confidence': 'high', + }) + # Network <-> Docker correlation + net_findings = by_component.get('networking', []) + docker_findings = by_component.get('docker', []) + if net_findings and docker_findings: + correlations.append({ + 'type': 'network-docker', + 'description': 'Network issues detected alongside Docker problems - network connectivity may affect container operations', + 'components': ['networking', 'docker'], + 'findingIds': [f.get('finding_id') for f in (net_findings[:3] + docker_findings[:3]) if f.get('finding_id')], + 'confidence': 'medium', + }) + # OOM <-> Task failures + oom_findings = [f for f in findings if any(k in f.get('description', '').lower() for k in ['oom', 'memory', 'exit code 137'])] + task_failures = [f for f in findings if any(k in f.get('description', '').lower() for k in ['task', 'container', 'stopped'])] + if oom_findings and task_failures: + correlations.append({ + 'type': 'oom-task-failure', + 'description': 'OOM kills detected alongside task failures - memory exhaustion likely causing task stops', + 'components': ['cgroups', 'ecs-agent'], + 'findingIds': [f.get('finding_id') for f in (oom_findings[:3] + task_failures[:3]) if f.get('finding_id')], + 'confidence': 'high', + }) + return correlations + + +def generate_recommendations(findings: List[Dict]) -> List[Dict]: + """Generate ECS-specific recommendations based on findings.""" + recs = [] + descs = ' '.join(f.get('description', '') for f in findings).lower() + if 'oom' in descs or 'memory' in descs or 'exit code 137' in descs: + recs.append({'priority': 'high', 'category': 'Resource', 'action': 'Increase task memory limits or investigate memory leaks', 'docs': 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/out-of-memory.html'}) + if 'pull' in descs or 'ecr' in descs or 'image' in descs: + recs.append({'priority': 'high', 'category': 'Image', 'action': 'Verify ECR permissions, image existence, and VPC endpoint connectivity', 'docs': 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_cannot_pull_image.html'}) + if 'agent' in descs and ('disconnect' in descs or 'not connected' in descs): + recs.append({'priority': 'high', 'category': 'Agent', 'action': 'Check ECS agent logs, verify IAM instance profile, and network connectivity to ECS endpoints'}) + if 'secret' in descs or 'parameter' in descs: + recs.append({'priority': 'high', 'category': 'Secrets', 'action': 'Verify task execution role has secretsmanager:GetSecretValue and ssm:GetParameters permissions', 'docs': 'https://repost.aws/knowledge-center/ecs-unable-to-pull-secrets'}) + if 'network' in descs or 'eni' in descs or 'subnet' in descs: + recs.append({'priority': 'high', 'category': 'Network', 'action': 'Check subnet IP availability, security group rules, and VPC endpoint configuration'}) + if 'circuit breaker' in descs or 'deployment' in descs: + recs.append({'priority': 'medium', 'category': 'Deployment', 'action': 'Review deployment configuration, health check settings, and task definition', 'docs': 'https://repost.aws/knowledge-center/ecs-troubleshoot-deployment-failures'}) + if 'health check' in descs or 'unhealthy' in descs: + recs.append({'priority': 'medium', 'category': 'Health', 'action': 'Review health check configuration, grace period, and container startup time'}) + if 'disk' in descs or 'space' in descs or 'inode' in descs: + recs.append({'priority': 'medium', 'category': 'Storage', 'action': 'Clean up unused images/containers, increase EBS volume size'}) + return recs + + +# ============================================================================ +# TRIAGE +# ============================================================================ + +def perform_ecs_triage(findings: List[Dict]) -> Dict: + """Perform ECS-specific triage using ECS_TRIAGE_CATEGORIES.""" + matched_categories = {} + for cat_id, cat_info in ECS_TRIAGE_CATEGORIES.items(): + cat_findings = [] + for f in findings: + line = f.get('line', '') + ' ' + f.get('description', '') + for pattern, priority in cat_info['patterns']: + if re.search(pattern, line, re.IGNORECASE): + cat_findings.append({**f, 'triagePriority': priority}) + break + if cat_findings: + matched_categories[cat_id] = { + 'name': cat_info['name'], + 'description': cat_info['description'], + 'logSources': cat_info['log_sources'], + 'findingCount': len(cat_findings), + 'findings': cat_findings[:10], + 'docs': cat_info.get('docs'), + 'runbook': cat_info.get('runbook'), + } + # Detect ECS task states + task_states = {} + for f in findings: + line = f.get('line', '') + for state in ['PENDING', 'PROVISIONING', 'RUNNING', 'STOPPED', 'DEPROVISIONING']: + if state in line: + task_states[state] = task_states.get(state, 0) + 1 + # Detect instance conditions + instance_conditions = {} + for f in findings: + line = f.get('line', '') + for cond in ['ACTIVE', 'DRAINING', 'DISCONNECTED', 'AGENT_DISCONNECTED']: + if cond in line: + instance_conditions[cond] = instance_conditions.get(cond, 0) + 1 + return { + 'categories': matched_categories, + 'taskStates': task_states, + 'instanceConditions': instance_conditions, + 'topCategory': max(matched_categories.keys(), key=lambda k: matched_categories[k]['findingCount']) if matched_categories else None, + } + + +# ============================================================================ +# TEMPORAL CORRELATION +# ============================================================================ + +def _build_temporal_clusters(findings: List[Dict], window_seconds: int = 60) -> List[Dict]: + """Group findings into temporal clusters.""" + timestamped = [] + for f in findings: + ts = extract_timestamp(f.get('line', '')) + if ts: + timestamped.append({**f, '_ts': ts}) + if not timestamped: + return [] + clusters = [] + current_cluster = [timestamped[0]] + for f in timestamped[1:]: + current_cluster.append(f) + if len(current_cluster) >= 20: + clusters.append({'events': current_cluster, 'count': len(current_cluster)}) + current_cluster = [] + if current_cluster: + clusters.append({'events': current_cluster, 'count': len(current_cluster)}) + return clusters + + +def _build_root_cause_chain(findings: List[Dict]) -> List[Dict]: + """Build ECS-specific causal chains.""" + chains = [] + # ECS causal patterns: kernel issue -> agent disconnect -> task failures + kernel_issues = [f for f in findings if categorize_log_source(f.get('file', '')) in ('kernel', 'dmesg') and f.get('severity') == 'critical'] + agent_issues = [f for f in findings if 'agent' in f.get('description', '').lower() or 'disconnect' in f.get('description', '').lower()] + task_issues = [f for f in findings if any(k in f.get('description', '').lower() for k in ['task', 'container', 'stopped', 'failed'])] + if kernel_issues and agent_issues: + chains.append({ + 'type': 'kernel-cascade', + 'description': 'Kernel instability → ECS Agent disconnect → Task failures', + 'rootCause': kernel_issues[0].get('finding_id'), + 'effects': [f.get('finding_id') for f in agent_issues[:3] + task_issues[:3] if f.get('finding_id')], + 'confidence': 'high', + }) + # OOM cascade: memory pressure -> OOM kill -> container exit -> task stop + oom_issues = [f for f in findings if any(k in f.get('description', '').lower() for k in ['oom', 'memory cgroup', 'invoked oom-killer'])] + exit_137 = [f for f in findings if 'exit code 137' in f.get('description', '').lower()] + if oom_issues: + chains.append({ + 'type': 'oom-cascade', + 'description': 'Memory pressure → OOM kill → Container exit (137) → Task stop', + 'rootCause': oom_issues[0].get('finding_id'), + 'effects': [f.get('finding_id') for f in exit_137[:3] + task_issues[:3] if f.get('finding_id')], + 'confidence': 'high', + }) + # Network cascade: ENI/subnet issue -> connection failures -> health check failures + net_root = [f for f in findings if any(k in f.get('description', '').lower() for k in ['eni', 'subnet', 'network unreachable'])] + conn_issues = [f for f in findings if any(k in f.get('description', '').lower() for k in ['connection refused', 'timeout', 'dns'])] + health_issues = [f for f in findings if 'health' in f.get('description', '').lower()] + if net_root and (conn_issues or health_issues): + chains.append({ + 'type': 'network-cascade', + 'description': 'Network/ENI issue → Connection failures → Health check failures', + 'rootCause': net_root[0].get('finding_id'), + 'effects': [f.get('finding_id') for f in conn_issues[:3] + health_issues[:3] if f.get('finding_id')], + 'confidence': 'medium', + }) + return chains + +# ============================================================================= +# LAMBDA HANDLER + TOOL ROUTING +# ============================================================================= + +def list_sops(arguments: Dict) -> Dict: + """List all SOPs in the SOP S3 bucket.""" + sop_bucket = os.environ.get('SOP_BUCKET_NAME', '') + if not sop_bucket: + return error_response(400, 'SOP_BUCKET_NAME not configured') + try: + s3 = boto3.client('s3') + response = s3.list_objects_v2(Bucket=sop_bucket) + if 'Contents' not in response: + return success_response({'sops': [], 'count': 0, 'bucket': sop_bucket}) + sops = [{'name': obj['Key'], 'size': obj['Size'], 'lastModified': obj['LastModified'].isoformat()} for obj in response['Contents']] + return success_response({'sops': sops, 'count': len(sops), 'bucket': sop_bucket}) + except Exception as e: + return error_response(500, f'Failed to list SOPs: {str(e)}') + + +def get_sop(arguments: Dict) -> Dict: + """Get a specific SOP by name from the SOP S3 bucket.""" + sop_bucket = os.environ.get('SOP_BUCKET_NAME', '') + if not sop_bucket: + return error_response(400, 'SOP_BUCKET_NAME not configured') + sop_name = arguments.get('sopName') + if not sop_name: + return error_response(400, 'sopName is required') + try: + s3 = boto3.client('s3') + response = s3.get_object(Bucket=sop_bucket, Key=sop_name) + content = response['Body'].read().decode('utf-8') + return success_response({ + 'sop': {'name': sop_name, 'content': content, 'size': response['ContentLength'], + 'lastModified': response['LastModified'].isoformat(), + 'contentType': response.get('ContentType', 'text/plain')} + }) + except ClientError as e: + if e.response['Error']['Code'] == 'NoSuchKey': + return error_response(404, f'SOP "{sop_name}" not found. Use list_sops to see available SOPs.') + return error_response(500, f'Failed to get SOP: {str(e)}') + except Exception as e: + return error_response(500, f'Failed to get SOP: {str(e)}') + + +def lambda_handler(event, context): + """Main Lambda handler - routes to appropriate tool function.""" + print(f"Received event: {json.dumps(event)}") + + delimiter = "___" + original_tool_name = context.client_context.custom.get('bedrockAgentCoreToolName', '') + + if delimiter in original_tool_name: + tool_name = original_tool_name[original_tool_name.index(delimiter) + len(delimiter):] + else: + tool_name = original_tool_name + + print(f"Executing tool: {tool_name}") + + tools = { + # Tier 1: Core Operations + 'collect': start_log_collection, + 'status': get_collection_status, + 'validate': validate_bundle_completeness, + 'errors': get_error_summary, + 'read': read_log_chunk, + # Tier 2: Advanced Analysis + 'search': search_logs_deep, + 'correlate': correlate_events, + 'artifact': get_artifact_reference, + 'summarize': generate_incident_summary, + 'history': list_collection_history, + # Tier 3: Cluster-Level Intelligence + 'cluster_health': cluster_health_check, + 'compare_instances': compare_instances, + 'batch_collect': batch_collect, + 'batch_status': batch_status, + 'network_diagnostics': network_diagnostics, + 'tcpdump_capture': tcpdump_capture, + 'tcpdump_analyze': tcpdump_analyze, + 'list_sops': list_sops, + 'get_sop': get_sop, + } + + if tool_name not in tools: + return error_response(400, f'Unknown tool: {tool_name}', { + 'available_tools': list(tools.keys()) + }) + + try: + return tools[tool_name](event) + except Exception as e: + print(f"Error executing {tool_name}: {str(e)}") + import traceback + traceback.print_exc() + return error_response(500, f'Internal error: {str(e)}') + + +# ============================================================================= +# TIER 1: CORE OPERATIONS +# ============================================================================= + +def start_log_collection(arguments: Dict) -> Dict: + """ + Start ECS log collection via AWSSupport-CollectECSInstanceLogs SSM Automation. + + Inputs: + instanceId: EC2 instance ID (required) + idempotencyToken: Optional token to prevent duplicate executions + region: AWS region where the instance runs (optional, auto-detected) + + Returns: + executionId, estimatedCompletionTime, status, region + """ + instance_id = arguments.get('instanceId') + idempotency_token = arguments.get('idempotencyToken') + + if not instance_id: + return error_response(400, 'instanceId is required') + + if not re.match(r'^i-[0-9a-f]{8,17}$', instance_id): + return error_response(400, f'Invalid instanceId format: {instance_id}. Expected: i-xxxxxxxxxxxxxxxxx') + + # Resolve and validate region + target_region, region_error = resolve_and_validate_region(arguments, instance_id) + if region_error: + return region_error + + # Validate instance belongs to an ECS cluster + instance_error = validate_ecs_instance(instance_id, target_region) + if instance_error: + return instance_error + + try: + regional_ssm = get_regional_client('ssm', target_region) + except Exception as e: + return error_response(500, f'Failed to create SSM client for region {target_region}: {str(e)}') + + print(f"Starting ECS log collection for {instance_id} in region {target_region}") + + # Verify instance state + try: + regional_ec2 = get_regional_client('ec2', target_region) + desc_resp = regional_ec2.describe_instances(InstanceIds=[instance_id]) + reservations = desc_resp.get('Reservations', []) + if reservations and reservations[0].get('Instances'): + state = reservations[0]['Instances'][0].get('State', {}).get('Name', 'unknown') + if state in ('terminated', 'shutting-down'): + return error_response(400, f'Instance {instance_id} is {state}. Cannot collect logs from terminated instances.') + if state == 'stopped': + return error_response(400, f'Instance {instance_id} is stopped. Start the instance first, then retry.') + except Exception as e: + print(f"Warning: Could not verify instance state: {str(e)}") + + # Idempotency check + if idempotency_token: + existing = find_execution_by_idempotency_token(instance_id, idempotency_token) + if existing: + return success_response({ + 'message': 'Returning existing execution (idempotent)', + 'executionId': existing['executionId'], + 'status': existing['status'], + 'instanceId': instance_id, + 'region': target_region, + 'idempotent': True, + }) + + try: + params = { + 'ECSInstanceId': [instance_id], + 'LogDestination': [LOGS_BUCKET], + 'AutomationAssumeRole': [SSM_AUTOMATION_ROLE_ARN], + } + + response = regional_ssm.start_automation_execution( + DocumentName='AWSSupport-CollectECSInstanceLogs', + Parameters=params, + ) + + execution_id = response['AutomationExecutionId'] + + if idempotency_token: + store_idempotency_mapping(instance_id, idempotency_token, execution_id) + + region_stored = store_execution_region(execution_id, target_region) + + response_data = { + 'message': 'ECS log collection started', + 'executionId': execution_id, + 'instanceId': instance_id, + 'region': target_region, + 's3Bucket': LOGS_BUCKET, + 'estimatedCompletionTime': '3-5 minutes', + 'suggestedPollIntervalSeconds': 15, + 'nextStep': f'Poll status with status(executionId="{execution_id}") every 15 seconds', + 'task': { + 'taskId': execution_id, + 'state': 'running', + 'message': 'Log collection started via SSM Automation', + 'progress': 0, + }, + } + + if not region_stored and target_region != DEFAULT_REGION: + response_data['warning'] = ( + f'Region mapping could not be persisted. Pass region="{target_region}" ' + f'explicitly in subsequent status/validate calls.' + ) + + return success_response(response_data) + + except regional_ssm.exceptions.AutomationDefinitionNotFoundException: + return error_response(404, 'AWSSupport-CollectECSInstanceLogs document not found', { + 'suggestion': f'This SSM document may not be available in region {target_region}. ' + f'Try us-east-1 or us-west-2.', + 'region': target_region, + }) + except Exception as e: + return error_response(500, f'Failed to start log collection in {target_region}: {str(e)}') + + +def get_collection_status(arguments: Dict) -> Dict: + """ + Get detailed status of ECS log collection with progress tracking. + + Inputs: + executionId: SSM Automation execution ID (required) + includeStepDetails: Include individual step status (default: true) + + Returns: + status, progress, stepDetails, failureReason (if failed) + """ + execution_id = arguments.get('executionId') + include_steps = arguments.get('includeStepDetails', True) + + if not execution_id: + return error_response(400, 'executionId is required') + + target_region = get_execution_region(execution_id) or arguments.get('region', DEFAULT_REGION) + try: + regional_ssm = get_regional_client('ssm', target_region) + except Exception as e: + return error_response(500, f'Failed to create SSM client for region {target_region}: {str(e)}') + + try: + response = regional_ssm.get_automation_execution(AutomationExecutionId=execution_id) + execution = response['AutomationExecution'] + status = execution['AutomationExecutionStatus'] + + result = { + 'executionId': execution_id, + 'status': status, + 'documentName': execution.get('DocumentName', ''), + 'startTime': execution.get('ExecutionStartTime'), + 'endTime': execution.get('ExecutionEndTime'), + } + + if status == 'Success': + result['progress'] = 100 + elif status == 'Failed': + result['progress'] = 0 + result['failureReason'] = parse_failure_reason(execution) + elif status == 'InProgress': + result['progress'] = estimate_progress(execution) + else: + result['progress'] = 0 + + if include_steps and 'StepExecutions' in execution: + result['stepDetails'] = [ + { + 'stepName': step.get('StepName'), + 'status': step.get('StepStatus'), + 'startTime': step.get('ExecutionStartTime'), + 'endTime': step.get('ExecutionEndTime'), + } + for step in execution.get('StepExecutions', []) + ] + + if 'Outputs' in execution: + result['outputs'] = execution['Outputs'] + + if status == 'Success': + result['nextStep'] = f'Validate bundle with validate(executionId="{execution_id}")' + elif status == 'InProgress': + result['suggestedPollIntervalSeconds'] = 15 + result['nextStep'] = 'Wait 15 seconds then poll again until status is Success or Failed' + elif status == 'Failed': + result['nextStep'] = 'Review failureReason and retry if appropriate' + + SSM_TO_TASK_STATE = { + 'Pending': 'running', 'InProgress': 'running', 'Waiting': 'running', + 'Success': 'completed', 'TimedOut': 'failed', 'Cancelling': 'cancelling', + 'Cancelled': 'cancelled', 'Failed': 'failed', + } + result['task'] = { + 'taskId': execution_id, + 'state': SSM_TO_TASK_STATE.get(status, 'running'), + 'message': result.get('failureReason', f'SSM status: {status}'), + 'progress': result.get('progress', 0), + } + + return success_response({'automation': result}) + + except regional_ssm.exceptions.AutomationExecutionNotFoundException: + return error_response(404, f'Execution {execution_id} not found') + except Exception as e: + return error_response(500, f'Failed to get status: {str(e)}') + + +def validate_bundle_completeness(arguments: Dict) -> Dict: + """ + Verify all expected files were extracted from ECS log bundle. + + Inputs: + executionId: SSM execution ID OR + instanceId: Instance ID to locate bundle + + Returns: + complete, fileCount, totalSize, missingPatterns, manifest + """ + execution_id = arguments.get('executionId') + instance_id = arguments.get('instanceId') + + if not execution_id and not instance_id: + return error_response(400, 'Either executionId or instanceId is required') + + try: + if instance_id: + prefix = f'ecs_{instance_id}' + else: + try: + target_region = get_execution_region(execution_id) or arguments.get('region', DEFAULT_REGION) + regional_ssm = get_regional_client('ssm', target_region) + except Exception as e: + return error_response(500, f'Failed to create SSM client for region: {str(e)}') + try: + exec_response = regional_ssm.get_automation_execution(AutomationExecutionId=execution_id) + params = exec_response['AutomationExecution'].get('Parameters', {}) + instance_id = params.get('ECSInstanceId', [''])[0] + prefix = f'ecs_{instance_id}' + except regional_ssm.exceptions.AutomationExecutionNotFoundException: + return error_response(404, f'Execution {execution_id} not found') + except Exception as e: + return error_response(500, f'Failed to get execution details: {str(e)}') + + # Use shared latest-bundle discovery + bundle_info = find_latest_bundle_files(instance_id) + + if not bundle_info['success']: + return success_response({ + 'complete': False, 'fileCount': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'missingPatterns': ['all'], 'foundPatterns': [], 'hasFindingsIndex': False, + 'instanceId': instance_id, 'manifest': [], + 'warning': bundle_info.get('error', 'Failed to list files'), + 'nextStep': 'Check if log collection completed successfully', + }) + + size_map = {obj['key']: obj for obj in bundle_info['all_objects']} + all_files = [size_map[k] for k in bundle_info['files'] if k in size_map] + + # Check for manifest.json in latest bundle + manifest_data = None + manifest_files = [obj for obj in bundle_info['all_objects'] + if obj['key'].endswith('manifest.json') and obj['key'].startswith(bundle_info['bundle_prefix'])] + if manifest_files: + manifest_files.sort(key=lambda x: x.get('last_modified', ''), reverse=True) + manifest_read = safe_s3_read(manifest_files[0]['key']) + if manifest_read['success']: + try: + manifest_data = json.loads(manifest_read['content']) + except json.JSONDecodeError: + manifest_data = None + + if not all_files: + return success_response({ + 'complete': False, 'fileCount': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'missingPatterns': ['all - no extracted logs found'], 'foundPatterns': [], + 'hasFindingsIndex': False, 'instanceId': instance_id, 'manifest': [], + 'info': 'No extracted log files found. Log collection may still be in progress.', + 'nextStep': 'Check log collection status with status', + }) + + total_size = sum(f['size'] for f in all_files) + + # ECS-specific expected patterns (aligned with amazon-ecs-logs-collector.sh output) + expected_patterns = [ + 'ecs', 'docker', 'messages', 'dmesg', 'networking', 'containers', + 'metadata', 'iptables', 'mounts', 'services', 'os-release', 'uname', + 'pkglist', 'ps.txt', 'top.txt', 'open-file', 'cgroup', + ] + found_patterns = set() + for f in all_files: + key_lower = f['key'].lower() + for pattern in expected_patterns: + if pattern in key_lower: + found_patterns.add(pattern) + + missing_patterns = list(set(expected_patterns) - found_patterns) + has_findings_index = any(FINDINGS_INDEX_FILE in f['key'] for f in all_files) + is_complete = len(all_files) >= 5 and len(found_patterns) >= 3 + + result = { + 'complete': is_complete, + 'fileCount': len(all_files), + 'totalSize': total_size, + 'totalSizeHuman': format_bytes(total_size), + 'missingPatterns': missing_patterns, + 'foundPatterns': list(found_patterns), + 'hasFindingsIndex': has_findings_index, + 'instanceId': instance_id, + } + + if manifest_data and manifest_data.get('version', 1) >= 2: + result['manifestVersion'] = manifest_data.get('version') + result['archiveSize'] = manifest_data.get('archiveSize', 0) + result['archiveSizeHuman'] = format_bytes(manifest_data.get('archiveSize', 0)) + manifest_file_count = manifest_data.get('totalFiles', 0) + if manifest_file_count > 0 and len(all_files) < manifest_file_count: + result['warning'] = ( + f'Manifest reports {manifest_file_count} files but only {len(all_files)} found in S3.' + ) + result['complete'] = False + + if missing_patterns: + result['info'] = f'Some log types not found: {", ".join(missing_patterns)}. May be normal depending on instance config.' + + result['manifest'] = [ + { + 'key': f['key'].split('/extracted/')[-1] if '/extracted/' in f['key'] else f['key'], + 'fullKey': f['key'], + 'size': f['size'], + 'sizeHuman': format_bytes(f['size']), + } + for f in sorted(all_files, key=lambda x: x['size'], reverse=True)[:50] + ] + + if is_complete: + result['nextStep'] = f'Get error summary with errors(instanceId="{instance_id}")' + else: + result['nextStep'] = 'Bundle may be incomplete. Check SSM Automation status or proceed with available logs.' + + return success_response(result) + + except Exception as e: + return success_response({ + 'complete': False, 'fileCount': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'missingPatterns': ['unknown'], 'foundPatterns': [], 'hasFindingsIndex': False, + 'instanceId': instance_id or 'unknown', 'manifest': [], + 'error': f'Unexpected error during validation: {str(e)}', + 'nextStep': 'Retry or check AWS console for log collection status', + }) + + +def get_error_summary(arguments: Dict) -> Dict: + """ + Get pre-indexed error findings with pagination and baseline support. + + Inputs: + instanceId: EC2 instance ID (required) + severity: Filter (critical|high|medium|low|info|all) + response_format: 'concise' (default) or 'detailed' + pageSize: Findings per page (default: 50, max: 200) + pageToken: Opaque token for next page + + Returns: + findings[], summary counts, coverage_report + """ + instance_id = arguments.get('instanceId') + severity_filter = arguments.get('severity', 'all') + response_format = arguments.get('response_format', 'concise') + page_size = min(arguments.get('pageSize', 50), 200) + page_token = arguments.get('pageToken') + cluster_context = arguments.get('clusterContext') + + if not instance_id: + return error_response(400, 'instanceId is required') + + page_offset = 0 + if page_token: + try: + import base64 + page_offset = int(base64.b64decode(page_token).decode('utf-8')) + except Exception: + page_offset = 0 + + try: + prefix = f'ecs_{instance_id}' + index_key = find_findings_index(prefix) + + if index_key: + read_result = safe_s3_read(index_key) + if read_result['success']: + try: + index_data = json.loads(read_result['content']) + findings = index_data.get('findings', []) + + for idx, f in enumerate(findings): + if 'finding_id' not in f: + f['finding_id'] = assign_finding_id(idx + 1) + + if cluster_context: + findings = annotate_findings_with_baselines(findings, cluster_context) + + allowed_severities = normalize_severity_filter(severity_filter) + if severity_filter != 'all': + findings = [f for f in findings if f.get('severity') in allowed_severities] + + total_findings = len(findings) + page_findings = findings[page_offset:page_offset + page_size] + has_more = (page_offset + page_size) < total_findings + + next_token = None + if has_more: + import base64 + next_token = base64.b64encode(str(page_offset + page_size).encode('utf-8')).decode('utf-8') + + summary = index_data.get('summary', {}) + + if response_format == 'concise': + page_findings = [ + { + 'finding_id': f.get('finding_id'), + 'severity': f.get('severity'), + 'pattern': f.get('pattern'), + 'file': f.get('file'), + 'count': f.get('count'), + **( + {'is_baseline': f.get('is_baseline', False), 'baseline_note': f.get('baseline_note')} + if f.get('is_baseline') else {} + ), + } + for f in page_findings + ] + + coverage_report = { + 'files_scanned': index_data.get('filesScanned', 0), + 'files_skipped': index_data.get('filesSkipped', 0), + 'scan_complete': True, + 'index_version': index_data.get('index_version', 'v1'), + } + + if cluster_context: + update_baselines(cluster_context, findings) + + return success_response({ + 'instanceId': instance_id, + 'indexedAt': index_data.get('indexedAt'), + 'findings': page_findings, + 'totalFindings': total_findings, + 'pageSize': page_size, + 'pageOffset': page_offset, + 'hasMore': has_more, + 'nextPageToken': next_token, + 'summary': summary, + 'cached': True, + 'coverage_report': coverage_report, + 'interpretationGuide': { + 'CannotPullContainerError': 'ECR auth or image not found. Check IAM role and image URI.', + 'OOMKilled': 'Container exceeded memory limit. Check task definition memory settings.', + 'STOPPED (Essential container exited)': 'Essential container crashed. Check container logs.', + 'AGENT_DISCONNECTED': 'ECS agent lost connection. Check instance networking and agent logs.', + 'ResourceNotFoundException': 'ECS resource not found. Service/task may have been deleted.', + }, + 'nextStep': 'Use search for detailed investigation, or summarize with finding_ids', + 'recommendedSOPs': match_sops_for_issues([], findings=page_findings), + }) + except json.JSONDecodeError: + print("Warning: Findings index corrupted, will scan on-demand") + + # Slow path: scan and index on-demand + result = scan_and_index_errors(instance_id, severity_filter) + return result + + except Exception as e: + return success_response({ + 'instanceId': instance_id, + 'findings': [], + 'totalFindings': 0, + 'summary': {'critical': 0, 'high': 0, 'medium': 0, 'low': 0, 'info': 0}, + 'cached': False, + 'warning': f'Could not retrieve error summary: {str(e)}', + 'nextStep': 'Check if logs exist with validate', + }) + + +def read_log_chunk(arguments: Dict) -> Dict: + """ + Byte-range streaming for large log files. NO TRUNCATION. Line-aligned. + + Inputs: + logKey: S3 key of log file (required) + startByte: Starting byte offset (default: 0) + endByte: Ending byte offset (optional, defaults to startByte + 1MB) + startLine: Starting line number (alternative to byte range) + lineCount: Number of lines to return (default: 1000) + + Returns: + content, startByte, endByte, totalSize, hasMore, nextChunkToken + """ + log_key = arguments.get('logKey') + start_byte = arguments.get('startByte', 0) + end_byte = arguments.get('endByte') + start_line = arguments.get('startLine') + line_count = arguments.get('lineCount', DEFAULT_LINE_COUNT) + + if not log_key: + return error_response(400, 'logKey is required') + + try: + head_result = safe_s3_head(log_key) + if not head_result['success']: + return success_response({ + 'logKey': log_key, 'content': '', 'startByte': 0, 'endByte': 0, + 'chunkSize': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'hasMore': False, 'nextChunkToken': None, 'truncated': False, + 'fileNotFound': True, + 'warning': head_result.get('error', 'File not found'), + 'suggestion': 'Try listing available logs first.', + }) + + total_size = head_result['size'] + + if total_size > MAX_CHUNK_SIZE * 10: + return get_artifact_reference({'logKey': log_key, 'reason': 'File too large for direct read'}) + + # Line-based reading + if start_line is not None: + return read_by_lines(LOGS_BUCKET, log_key, start_line, min(line_count, MAX_LINE_COUNT)) + + # Byte-range reading + if end_byte is None: + end_byte = min(start_byte + DEFAULT_CHUNK_SIZE, total_size) + + start_byte = max(0, start_byte) + end_byte = min(end_byte, total_size) + chunk_size = end_byte - start_byte + + if chunk_size > MAX_CHUNK_SIZE: + end_byte = start_byte + MAX_CHUNK_SIZE + chunk_size = MAX_CHUNK_SIZE + + if total_size == 0 or chunk_size <= 0: + return success_response({ + 'logKey': log_key, 'content': '', 'startByte': 0, 'endByte': 0, + 'chunkSize': 0, 'totalSize': total_size, 'totalSizeHuman': format_bytes(total_size), + 'hasMore': False, 'nextChunkToken': None, 'truncated': False, + 'info': 'File is empty or requested range is invalid', + }) + + # Line-aligned byte-range reads + BOUNDARY_SCAN = 4096 + actual_start = max(0, start_byte - 1) if start_byte > 0 else 0 + actual_end = min(end_byte + BOUNDARY_SCAN, total_size) + + range_header = f'bytes={actual_start}-{actual_end - 1}' + read_result = safe_s3_read(log_key, range_bytes=range_header) + + if not read_result['success']: + return success_response({ + 'logKey': log_key, 'content': '', 'startByte': start_byte, 'endByte': end_byte, + 'chunkSize': 0, 'totalSize': total_size, 'totalSizeHuman': format_bytes(total_size), + 'hasMore': False, 'nextChunkToken': None, 'truncated': False, + 'warning': read_result.get('error', 'Failed to read file content'), + }) + + raw = read_result['content'] + + aligned_start = start_byte + if start_byte > 0: + first_nl = raw.find('\n') + if first_nl >= 0: + aligned_start = actual_start + first_nl + 1 + raw = raw[first_nl + 1:] + + content_end_offset = end_byte - aligned_start + if content_end_offset < len(raw) and end_byte < total_size: + nl_pos = raw.find('\n', content_end_offset) + if nl_pos >= 0: + raw = raw[:nl_pos + 1] + aligned_end = aligned_start + nl_pos + 1 + else: + raw = raw[:content_end_offset] + aligned_end = end_byte + else: + aligned_end = aligned_start + len(raw) + + has_more = aligned_end < total_size + + return success_response({ + 'logKey': log_key, + 'content': raw, + 'startByte': aligned_start, + 'endByte': aligned_end, + 'chunkSize': len(raw), + 'totalSize': total_size, + 'totalSizeHuman': format_bytes(total_size), + 'hasMore': has_more, + 'nextChunkToken': str(aligned_end) if has_more else None, + 'truncated': False, + 'lineAligned': True, + }) + + except Exception as e: + return success_response({ + 'logKey': log_key, 'content': '', 'startByte': 0, 'endByte': 0, + 'chunkSize': 0, 'totalSize': 0, 'totalSizeHuman': '0 B', + 'hasMore': False, 'nextChunkToken': None, 'truncated': False, + 'error': f'Unexpected error reading log: {str(e)}', + }) + + +# ============================================================================= +# TIER 2: ADVANCED ANALYSIS +# ============================================================================= + +def search_logs_deep(arguments: Dict) -> Dict: + """ + Full-text regex search across all collected logs. + + Inputs: + instanceId: EC2 instance ID (required) + query: Regex pattern to search (required) + logTypes: Comma-separated log types to search (optional) + maxResults: Max results per file (default: 100) + + Returns: + matches[], pagination info, coverage_report + """ + instance_id = arguments.get('instanceId') + query = arguments.get('query') + log_types_str = arguments.get('logTypes', '') + max_results = min(arguments.get('maxResults', 100), 500) + + if not instance_id: + return error_response(400, 'instanceId is required') + if not query: + return error_response(400, 'query is required') + if len(query) > 500: + return error_response(400, 'query too long (max 500 characters)') + + try: + try: + pattern = re.compile(query, re.IGNORECASE) + except re.error as e: + return error_response(400, f'Invalid regex pattern: {str(e)}') + + file_patterns = None + if log_types_str: + file_patterns = [] + for log_type in log_types_str.split(','): + log_type = log_type.strip().lower() + if log_type in ECS_LOG_TYPE_PATTERNS: + file_patterns.extend(ECS_LOG_TYPE_PATTERNS[log_type]) + + # Use shared latest-bundle discovery + bundle_info = find_latest_bundle_files(instance_id) + + if not bundle_info['success']: + return success_response({ + 'instanceId': instance_id, 'query': query, + 'filesSearched': 0, 'filesWithMatches': 0, 'totalMatches': 0, + 'results': [], 'truncated': False, + 'warning': bundle_info.get('error', 'Failed to list log files'), + 'nextStep': 'Check if logs exist with validate', + }) + + files_to_search = [] + large_file_count = 0 + size_map = {obj['key']: obj['size'] for obj in bundle_info['all_objects']} + for key in bundle_info['files']: + if any(key.endswith(ext) for ext in ['.tar.gz', '.zip', '.gz', '.bin', '.so']): + continue + fsize = size_map.get(key, 0) + if fsize > 52428800: + large_file_count += 1 + continue + if file_patterns: + if not any(p in key.lower() for p in file_patterns): + continue + files_to_search.append({'key': key, 'size': fsize}) + + if not files_to_search: + return success_response({ + 'instanceId': instance_id, 'query': query, + 'filesSearched': 0, 'filesWithMatches': 0, 'totalMatches': 0, + 'results': [], 'truncated': False, + 'info': 'No log files found matching criteria.', + 'nextStep': 'Check log collection status or try different log types', + }) + + all_matches = [] + files_searched = 0 + files_with_errors = 0 + + for file_info in files_to_search[:50]: + files_searched += 1 + matches = search_file_for_pattern(LOGS_BUCKET, file_info['key'], pattern, max_results) + if matches is None: + files_with_errors += 1 + continue + if matches: + filename = file_info['key'].split('/extracted/')[-1] + all_matches.append({ + 'file': filename, 'fullKey': file_info['key'], + 'matchCount': len(matches), 'matches': matches, + }) + if sum(len(m['matches']) for m in all_matches) >= max_results * 3: + break + + all_matches.sort(key=lambda x: x['matchCount'], reverse=True) + + finding_counter = 0 + for match_group in all_matches: + finding_counter += 1 + match_group['finding_id'] = f"S-{finding_counter:03d}" + + total_matches_kept = 0 + for match_group in all_matches: + remaining_budget = max(10, max_results * 3 - total_matches_kept) + if len(match_group['matches']) > remaining_budget: + match_group['matches'] = match_group['matches'][:remaining_budget] + match_group['matchCount'] = len(match_group['matches']) + match_group['matchesTruncated'] = True + total_matches_kept += len(match_group['matches']) + + return success_response({ + 'instanceId': instance_id, + 'query': query, + 'filesSearched': files_searched, + 'filesWithMatches': len(all_matches), + 'totalMatches': sum(m['matchCount'] for m in all_matches), + 'results': all_matches, + 'truncated': files_searched < len(files_to_search), + 'coverage_report': { + 'files_searched': files_searched, + 'files_available': len(files_to_search), + 'files_skipped_size': large_file_count, + 'files_with_errors': files_with_errors, + 'scan_complete': files_searched >= len(files_to_search), + }, + 'interpretationGuide': { + 'CannotPullContainerError': 'ECR auth failure or image not found. Check task execution role.', + 'OOMKilled': 'Container exceeded memory limit. Check task definition memory.', + 'AGENT_DISCONNECTED': 'ECS agent lost connection. Check instance networking.', + 'connection timed out': 'Network connectivity issue. Check security groups and NACLs.', + }, + 'nextStep': 'Use read to get full context around specific matches', + }) + + except Exception as e: + return success_response({ + 'instanceId': instance_id, 'query': query, + 'filesSearched': 0, 'filesWithMatches': 0, 'totalMatches': 0, + 'results': [], 'truncated': False, + 'error': f'Search encountered an error: {str(e)}', + 'nextStep': 'Check if logs exist with validate', + }) + + +def correlate_events(arguments: Dict) -> Dict: + """ + Cross-file timeline correlation for ECS incident analysis. + + Inputs: + instanceId: EC2 instance ID (required) + timeWindow: Seconds around pivot event (default: 60) + + Returns: + timeline[], correlations, temporal_clusters, potential_root_cause_chain + """ + instance_id = arguments.get('instanceId') + time_window = arguments.get('timeWindow', 60) + + if not instance_id: + return error_response(400, 'instanceId is required') + + try: + prefix = f'ecs_{instance_id}' + index_key = find_findings_index(prefix) + findings = [] + files_scanned = 0 + + if index_key: + read_result = safe_s3_read(index_key) + if read_result['success']: + try: + index_data = json.loads(read_result['content']) + findings = index_data.get('findings', []) + files_scanned = index_data.get('filesScanned', 0) + except json.JSONDecodeError: + pass + + if not findings: + error_summary = scan_and_index_errors(instance_id, 'all') + if error_summary['statusCode'] != 200: + return success_response({ + 'instanceId': instance_id, 'timeWindow': time_window, + 'timeline': [], 'byComponent': {}, 'correlations': [], + 'temporal_clusters': [], 'potential_root_cause_chain': [], + 'coverage_report': {'files_scanned': 0, 'scan_complete': False}, + 'confidence': 'none', 'gaps': ['Could not retrieve error data'], + 'nextStep': 'Check if logs exist with validate', + }) + summary_data = json.loads(error_summary['body']) + findings = summary_data.get('findings', []) + files_scanned = summary_data.get('coverage_report', {}).get('files_scanned', 0) + + if not findings: + return success_response({ + 'instanceId': instance_id, 'timeWindow': time_window, + 'timeline': [], 'byComponent': {}, 'correlations': [], + 'temporal_clusters': [], 'potential_root_cause_chain': [], + 'coverage_report': {'files_scanned': files_scanned, 'scan_complete': True}, + 'confidence': 'none', 'gaps': [], + 'info': 'No error findings to correlate. Instance may be healthy.', + 'nextStep': 'Use search to search for specific patterns', + }) + + # Build timeline + timeline = [] + for idx, finding in enumerate(findings): + timestamp = extract_timestamp(finding.get('sample', '')) + timeline.append({ + 'finding_id': finding.get('finding_id', assign_finding_id(idx + 1)), + 'timestamp': timestamp, + 'source': finding.get('file', 'unknown'), + 'severity': finding.get('severity', 'info'), + 'event': finding.get('pattern', ''), + 'sample': finding.get('sample', '')[:200], + 'count': finding.get('count', 1), + }) + + timeline.sort(key=lambda x: (SEVERITY_ORDER.get(x['severity'], 4), -x['count'])) + + by_component = {} + for event in timeline: + component = categorize_log_source(event['source']) + if component not in by_component: + by_component[component] = [] + by_component[component].append(event) + + temporal_clusters = _build_temporal_clusters(timeline, time_window) + root_cause_chain = _build_root_cause_chain(findings) + + critical_count = len([e for e in timeline if e['severity'] == 'critical']) + if critical_count > 0 and len(timeline) >= 3: + confidence = 'high' + elif len(timeline) >= 2: + confidence = 'medium' + else: + confidence = 'low' + + gaps = [] + if files_scanned < 10: + gaps.append('Few files scanned — some log sources may be missing') + timestamps_present = sum(1 for e in timeline if e.get('timestamp')) + if timestamps_present < len(timeline) * 0.5: + gaps.append('Many events lack timestamps — temporal ordering may be unreliable') + + return success_response({ + 'instanceId': instance_id, + 'timeWindow': time_window, + 'timeline': timeline[:50], + 'byComponent': by_component, + 'correlations': find_correlations(timeline), + 'temporal_clusters': temporal_clusters, + 'potential_root_cause_chain': root_cause_chain, + 'confidence': confidence, + 'gaps': gaps, + 'coverage_report': { + 'files_scanned': files_scanned, + 'components_found': list(by_component.keys()), + 'events_with_timestamps': timestamps_present, + 'events_total': len(timeline), + 'scan_complete': True, + }, + 'caveat': ( + 'Timeline correlation is based on pattern matching across log files. ' + 'Timestamps may not be perfectly synchronized across components. ' + 'Correlation does not imply causation.' + ), + 'nextStep': 'Use search to investigate specific events', + 'recommendedSOPs': match_sops_for_issues([], findings=findings), + }) + + except Exception as e: + return success_response({ + 'instanceId': instance_id, 'timeWindow': time_window, + 'timeline': [], 'byComponent': {}, 'correlations': [], + 'temporal_clusters': [], 'potential_root_cause_chain': [], + 'confidence': 'none', 'gaps': [f'Correlation error: {str(e)}'], + 'coverage_report': {'files_scanned': 0, 'scan_complete': False}, + 'error': f'Correlation encountered an error: {str(e)}', + 'nextStep': 'Check if logs exist with validate', + }) + + +def get_artifact_reference(arguments: Dict) -> Dict: + """ + Get secure presigned URL for large artifacts. + + Inputs: + logKey: S3 key of artifact (required) + expirationMinutes: URL expiration (default: 15, max: 60) + + Returns: + presignedUrl, s3Uri, size + """ + log_key = arguments.get('logKey') + expiration_minutes = min(arguments.get('expirationMinutes', 15), 60) + + if not log_key: + return error_response(400, 'logKey is required') + + try: + head_result = safe_s3_head(log_key) + if not head_result['success']: + return success_response({ + 'logKey': log_key, 'presignedUrl': None, + 's3Uri': f's3://{LOGS_BUCKET}/{log_key}', + 'size': 0, 'sizeHuman': '0 B', 'fileNotFound': True, + 'warning': head_result.get('error', 'File not found'), + }) + + try: + presigned_url = s3_client.generate_presigned_url( + 'get_object', + Params={'Bucket': LOGS_BUCKET, 'Key': log_key}, + ExpiresIn=expiration_minutes * 60, + ) + except Exception as e: + return success_response({ + 'logKey': log_key, 'presignedUrl': None, + 's3Uri': f's3://{LOGS_BUCKET}/{log_key}', + 'size': head_result['size'], 'sizeHuman': format_bytes(head_result['size']), + 'warning': f'Could not generate presigned URL: {str(e)}', + }) + + return success_response({ + 'logKey': log_key, + 'presignedUrl': presigned_url, + 's3Uri': f's3://{LOGS_BUCKET}/{log_key}', + 'size': head_result['size'], + 'sizeHuman': format_bytes(head_result['size']), + 'contentType': head_result.get('content_type', 'application/octet-stream'), + 'lastModified': head_result.get('last_modified'), + 'expiresIn': f'{expiration_minutes} minutes', + }) + + except Exception as e: + return success_response({ + 'logKey': log_key, 'presignedUrl': None, + 's3Uri': f's3://{LOGS_BUCKET}/{log_key}', + 'size': 0, 'sizeHuman': '0 B', + 'error': f'Unexpected error: {str(e)}', + }) + + +def generate_incident_summary(arguments: Dict) -> Dict: + """ + Generate structured incident summary grounded in finding_ids. + + Inputs: + instanceId: EC2 instance ID (required) + finding_ids: List of finding IDs from errors/search (required) + includeRecommendations: Include remediation suggestions (default: true) + includeTriage: Include ECS triage analysis (default: true) + + Returns: + summary with criticalFindings, timeline, recommendations, ecs_triage + """ + import time as _time + start_time = _time.time() + MAX_EXECUTION_TIME = 25 + + def check_timeout(): + if _time.time() - start_time > MAX_EXECUTION_TIME: + raise TimeoutError(f"Execution time exceeded {MAX_EXECUTION_TIME}s") + + instance_id = arguments.get('instanceId') + finding_ids = arguments.get('finding_ids', []) + include_recommendations = arguments.get('includeRecommendations', True) + include_triage = arguments.get('includeTriage', True) + + if not instance_id: + return error_response(400, 'instanceId is required') + if not finding_ids: + return error_response(400, + 'finding_ids is required. Call errors tool first to get finding_ids (F-001 format), ' + 'then pass them here to ground the summary in verified evidence.') + + try: + bundle_data = {} + try: + check_timeout() + bundle_result = validate_bundle_completeness({'instanceId': instance_id}) + if bundle_result['statusCode'] == 200: + bundle_data = json.loads(bundle_result['body']) + except TimeoutError: + raise + except Exception as e: + print(f"Warning: Could not get bundle completeness: {str(e)}") + + error_data = {} + try: + check_timeout() + error_result = get_error_summary({'instanceId': instance_id, 'severity': 'all', 'pageSize': 200}) + if error_result['statusCode'] == 200: + error_data = json.loads(error_result['body']) + except TimeoutError: + raise + except Exception as e: + print(f"Warning: Could not get error summary: {str(e)}") + + check_timeout() + + all_findings = error_data.get('findings', []) + summary_counts = error_data.get('summary', {'critical': 0, 'high': 0, 'medium': 0, 'low': 0, 'info': 0}) + + finding_id_set = set(finding_ids) + findings = [f for f in all_findings if f.get('finding_id') in finding_id_set] + unresolved_ids = finding_id_set - {f.get('finding_id') for f in findings} + + critical_findings = [f for f in findings if f.get('severity') == 'critical'][:10] + high_findings = [f for f in findings if f.get('severity') == 'high'][:10] + medium_findings = [f for f in findings if f.get('severity') == 'medium'][:5] + + affected_components = set() + for finding in findings: + affected_components.add(categorize_log_source(finding.get('file', ''))) + + if len(findings) >= 3 and critical_findings: + confidence = 'high' + elif len(findings) >= 1: + confidence = 'medium' + else: + confidence = 'low' + + gaps = [] + if unresolved_ids: + gaps.append(f'{len(unresolved_ids)} finding_ids could not be resolved: {list(unresolved_ids)[:5]}') + + summary = { + 'instanceId': instance_id, + 'generatedAt': datetime.utcnow().isoformat(), + 'executionTimeMs': int((_time.time() - start_time) * 1000), + 'grounded': True, + 'confidence': confidence, + 'gaps': gaps, + 'bundleStatus': { + 'complete': bundle_data.get('complete', False), + 'fileCount': bundle_data.get('fileCount', 0), + 'totalSize': bundle_data.get('totalSizeHuman', 'unknown'), + }, + 'errorSummary': { + 'critical': summary_counts.get('critical', 0), + 'high': summary_counts.get('high', 0), + 'medium': summary_counts.get('medium', 0), + 'low': summary_counts.get('low', 0), + 'info': summary_counts.get('info', 0), + 'total': len(all_findings), + }, + 'criticalFindings': [ + {'finding_id': f.get('finding_id'), 'file': f.get('file'), 'fullKey': f.get('fullKey'), + 'pattern': f.get('pattern'), 'count': f.get('count'), 'sample': f.get('sample', '')[:200]} + for f in critical_findings + ], + 'highFindings': [ + {'finding_id': f.get('finding_id'), 'file': f.get('file'), 'fullKey': f.get('fullKey'), + 'pattern': f.get('pattern'), 'count': f.get('count')} + for f in high_findings + ], + 'affectedComponents': list(affected_components), + } + + if not findings: + summary['info'] = 'No error findings detected. Instance may be healthy.' + + if include_recommendations: + summary['recommendations'] = generate_recommendations(critical_findings, high_findings, medium_findings) + + summary['artifactLinks'] = [] + for finding in critical_findings[:5]: + if finding.get('fullKey'): + summary['artifactLinks'].append({ + 'finding_id': finding.get('finding_id'), + 'file': finding.get('file'), + 'key': finding.get('fullKey'), + 'action': f'read(logKey="{finding.get("fullKey")}")', + }) + + if include_triage and findings: + try: + check_timeout() + triage_result = perform_ecs_triage(instance_id, findings, bundle_data) + summary['ecs_triage'] = triage_result + except TimeoutError: + summary['ecs_triage'] = {'triageVersion': '1.0', 'warning': 'Triage skipped due to time constraints.'} + except Exception as e: + summary['ecs_triage'] = {'triageVersion': '1.0', 'error': f'Triage failed: {str(e)}'} + elif include_triage: + summary['ecs_triage'] = { + 'triageVersion': '1.0', 'info': 'No findings to triage.', + 'task_states_detected': [], 'instance_conditions_detected': [], + 'most_likely_root_cause': None, 'evidence': [], + } + + summary['caveat'] = ( + 'Root cause analysis is based on log pattern matching only. ' + 'Verify findings by checking ECS agent logs, Docker daemon config, ' + 'task definitions, and instance networking. ' + 'Log patterns indicate symptoms, not always root causes.' + ) + + if summary.get('ecs_triage', {}).get('most_likely_root_cause'): + root_cause = summary['ecs_triage']['most_likely_root_cause'] + summary['nextStep'] = f"Root cause: {root_cause['category_name']} ({root_cause['confidence']}). Follow remediation steps." + else: + summary['nextStep'] = 'Use search for detailed investigation of specific patterns' + + # Auto-match SOPs based on findings and triage category + triage_cat = None + if summary.get('ecs_triage', {}).get('most_likely_root_cause'): + triage_cat = summary['ecs_triage']['most_likely_root_cause'].get('category') + summary['recommendedSOPs'] = match_sops_for_issues( + [], findings=findings, triage_category=triage_cat + ) + + summary['executionTimeMs'] = int((_time.time() - start_time) * 1000) + return success_response(summary) + + except TimeoutError: + return success_response({ + 'instanceId': instance_id, 'generatedAt': datetime.utcnow().isoformat(), + 'grounded': True, 'confidence': 'low', + 'gaps': ['Execution timed out — partial results only'], + 'warning': 'Execution timed out', + 'nextStep': 'Call errors first, then summarize with includeTriage=false', + }) + except Exception as e: + return success_response({ + 'instanceId': instance_id, 'generatedAt': datetime.utcnow().isoformat(), + 'grounded': True, 'confidence': 'none', + 'gaps': [f'Summary generation failed: {str(e)}'], + 'error': f'Could not generate complete summary: {str(e)}', + 'nextStep': 'Check if logs exist with validate', + }) + + +def list_collection_history(arguments: Dict) -> Dict: + """ + List historical ECS log collections for audit and comparison. + + Inputs: + instanceId: Filter by instance (optional) + maxResults: Max results (default: 20) + status: Filter by status (optional) + + Returns: + collections[], count + """ + instance_id = arguments.get('instanceId') + max_results = min(arguments.get('maxResults', 20), 50) + status_filter = arguments.get('status') + document_name = 'AWSSupport-CollectECSInstanceLogs' + + try: + filters = [{'Key': 'DocumentNamePrefix', 'Values': [document_name]}] + if status_filter: + filters.append({'Key': 'ExecutionStatus', 'Values': [status_filter]}) + + target_region = arguments.get('region', DEFAULT_REGION) + regions_to_try = [target_region] + common_regions = ['us-west-2', 'us-east-1', 'eu-west-1', 'ap-southeast-1'] + for r in common_regions: + if r not in regions_to_try: + regions_to_try.append(r) + + collections = [] + searched_regions = [] + + for region in regions_to_try: + try: + regional_ssm = get_regional_client('ssm', region) + response = regional_ssm.describe_automation_executions( + Filters=filters, MaxResults=max_results, + ) + for exec_meta in response.get('AutomationExecutionMetadataList', []): + if instance_id: + params = exec_meta.get('Parameters', {}) + exec_instance = params.get('ECSInstanceId', [''])[0] + if instance_id not in exec_instance: + continue + + exec_id = exec_meta['AutomationExecutionId'] + params = exec_meta.get('Parameters', {}) + exec_instance = params.get('ECSInstanceId', [''])[0] + bundle_exists = False + if exec_instance: + s3_check = safe_s3_list(f"ecs_{exec_instance}_{exec_id}/", max_keys=1) + bundle_exists = bool(s3_check.get('success') and s3_check.get('objects')) + + collections.append({ + 'executionId': exec_id, + 'documentName': exec_meta.get('DocumentName', ''), + 'status': exec_meta['AutomationExecutionStatus'], + 'startTime': exec_meta.get('ExecutionStartTime'), + 'endTime': exec_meta.get('ExecutionEndTime'), + 'instanceId': exec_instance or None, + 'region': region, + 'bundleExists': bundle_exists, + }) + + searched_regions.append(region) + if collections: + break + except Exception: + searched_regions.append(f"{region} (error)") + continue + + return success_response({ + 'collections': collections, + 'count': len(collections), + 'searchedRegions': searched_regions, + 'filters': {'instanceId': instance_id, 'status': status_filter, 'documentName': document_name}, + }) + + except Exception as e: + return error_response(500, f'Failed to list history: {str(e)}') + + +# ============================================================================= +# TIER 3: CLUSTER-LEVEL INTELLIGENCE +# ============================================================================= + +def cluster_health_check(arguments: Dict) -> Dict: + """ + Comprehensive ECS cluster health overview. + Enumerates container instances, checks SSM status, flags unhealthy instances. + + Inputs: + clusterName: ECS cluster name (required) + region: AWS region (optional) + includeSSMStatus: Check SSM agent per instance (default: true) + + Returns: + clusterInfo, instances[], healthSummary + """ + cluster_name = arguments.get('clusterName') + if not cluster_name: + return error_response(400, 'clusterName is required') + + include_ssm = arguments.get('includeSSMStatus', True) + target_region = resolve_region(arguments) + + try: + regional_ecs = get_regional_client('ecs', target_region) + regional_ec2 = get_regional_client('ec2', target_region) + regional_ssm = get_regional_client('ssm', target_region) + + # 1. Describe the cluster + try: + cluster_resp = regional_ecs.describe_clusters(clusters=[cluster_name], include=['STATISTICS', 'SETTINGS']) + clusters = cluster_resp.get('clusters', []) + if not clusters: + return error_response(404, f'Cluster {cluster_name} not found in {target_region}') + cluster_info = clusters[0] + cluster_meta = { + 'name': cluster_info.get('clusterName'), + 'status': cluster_info.get('status'), + 'registeredContainerInstancesCount': cluster_info.get('registeredContainerInstancesCount', 0), + 'runningTasksCount': cluster_info.get('runningTasksCount', 0), + 'pendingTasksCount': cluster_info.get('pendingTasksCount', 0), + 'activeServicesCount': cluster_info.get('activeServicesCount', 0), + 'region': target_region, + } + except Exception as e: + return error_response(404, f'Cluster {cluster_name} not found in {target_region}: {str(e)}') + + # 2. List container instances + container_instance_arns = [] + try: + paginator = regional_ecs.get_paginator('list_container_instances') + for page in paginator.paginate(cluster=cluster_name): + container_instance_arns.extend(page.get('containerInstanceArns', [])) + except Exception as e: + return error_response(500, f'Failed to list container instances: {str(e)}') + + instances = [] + instance_ids = [] + + if container_instance_arns: + # Describe in batches of 100 + for i in range(0, len(container_instance_arns), 100): + batch = container_instance_arns[i:i + 100] + try: + desc_resp = regional_ecs.describe_container_instances( + cluster=cluster_name, containerInstances=batch, + ) + for ci in desc_resp.get('containerInstances', []): + ec2_id = ci.get('ec2InstanceId', '') + if ec2_id: + instance_ids.append(ec2_id) + instances.append({ + 'containerInstanceArn': ci.get('containerInstanceArn'), + 'ec2InstanceId': ec2_id, + 'status': ci.get('status'), + 'agentConnected': ci.get('agentConnected'), + 'runningTasksCount': ci.get('runningTasksCount', 0), + 'pendingTasksCount': ci.get('pendingTasksCount', 0), + 'registeredAt': ci.get('registeredAt'), + 'agentVersion': ci.get('versionInfo', {}).get('agentVersion'), + 'dockerVersion': ci.get('versionInfo', {}).get('dockerVersion'), + 'ssmStatus': None, + }) + except Exception: + pass + + # 3. Enrich with EC2 metadata + if instance_ids: + try: + ec2_paginator = regional_ec2.get_paginator('describe_instances') + ec2_map = {} + for page in ec2_paginator.paginate(InstanceIds=instance_ids): + for res in page.get('Reservations', []): + for inst in res.get('Instances', []): + ec2_map[inst['InstanceId']] = { + 'instanceType': inst.get('InstanceType'), + 'availabilityZone': inst.get('Placement', {}).get('AvailabilityZone'), + 'state': inst.get('State', {}).get('Name'), + 'privateIp': inst.get('PrivateIpAddress'), + 'imageId': inst.get('ImageId'), + 'launchTime': inst.get('LaunchTime'), + } + for inst in instances: + ec2_data = ec2_map.get(inst['ec2InstanceId'], {}) + inst.update(ec2_data) + except Exception: + pass + + # 4. Check SSM status + if include_ssm and instance_ids: + ssm_status_map = {} + try: + for i in range(0, len(instance_ids), 50): + chunk = instance_ids[i:i + 50] + ssm_paginator = regional_ssm.get_paginator('describe_instance_information') + for page in ssm_paginator.paginate(Filters=[{'Key': 'InstanceIds', 'Values': chunk}]): + for info in page.get('InstanceInformationList', []): + ssm_status_map[info['InstanceId']] = { + 'pingStatus': info.get('PingStatus'), + 'agentVersion': info.get('AgentVersion'), + 'lastPingTime': info.get('LastPingDateTime'), + } + except Exception as e: + print(f"SSM status check failed: {e}") + + for inst in instances: + ssm_info = ssm_status_map.get(inst.get('ec2InstanceId')) + inst['ssmStatus'] = ssm_info or {'pingStatus': 'NotRegistered', 'agentVersion': None} + + # 5. Build health summary + total = len(instances) + active = sum(1 for i in instances if i.get('status') == 'ACTIVE') + agent_connected = sum(1 for i in instances if i.get('agentConnected')) + ssm_online = sum(1 for i in instances if i.get('ssmStatus', {}).get('pingStatus') == 'Online') + + az_distribution = {} + for i in instances: + az = i.get('availabilityZone', 'unknown') + az_distribution[az] = az_distribution.get(az, 0) + 1 + + unhealthy = [] + for i in instances: + issues = [] + if i.get('status') != 'ACTIVE': + issues.append(f"ecsStatus={i.get('status')}") + if not i.get('agentConnected'): + issues.append('agentDisconnected') + if include_ssm and i.get('ssmStatus', {}).get('pingStatus') != 'Online': + issues.append(f"ssm={i.get('ssmStatus', {}).get('pingStatus', 'unknown')}") + if issues: + unhealthy.append({'instanceId': i.get('ec2InstanceId'), 'issues': issues}) + + health_summary = { + 'totalInstances': total, + 'active': active, + 'agentConnected': agent_connected, + 'ssmOnline': ssm_online, + 'unhealthyCount': len(unhealthy), + 'azDistribution': az_distribution, + } + + gaps = [] + if not include_ssm: + gaps.append('SSM status not checked') + if total == 0: + gaps.append('No container instances found — cluster may be empty or Fargate-only') + + if total > 0 and include_ssm and ssm_online == total: + confidence = 'high' + elif total > 0: + confidence = 'medium' + else: + confidence = 'none' + + return success_response({ + 'cluster': cluster_meta, + 'instances': instances, + 'unhealthyInstances': unhealthy, + 'healthSummary': health_summary, + 'region': target_region, + 'confidence': confidence, + 'gaps': gaps, + 'nextStep': 'Use compare_instances to diff specific instances, or batch_collect to sample unhealthy ones' if unhealthy else 'Cluster looks healthy.', + }) + + except Exception as e: + return error_response(500, f'cluster_health failed: {str(e)}') + + +def compare_instances(arguments: Dict) -> Dict: + """ + Diff error findings between two or more ECS container instances. + + Inputs: + instanceIds: list of 2+ instance IDs (required) + compareFields: "errors", "config", "all" (default: "all") + + Returns: + commonFindings[], uniqueFindings{}, comparisonMatrix + """ + instance_ids = arguments.get('instanceIds', []) + if not instance_ids or len(instance_ids) < 2: + return error_response(400, 'instanceIds must contain at least 2 instance IDs') + + seen = set() + deduped = [] + for iid in instance_ids: + if iid not in seen: + seen.add(iid) + deduped.append(iid) + instance_ids = deduped + if len(instance_ids) < 2: + return error_response(400, 'instanceIds must contain at least 2 distinct instance IDs') + if len(instance_ids) > 10: + return error_response(400, 'Maximum 10 instances for comparison') + + compare_fields = arguments.get('compareFields', 'all') + + try: + node_findings = {} + node_configs = {} + + def _gather_instance_data(iid): + nf = [] + nc = {} + if compare_fields in ('errors', 'all'): + prefix = f"ecs_{iid}" + try: + idx = find_findings_index(prefix) + if idx: + resp = s3_client.get_object(Bucket=LOGS_BUCKET, Key=idx) + findings_data = json.loads(resp['Body'].read().decode('utf-8')) + nf = findings_data.get('findings', []) + else: + nf = [{'error': f'No findings index for {iid}. Run collect first.', 'needsCollection': True}] + except Exception as e: + nf = [{'error': f'Could not load findings: {str(e)}'}] + + if compare_fields in ('config', 'all'): + bundle_info = find_latest_bundle_files(iid) + extracted_prefix = None + if bundle_info['success']: + extracted_prefix = bundle_info['bundle_prefix'] + '/extracted/' + if extracted_prefix: + config_files = [ + ('ecs_config', f"{extracted_prefix}ecs.config"), + ('docker_daemon', f"{extracted_prefix}docker-daemon.json"), + ] + else: + config_files = [] + for config_name, config_key in config_files: + result = safe_s3_read(config_key, max_size=65536) + nc[config_name] = result['content'][:2000] if result.get('success') else None + return iid, nf, nc + + with ThreadPoolExecutor(max_workers=min(len(instance_ids), 10)) as executor: + futures = {executor.submit(_gather_instance_data, iid): iid for iid in instance_ids} + for future in as_completed(futures): + iid_key = futures[future] + try: + iid, nf, nc = future.result() + node_findings[iid] = nf + node_configs[iid] = nc + except Exception as e: + node_findings[iid_key] = [{'error': f'Failed: {str(e)}'}] + node_configs[iid_key] = {} + + # Build comparison + common_findings = [] + unique_findings = {} + + if compare_fields in ('errors', 'all') and node_findings: + def finding_signature(f): + if 'error' in f and 'severity' not in f: + return f"__error__{f.get('error', 'unknown')[:80]}" + return f"{f.get('severity', '')}__{f.get('category', '')}__{f.get('pattern', f.get('message', ''))[:80]}" + + sig_to_nodes = {} + for iid, findings in node_findings.items(): + unique_findings[iid] = [] + for f in findings: + sig = finding_signature(f) + if sig not in sig_to_nodes: + sig_to_nodes[sig] = {'finding': f, 'nodes': []} + sig_to_nodes[sig]['nodes'].append(iid) + + for sig, data in sig_to_nodes.items(): + if len(data['nodes']) == len(instance_ids): + common_findings.append({**data['finding'], 'presentOnAllInstances': True}) + else: + for iid in data['nodes']: + unique_findings[iid].append({**data['finding'], 'uniqueTo': iid}) + + # Config diff + config_diffs = {} + if compare_fields in ('config', 'all') and node_configs: + ref_id = instance_ids[0] + ref_config = node_configs.get(ref_id, {}) + for iid in instance_ids[1:]: + other_config = node_configs.get(iid, {}) + diffs = [] + all_keys = set(list(ref_config.keys()) + list(other_config.keys())) + for key in all_keys: + if ref_config.get(key) != other_config.get(key): + diffs.append({ + 'configFile': key, 'referenceInstance': ref_id, + 'comparedInstance': iid, 'match': False, + 'note': 'Content differs' if (ref_config.get(key) and other_config.get(key)) else 'Missing on one instance', + }) + config_diffs[f"{ref_id}_vs_{iid}"] = diffs if diffs else [{'match': True, 'note': 'Configs identical'}] + + matrix = [] + for iid in instance_ids: + total_f = len(node_findings.get(iid, [])) + unique_count = len(unique_findings.get(iid, [])) + critical_count = sum(1 for f in node_findings.get(iid, []) if f.get('severity') == 'critical') + matrix.append({ + 'instanceId': iid, 'totalFindings': total_f, + 'criticalFindings': critical_count, 'uniqueFindings': unique_count, + 'commonFindings': total_f - unique_count, + }) + + common_count = len(common_findings) + total_unique = sum(len(v) for v in unique_findings.values()) + if common_count > 0 and total_unique == 0: + insight = f"All {len(instance_ids)} instances share the same {common_count} findings. Likely a cluster-wide issue." + elif common_count == 0 and total_unique > 0: + insight = "No common findings. Each instance has unique issues — investigate individually." + elif common_count > total_unique: + insight = f"{common_count} common vs {total_unique} unique. Mostly a shared problem." + else: + insight = f"{common_count} common, {total_unique} unique. Mixed picture." + + return success_response({ + 'comparedInstances': instance_ids, + 'commonFindings': common_findings, + 'commonFindingsCount': common_count, + 'uniqueFindings': unique_findings, + 'configDiffs': config_diffs, + 'comparisonMatrix': matrix, + 'insight': insight, + 'caveat': 'Comparison is based on pre-indexed error findings. Differences may reflect different workloads.', + 'nextStep': 'Common findings suggest cluster-wide issue. Unique findings point to instance-specific problems.', + }) + + except Exception as e: + return error_response(500, f'compare_instances failed: {str(e)}') + + +def batch_collect(arguments: Dict) -> Dict: + """ + Smart batch log collection with statistical sampling for ECS clusters. + + Inputs: + clusterName: ECS cluster name (required) + region: AWS region (optional) + filter: "all", "unhealthy", "disconnected" (default: "unhealthy") + strategy: "sample" or "all" (default: "sample") + samplesPerBucket: instances per bucket (default: 3, max: 5) + maxTotalCollections: hard cap (default: 15, max: 15) + dryRun: preview only (default: false) + + Returns: + buckets[], plannedCollections, executions[] (if not dryRun) + """ + cluster_name = arguments.get('clusterName') + if not cluster_name: + return error_response(400, 'clusterName is required') + + target_region = resolve_region(arguments) + node_filter = arguments.get('filter', 'unhealthy') + valid_filters = ('all', 'unhealthy', 'disconnected') + if node_filter not in valid_filters: + return error_response(400, f"Invalid filter '{node_filter}'. Must be one of: {', '.join(valid_filters)}") + strategy = arguments.get('strategy', 'sample') + samples_per_bucket = min(arguments.get('samplesPerBucket', 3), 5) + max_total = min(arguments.get('maxTotalCollections', 15), 15) + dry_run = arguments.get('dryRun', False) + + try: + regional_ecs = get_regional_client('ecs', target_region) + regional_ec2 = get_regional_client('ec2', target_region) + regional_ssm = get_regional_client('ssm', target_region) + + # 1. List container instances + ci_arns = [] + paginator = regional_ecs.get_paginator('list_container_instances') + for page in paginator.paginate(cluster=cluster_name): + ci_arns.extend(page.get('containerInstanceArns', [])) + + if not ci_arns: + return error_response(404, f'No container instances found for cluster {cluster_name}') + + # 2. Describe container instances + all_instances = [] + for i in range(0, len(ci_arns), 100): + batch = ci_arns[i:i + 100] + desc_resp = regional_ecs.describe_container_instances(cluster=cluster_name, containerInstances=batch) + for ci in desc_resp.get('containerInstances', []): + ec2_id = ci.get('ec2InstanceId', '') + all_instances.append({ + 'instanceId': ec2_id, + 'status': ci.get('status'), + 'agentConnected': ci.get('agentConnected'), + 'runningTasks': ci.get('runningTasksCount', 0), + }) + + # 3. Check SSM status + instance_ids = [i['instanceId'] for i in all_instances if i['instanceId']] + ssm_status = {} + if instance_ids: + try: + for i in range(0, len(instance_ids), 50): + chunk = instance_ids[i:i + 50] + ssm_paginator = regional_ssm.get_paginator('describe_instance_information') + for page in ssm_paginator.paginate(Filters=[{'Key': 'InstanceIds', 'Values': chunk}]): + for info in page.get('InstanceInformationList', []): + ssm_status[info['InstanceId']] = info.get('PingStatus', 'Unknown') + except Exception: + pass + + # 4. Apply filter + filtered = [] + for inst in all_instances: + ssm_ping = ssm_status.get(inst['instanceId'], 'NotRegistered') + inst['ssmPingStatus'] = ssm_ping + is_unhealthy = inst['status'] != 'ACTIVE' or not inst['agentConnected'] or ssm_ping != 'Online' + is_disconnected = not inst['agentConnected'] or ssm_ping != 'Online' + + if node_filter == 'all': + filtered.append(inst) + elif node_filter == 'unhealthy' and is_unhealthy: + filtered.append(inst) + elif node_filter == 'disconnected' and is_disconnected: + filtered.append(inst) + + if not filtered and node_filter in ('unhealthy', 'disconnected'): + return success_response({ + 'message': f'No {node_filter} instances found — cluster looks healthy', + 'totalInstances': len(all_instances), 'filteredInstances': 0, + 'filter': node_filter, 'buckets': [], 'plannedCollections': 0, + }) + + # 5. Group into buckets by status + SSM + buckets = {} + for inst in filtered: + key = f"{inst['status']}|{inst['ssmPingStatus']}" + if key not in buckets: + buckets[key] = {'signature': key, 'nodes': [], 'count': 0} + buckets[key]['nodes'].append(inst) + buckets[key]['count'] += 1 + + # 6. Select samples + bucket_list = [] + total_planned = 0 + for sig, bucket in buckets.items(): + sample_count = min(samples_per_bucket, bucket['count']) if strategy == 'sample' else bucket['count'] + if total_planned + sample_count > max_total: + sample_count = max(0, max_total - total_planned) + sample_nodes = bucket['nodes'][:sample_count] + total_planned += len(sample_nodes) + bucket_list.append({ + 'signature': sig, 'totalInstances': bucket['count'], + 'sampleCount': len(sample_nodes), + 'sampleInstances': [n['instanceId'] for n in sample_nodes], + }) + + if dry_run: + return success_response({ + 'dryRun': True, 'clusterName': cluster_name, 'region': target_region, + 'totalInstances': len(all_instances), 'filteredInstances': len(filtered), + 'filter': node_filter, 'strategy': strategy, + 'bucketCount': len(bucket_list), 'buckets': bucket_list, + 'plannedCollections': total_planned, + 'message': f'{len(filtered)} instances grouped into {len(bucket_list)} buckets. Will collect from {total_planned}. Re-run with dryRun=false to proceed.', + }) + + # 7. Execute collections + batch_id = hashlib.md5(f"{cluster_name}-{datetime.utcnow().isoformat()}".encode()).hexdigest()[:12] + executions = [] + + for bucket in bucket_list: + for iid in bucket['sampleInstances']: + try: + collect_args = { + 'instanceId': iid, 'region': target_region, + 'idempotencyToken': f"batch-{batch_id}-{iid}", + } + result = start_log_collection(collect_args) + result_body = json.loads(result.get('body', '{}')) + executions.append({ + 'instanceId': iid, 'bucket': bucket['signature'], + 'executionId': result_body.get('executionId'), + 'status': 'Started' if result_body.get('success') else 'Failed', + 'error': result_body.get('error'), + }) + except Exception as e: + executions.append({'instanceId': iid, 'bucket': bucket['signature'], 'status': 'Failed', 'error': str(e)}) + + # Store batch metadata + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, Key=f"batches/{batch_id}/metadata.json", + Body=json.dumps({'batchId': batch_id, 'clusterName': cluster_name, 'region': target_region, + 'createdAt': datetime.utcnow().isoformat(), 'executions': executions, 'buckets': bucket_list}, default=str), + ContentType='application/json', + ) + except Exception: + pass + + started = sum(1 for e in executions if e['status'] == 'Started') + failed = sum(1 for e in executions if e['status'] == 'Failed') + + return success_response({ + 'batchId': batch_id, 'clusterName': cluster_name, 'region': target_region, + 'totalInstances': len(all_instances), 'filteredInstances': len(filtered), + 'bucketCount': len(bucket_list), 'buckets': bucket_list, + 'executions': executions, 'collectionsStarted': started, 'collectionsFailed': failed, + 'task': { + 'taskId': batch_id, 'state': 'running' if started > 0 else 'failed', + 'message': f'{started} collections started, {failed} failed', 'progress': 0, + }, + 'nextStep': f'Use batch_status(batchId="{batch_id}") to poll all collections.', + }) + + except Exception as e: + return error_response(500, f'batch_collect failed: {str(e)}') + + +def batch_status(arguments: Dict) -> Dict: + """ + Poll status of multiple log collections at once. + Returns consolidated view with allComplete flag. + + Inputs: + executionIds: list of SSM execution IDs (required if no batchId) + batchId: batch ID from batch_collect (alternative to executionIds) + + Returns: + allComplete, summary counts, per-execution status + """ + execution_ids = arguments.get('executionIds', []) + batch_id = arguments.get('batchId') + + # If batchId provided, load execution IDs from stored metadata + if batch_id and not execution_ids: + try: + meta_result = safe_s3_read(f"batches/{batch_id}/metadata.json") + if meta_result.get('success'): + meta = json.loads(meta_result['content']) + execution_ids = [ + e['executionId'] for e in meta.get('executions', []) + if e.get('executionId') + ] + except Exception: + pass + + if not execution_ids: + return error_response(400, 'executionIds list or batchId is required') + + # Deduplicate + execution_ids = list(dict.fromkeys(execution_ids)) + + # Poll all executions in parallel + results = [] + + def _poll(eid): + try: + target_region = get_execution_region(eid) or DEFAULT_REGION + regional_ssm = get_regional_client('ssm', target_region) + resp = regional_ssm.get_automation_execution(AutomationExecutionId=eid) + execution = resp['AutomationExecution'] + status = execution['AutomationExecutionStatus'] + params = execution.get('Parameters', {}) + instance_id = params.get('ECSInstanceId', [None])[0] if params.get('ECSInstanceId') else None + return { + 'executionId': eid, + 'instanceId': instance_id, + 'status': status, + 'progress': 100 if status == 'Success' else (0 if status == 'Failed' else estimate_progress(execution)), + 'failureReason': parse_failure_reason(execution) if status == 'Failed' else None, + } + except Exception as e: + return { + 'executionId': eid, + 'instanceId': None, + 'status': 'Unknown', + 'progress': 0, + 'error': str(e), + } + + with ThreadPoolExecutor(max_workers=min(len(execution_ids), 15)) as executor: + results = list(executor.map(_poll, execution_ids)) + + # Compute summary + succeeded = [r for r in results if r['status'] == 'Success'] + failed = [r for r in results if r['status'] == 'Failed'] + in_progress = [r for r in results if r['status'] in ('InProgress', 'Pending', 'Waiting')] + unknown = [r for r in results if r['status'] not in ('Success', 'Failed', 'InProgress', 'Pending', 'Waiting')] + + all_complete = len(in_progress) == 0 and len(unknown) == 0 + + response_data = { + 'allComplete': all_complete, + 'summary': { + 'total': len(results), + 'succeeded': len(succeeded), + 'failed': len(failed), + 'inProgress': len(in_progress), + 'unknown': len(unknown), + }, + 'executions': results, + } + + if all_complete: + ready_instances = [r['instanceId'] for r in succeeded if r['instanceId']] + failed_instances = [r['instanceId'] for r in failed if r['instanceId']] + response_data['nextStep'] = ( + f"All collections complete. {len(succeeded)} succeeded, {len(failed)} failed. " + f"Use errors/search/network_diagnostics on succeeded instances: {ready_instances[:5]}." + ) + if failed_instances: + response_data['failedInstances'] = failed_instances + else: + response_data['nextStep'] = f'{len(in_progress)} still running. Poll again in 15 seconds.' + response_data['suggestedPollIntervalSeconds'] = 15 + + return success_response(response_data) + + +def network_diagnostics(arguments: Dict) -> Dict: + """ + Extract and structure networking info from collected ECS log bundles. + Parses iptables, docker networking, routes, DNS, ENI, and security groups. + + Inputs: + instanceId: EC2 instance ID (required) + sections: comma-separated: "iptables,docker,routes,dns,eni,security-groups" or "all" (default: "all") + + Returns: + Structured networking diagnostics per section + """ + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + + sections_str = arguments.get('sections', 'all') + valid_sections = {'iptables', 'docker', 'routes', 'dns', 'eni', 'security-groups'} + if sections_str == 'all': + sections = ['iptables', 'docker', 'routes', 'dns', 'eni', 'security-groups'] + else: + sections = [s.strip() for s in sections_str.split(',')] + invalid = [s for s in sections if s not in valid_sections] + if invalid: + return error_response(400, f"Invalid section(s): {', '.join(invalid)}. Valid: {', '.join(sorted(valid_sections))}") + if not sections: + return error_response(400, 'At least one section is required') + + prefix = f"logs/{instance_id}/extracted/" + results = {} + issues_found = [] + + try: + # Use shared latest-bundle discovery + bundle_info = find_latest_bundle_files(instance_id) + if not bundle_info['success']: + return error_response(404, bundle_info.get('error', f'No extracted log bundle found for {instance_id}. Run collect first.')) + + bundle_files = bundle_info['files'] + bundle_age_minutes = bundle_info['bundle_age_minutes'] + bundle_collected_at = bundle_info['bundle_collected_at'] + + # HARD BLOCK: Do NOT analyze stale bundles — force fresh collection + STALE_THRESHOLD_MINUTES = 15 + if bundle_age_minutes is not None and bundle_age_minutes > STALE_THRESHOLD_MINUTES: + return error_response(409, ( + f'STALE BUNDLE: The log bundle for {instance_id} is {bundle_age_minutes} minutes old ' + f'(collected at {bundle_collected_at}). The instance state has likely changed since then. ' + f'You MUST run the collect tool first to gather fresh logs, wait for it to complete ' + f'(poll status until success), then call network_diagnostics again. ' + f'Do NOT draw conclusions from stale data.' + ), { + 'bundleInfo': { + 'collectedAt': bundle_collected_at, + 'ageMinutes': bundle_age_minutes, + 'isStale': True, + 'staleThresholdMinutes': STALE_THRESHOLD_MINUTES, + }, + 'action': 'Run collect tool, poll status until complete, then retry network_diagnostics', + }) + + def find_files(patterns): + """Find bundle files matching any of the given patterns.""" + matched = [] + for f in bundle_files: + fname = f.lower() + for p in patterns: + if p in fname: + matched.append(f) + break + return matched + + # Pre-fetch all needed files in parallel + files_to_fetch = set() + section_file_map = {} + fetch_sizes = {} + + if 'iptables' in sections: + keys = find_files(['iptables', 'ip-tables', 'iptable'])[:6] + section_file_map['iptables'] = keys + for k in keys: files_to_fetch.add(k); fetch_sizes[k] = 262144 + if 'docker' in sections: + keys = find_files(['docker', 'containerd', 'bridge', 'docker-network', 'brctlshow', 'veth'])[:5] + section_file_map['docker'] = keys + for k in keys: files_to_fetch.add(k); fetch_sizes[k] = 262144 + if 'routes' in sections: + r_keys = find_files(['ip-route', 'ip_route', 'route-table', 'routes'])[:3] + i_keys = find_files(['ifconfig', 'ip-addr', 'ip_addr', 'interfaces', 'ipaddrshow'])[:2] + section_file_map['routes'] = r_keys + section_file_map['routes_iface'] = i_keys + for k in r_keys + i_keys: files_to_fetch.add(k); fetch_sizes[k] = 262144 + if 'dns' in sections: + keys = find_files(['resolv', 'dns'])[:5] + section_file_map['dns'] = keys + for k in keys: files_to_fetch.add(k); fetch_sizes[k] = 262144 + if 'eni' in sections: + keys = find_files(['eni', 'network-interface', 'eth'])[:3] + section_file_map['eni'] = keys + for k in keys: files_to_fetch.add(k); fetch_sizes[k] = 32768 + if 'security-groups' in sections: + section_file_map['security-groups'] = [] # Fetched via API + + # Parallel S3 reads + file_contents = {} + def _fetch(key): + r = safe_s3_read(key, max_size=fetch_sizes.get(key, 262144)) + return key, r.get('content', '') if r.get('success') else None + + with ThreadPoolExecutor(max_workers=10) as executor: + fetch_list = list(files_to_fetch) + for key, content in executor.map(_fetch, fetch_list): + file_contents[key] = content + + def read_file_content(key, max_size=262144): + cached = file_contents.get(key) + if cached is not None: + return cached + r = safe_s3_read(key, max_size=max_size) + return r.get('content', '') if r.get('success') else None + + # ================================================================= + # IPTABLES + # ================================================================= + if 'iptables' in sections: + ipt_data = {'chainCount': 0, 'ruleCount': 0, 'natRules': [], 'dockerRules': [], 'issues': []} + + # BUG FIX: Parse ALL iptables files and merge results. + # Docker DNAT/MASQUERADE rules live in the NAT table, not the filter table. + # Previously we broke after the first file (usually iptables-filter.txt), + # missing NAT table rules entirely. + ipt_files = find_files(['iptables', 'ip-tables', 'iptable']) + + all_lines_merged = [] + all_nat_rules = [] + all_docker_rules = [] + total_rule_count = 0 + total_chain_count = 0 + source_files = [] + + # Prefer iptables-save.txt if available + save_file = None + for f in ipt_files: + fl = f.lower() + if 'iptables-save' in fl or 'iptables_save' in fl: + save_file = f + break + + files_to_parse = [save_file] if save_file else ipt_files[:6] + + for f in files_to_parse: + content = read_file_content(f) + if not content: + continue + lines = content.splitlines() + source_files.append(f) + all_lines_merged.extend(lines) + + total_rule_count += sum(1 for l in lines if l.strip() and not l.startswith('#') and not l.startswith('*') and not l.startswith(':')) + total_chain_count += sum(1 for l in lines if l.startswith(':')) + all_nat_rules.extend(l.strip() for l in lines if 'DNAT' in l or 'SNAT' in l or 'MASQUERADE' in l) + all_docker_rules.extend(l.strip() for l in lines if 'DOCKER' in l) + + ipt_data['ruleCount'] = total_rule_count + ipt_data['chainCount'] = total_chain_count + ipt_data['natRules'] = all_nat_rules[:20] + ipt_data['dockerRules'] = all_docker_rules[:20] + ipt_data['sourceFiles'] = source_files + ipt_data['sourceFile'] = source_files[0] if source_files else None + + if all_lines_merged: + if total_rule_count == 0: + ipt_data['issues'].append('No iptables rules found — Docker networking may be broken') + issues_found.append({'section': 'iptables', 'severity': 'critical', 'message': 'No iptables rules found'}) + if not any('DOCKER' in l for l in all_lines_merged): + ipt_data['issues'].append('DOCKER chain missing — Docker bridge networking not configured') + issues_found.append({'section': 'iptables', 'severity': 'warning', 'message': 'DOCKER chain missing'}) + + results['iptables'] = ipt_data + + # ================================================================= + # DOCKER NETWORKING + # ================================================================= + if 'docker' in sections: + docker_data = {'networkMode': None, 'bridgeConfig': {}, 'errors': [], 'issues': []} + docker_files = find_files(['docker', 'containerd', 'bridge', 'docker-network']) + for f in docker_files[:5]: + content = read_file_content(f) + if content: + # Parse docker daemon config + if f.endswith('.json') or 'daemon' in f.lower(): + try: + cfg = json.loads(content) + docker_data['bridgeConfig'] = cfg + if 'bridge' in cfg: + docker_data['networkMode'] = 'bridge' + if cfg.get('iptables') is False: + docker_data['issues'].append('Docker iptables disabled — container networking may fail') + issues_found.append({'section': 'docker', 'severity': 'critical', 'message': 'Docker iptables disabled'}) + except json.JSONDecodeError: + pass + # Parse docker logs for network errors + for line in content.split('\n'): + ll = line.lower() + if ('error' in ll or 'failed' in ll) and ('network' in ll or 'bridge' in ll or 'eni' in ll): + docker_data['errors'].append(line.strip()[:200]) + if docker_data['errors']: + docker_data['issues'].append(f"{len(docker_data['errors'])} Docker networking errors found") + issues_found.append({'section': 'docker', 'severity': 'warning', 'message': f"{len(docker_data['errors'])} Docker network errors"}) + docker_data['errors'] = docker_data['errors'][:20] + results['docker'] = docker_data + + # ================================================================= + # ROUTE TABLES + # ================================================================= + if 'routes' in sections: + route_data = {'routes': [], 'defaultGateway': None, 'interfaces': [], 'issues': []} + route_files = find_files(['ip-route', 'ip_route', 'route-table', 'routes']) + for f in route_files[:3]: + content = read_file_content(f) + if content: + for line in content.split('\n'): + line = line.strip() + if not line: + continue + route_data['routes'].append(line) + if line.startswith('default') or 'default' in line: + route_data['defaultGateway'] = line + # Parse interfaces + iface_files = find_files(['ifconfig', 'ip-addr', 'ip_addr', 'interfaces']) + for f in iface_files[:2]: + content = read_file_content(f) + if content: + current_iface = None + for line in content.split('\n'): + if re.match(r'^\d+:\s+\S+', line) or re.match(r'^\S+:', line): + iface_match = re.search(r'(\S+?)[@:]', line) + if iface_match: + current_iface = iface_match.group(1) + if 'inet ' in line and current_iface: + ip_match = re.search(r'inet\s+(\S+)', line) + if ip_match: + route_data['interfaces'].append({'name': current_iface, 'ip': ip_match.group(1)}) + if not route_data['defaultGateway']: + route_data['issues'].append('No default gateway found') + issues_found.append({'section': 'routes', 'severity': 'critical', 'message': 'No default gateway'}) + route_data['routeCount'] = len(route_data['routes']) + route_data['routes'] = route_data['routes'][:50] + results['routes'] = route_data + + # ================================================================= + # DNS + # ================================================================= + if 'dns' in sections: + dns_data = {'resolv_conf': {}, 'nameservers': [], 'searchDomains': [], 'issues': []} + dns_files = find_files(['resolv', 'dns']) + for f in dns_files[:5]: + content = read_file_content(f) + if content: + if 'resolv' in f.lower(): + for line in content.split('\n'): + line = line.strip() + if line.startswith('nameserver'): + ns = line.split(None, 1)[1] if len(line.split()) > 1 else '' + dns_data['nameservers'].append(ns) + elif line.startswith('search'): + dns_data['searchDomains'] = line.split()[1:] + elif line.startswith('options'): + dns_data['resolv_conf']['options'] = line + dns_data['resolv_conf']['raw'] = content[:500] + if not dns_data['nameservers']: + dns_data['issues'].append('No nameservers in resolv.conf') + issues_found.append({'section': 'dns', 'severity': 'critical', 'message': 'No nameservers configured'}) + dns_data['_note'] = ( + "This is the NODE-LEVEL /etc/resolv.conf. It is expected to show VPC DNS " + "(e.g., 172.31.0.2 = VPC CIDR+2). Container DNS is configured separately " + "by Docker/ECS agent via --dns flags or task-level dnsServers config." + ) + results['dns'] = dns_data + + # ================================================================= + # ENI (Elastic Network Interfaces) + # ================================================================= + if 'eni' in sections: + eni_data = {'attachedENIs': [], 'eniCount': 0, 'issues': []} + try: + target_region = resolve_region(arguments, instance_id) + regional_ec2 = get_regional_client('ec2', target_region) + eni_resp = regional_ec2.describe_network_interfaces( + Filters=[{'Name': 'attachment.instance-id', 'Values': [instance_id]}] + ) + for eni in eni_resp.get('NetworkInterfaces', []): + eni_data['attachedENIs'].append({ + 'eniId': eni['NetworkInterfaceId'], + 'subnetId': eni.get('SubnetId'), + 'privateIp': eni.get('PrivateIpAddress'), + 'secondaryIps': [addr['PrivateIpAddress'] for addr in eni.get('PrivateIpAddresses', []) if not addr.get('Primary')], + 'status': eni.get('Status'), + 'description': eni.get('Description', '')[:100], + 'securityGroups': [sg['GroupId'] for sg in eni.get('Groups', [])], + }) + eni_data['eniCount'] = len(eni_data['attachedENIs']) + eni_data['totalSecondaryIPs'] = sum(len(e['secondaryIps']) for e in eni_data['attachedENIs']) + if eni_data['eniCount'] == 0: + eni_data['issues'].append('No ENIs attached — instance may be detached from VPC') + issues_found.append({'section': 'eni', 'severity': 'critical', 'message': 'No ENIs attached'}) + except Exception as e: + eni_data['issues'].append(f'Could not query ENI info: {str(e)}') + # Also check from bundle files + eni_files = find_files(['eni', 'network-interface', 'eth']) + for f in eni_files[:3]: + content = read_file_content(f, max_size=32768) + if content: + eni_data['bundleNetworkInfo'] = content[:2000] + break + results['eni'] = eni_data + + # ================================================================= + # SECURITY GROUPS + # ================================================================= + if 'security-groups' in sections: + sg_data = {'securityGroups': [], 'issues': []} + try: + target_region = resolve_region(arguments, instance_id) + regional_ec2 = get_regional_client('ec2', target_region) + inst_resp = regional_ec2.describe_instances(InstanceIds=[instance_id]) + sgs = [] + for res in inst_resp.get('Reservations', []): + for inst in res.get('Instances', []): + sgs = inst.get('SecurityGroups', []) + sg_ids = [sg['GroupId'] for sg in sgs] + if sg_ids: + sg_resp = regional_ec2.describe_security_groups(GroupIds=sg_ids) + for sg in sg_resp.get('SecurityGroups', []): + ingress_rules = [] + for rule in sg.get('IpPermissions', []): + for cidr in rule.get('IpRanges', []): + ingress_rules.append({ + 'protocol': rule.get('IpProtocol', 'all'), + 'fromPort': rule.get('FromPort'), + 'toPort': rule.get('ToPort'), + 'cidr': cidr.get('CidrIp'), + }) + for sg_ref in rule.get('UserIdGroupPairs', []): + ingress_rules.append({ + 'protocol': rule.get('IpProtocol', 'all'), + 'fromPort': rule.get('FromPort'), + 'toPort': rule.get('ToPort'), + 'sourceGroup': sg_ref.get('GroupId'), + }) + sg_data['securityGroups'].append({ + 'groupId': sg['GroupId'], + 'groupName': sg.get('GroupName', ''), + 'description': sg.get('Description', '')[:100], + 'ingressRuleCount': len(ingress_rules), + 'ingressRules': ingress_rules[:30], + }) + # Check for ECS-specific port requirements + all_ingress_ports = set() + for sg_info in sg_data['securityGroups']: + for rule in sg_info['ingressRules']: + fp = rule.get('fromPort') + tp = rule.get('toPort') + if fp and tp: + all_ingress_ports.update(range(fp, tp + 1)) + # ECS agent needs outbound (usually allowed by default), but check for common task ports + if not sg_ids: + sg_data['issues'].append('No security groups attached to instance') + issues_found.append({'section': 'security-groups', 'severity': 'critical', 'message': 'No security groups'}) + except Exception as e: + sg_data['issues'].append(f'Could not query security groups: {str(e)}') + results['security-groups'] = sg_data + + # ================================================================= + # ECS NETWORKING GUARDRAILS — prevent misinterpretation of findings + # ================================================================= + ecs_context = { + '_purpose': 'ECS-specific networking context to prevent misinterpretation of findings. ' + 'ECS container instances use Docker bridge/awsvpc networking which differs from ' + 'bare-metal Linux hosts. DO NOT diagnose ECS instances like standalone servers.', + 'guardrails': [], + } + + # Docker bridge networking context + docker_cfg = results.get('docker', {}).get('bridgeConfig', {}) + if docker_cfg: + ecs_context['guardrails'].append( + 'ECS EC2 launch type uses Docker bridge networking by default for tasks without ' + 'awsvpc network mode. In bridge mode, containers share the host ENI via port mappings ' + '(DNAT rules). Missing DOCKER iptables chains means Docker networking is broken, ' + 'but missing SNAT/MASQUERADE rules may be normal if tasks use awsvpc mode.' + ) + + # awsvpc mode context + eni_count = results.get('eni', {}).get('eniCount', 0) + if eni_count > 1: + ecs_context['guardrails'].append( + f'Multiple ENIs detected ({eni_count}): ECS tasks using awsvpc network mode get their own ENI. ' + 'Each task ENI has its own security group and private IP. This is NORMAL for awsvpc tasks. ' + 'Do NOT flag extra ENIs as anomalous on ECS instances running awsvpc tasks.' + ) + + # ANTI-HALLUCINATION: Docker iptables=false + if docker_cfg.get('iptables') is False: + ecs_context['guardrails'].append( + 'CRITICAL ANTI-HALLUCINATION: Docker daemon has "iptables": false in daemon.json. ' + 'This means Docker will NOT manage iptables rules for container networking. ' + 'This breaks bridge-mode container networking (no DNAT for port mappings, no MASQUERADE for egress). ' + 'However, awsvpc-mode tasks are UNAFFECTED because they use their own ENI directly. ' + 'Do NOT claim all container networking is broken — check the task network mode first.' + ) + + # ANTI-HALLUCINATION: Missing default gateway on awsvpc instances + if not results.get('routes', {}).get('defaultGateway') and eni_count > 1: + # Downgrade the "no default gateway" issue if awsvpc ENIs are present + for issue in issues_found: + if issue.get('section') == 'routes' and 'default gateway' in issue.get('message', '').lower(): + issue['severity'] = 'info' + issue['message'] += ' [May be EXPECTED: awsvpc tasks use per-task ENI routing]' + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: On ECS instances with awsvpc tasks, the host route table may appear ' + 'minimal. awsvpc tasks have their own network namespace with separate routing. ' + 'Check per-task ENI configuration before flagging host routing as broken.' + ) + + # ANTI-HALLUCINATION: ECS Agent connectivity vs container networking + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: ECS Agent connectivity issues (AGENT_DISCONNECTED) and container ' + 'networking issues are SEPARATE problems. The ECS Agent communicates with the ECS service ' + 'endpoint via HTTPS (port 443). Container networking depends on the task network mode ' + '(bridge/awsvpc/host). An agent disconnect does NOT mean container networking is broken, ' + 'and vice versa. Diagnose them independently.' + ) + + # ANTI-HALLUCINATION: Security group requirements differ by network mode + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: Security group requirements differ by ECS network mode. ' + 'Bridge mode: Only the instance security group matters — containers use host ports via DNAT. ' + 'awsvpc mode: Each task has its own security group — the instance SG does NOT apply to task traffic. ' + 'Host mode: Tasks share the instance security group directly. ' + 'Do NOT apply bridge-mode SG analysis to awsvpc tasks or vice versa.' + ) + + # ANTI-HALLUCINATION: DNS configuration + if results.get('dns', {}).get('nameservers'): + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: The node-level /etc/resolv.conf shows VPC DNS resolver ' + '(typically VPC CIDR + 2, e.g., 172.31.0.2). Container DNS is configured separately: ' + 'bridge mode uses Docker --dns flags (default: host DNS). ' + 'awsvpc mode uses the VPC DNS directly in the task network namespace. ' + 'Custom DNS can be set via task definition dnsServers/dnsSearchDomains. ' + 'Do NOT assume container DNS matches host DNS without checking the task network mode.' + ) + + # ANTI-HALLUCINATION: Conntrack exhaustion is kernel-level + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: "nf_conntrack: table full" is a kernel resource exhaustion issue. ' + 'Do NOT blame Docker or the ECS Agent. Fix: increase nf_conntrack_max via sysctl. ' + 'High-traffic instances (running many tasks with bridge networking) are most susceptible ' + 'because all containers share the host conntrack table in bridge mode.' + ) + + # ANTI-HALLUCINATION: ENI limits and task density + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: Each EC2 instance type has a maximum number of ENIs and IPs per ENI. ' + 'awsvpc tasks each consume one ENI (or one IP with ENI trunking enabled). ' + 'If ENI allocation fails, it may be an instance limit, NOT a subnet IP exhaustion issue. ' + 'Check the instance type ENI limit before blaming the subnet. ' + 'ENI trunking (account-level opt-in) allows more tasks per instance by sharing trunk ENIs.' + ) + + # ANTI-HALLUCINATION: ECS service connect vs service discovery + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: ECS Service Connect and ECS Service Discovery (Cloud Map) are ' + 'DIFFERENT features. Service Connect uses an Envoy sidecar proxy for service mesh. ' + 'Service Discovery uses Route 53 DNS (A/SRV records). They can coexist but have ' + 'different failure modes. DNS failures may be Service Discovery issues, while ' + 'connection proxy errors point to Service Connect. Do NOT conflate them.' + ) + + # ANTI-HALLUCINATION: Container health checks vs ELB health checks + ecs_context['guardrails'].append( + 'ANTI-HALLUCINATION: ECS container health checks (HEALTHCHECK in Dockerfile or ' + 'healthCheck in task definition) and ELB target health checks are INDEPENDENT. ' + 'A container can be healthy per its own health check but unhealthy per the ALB/NLB. ' + 'Common cause: security group not allowing health check traffic from the load balancer. ' + 'Diagnose each health check type separately.' + ) + + results['ecsContext'] = ecs_context + + # ================================================================= + # OVERALL SUMMARY + # ================================================================= + total_issues = len(issues_found) + critical_issues = sum(1 for i in issues_found if i.get('severity') == 'critical') + warning_issues = sum(1 for i in issues_found if i.get('severity') == 'warning') + + sections_with_data = sum(1 for s in sections if s in results and results[s]) + if sections_with_data >= 4 and critical_issues > 0: + confidence = 'high' + elif sections_with_data >= 2 and total_issues > 0: + confidence = 'medium' + elif sections_with_data >= 1: + confidence = 'low' + else: + confidence = 'none' + + gaps = [] + if not bundle_files: + gaps.append('No extracted bundle found — collect and wait for completion first') + sections_without_files = [s for s in sections if s not in section_file_map or not section_file_map.get(s)] + if sections_without_files: + gaps.append(f'No files found for sections: {", ".join(sections_without_files)}') + + # Build response (stale bundles are already rejected above, so this is always fresh) + response = { + 'instanceId': instance_id, + } + + # Bundle freshness info (always fresh since stale bundles are hard-blocked) + if bundle_age_minutes is not None: + response['bundleInfo'] = { + 'collectedAt': bundle_collected_at, + 'ageMinutes': bundle_age_minutes, + 'isStale': False, + } + + response['sections'] = sections + response['diagnostics'] = results + response['issuesSummary'] = { + 'total': total_issues, + 'critical': critical_issues, + 'warning': warning_issues, + 'issues': issues_found, + } + response['confidence'] = confidence + response['gaps'] = gaps + response['overallAssessment'] = _network_assessment(issues_found) + response['nextStep'] = 'Use search tool to dig deeper into specific networking errors, or correlate to build a timeline.' if issues_found else 'No networking issues detected in the bundle.' + response['recommendedSOPs'] = match_sops_for_issues(issues_found) + + return success_response(response) + + except Exception as e: + return error_response(500, f'network_diagnostics failed: {str(e)}') + + +def _network_assessment(issues: List[Dict]) -> str: + """Generate overall network health assessment.""" + if not issues: + return "HEALTHY — No networking issues detected in the log bundle." + critical = [i for i in issues if i.get('severity') == 'critical'] + if critical: + sections = set(i['section'] for i in critical) + return f"CRITICAL — {len(critical)} critical networking issues in: {', '.join(sections)}. Immediate investigation needed." + return f"WARNING — {len(issues)} non-critical networking issues found. Review recommended." + +def tcpdump_capture(arguments: Dict) -> Dict: + """ + Run tcpdump on an ECS container instance via SSM Run Command for a specified duration, + then upload the pcap file to S3. + + Inputs: + instanceId: EC2 instance ID (required) + durationSeconds: Capture duration in seconds (default: 120, max: 300) + interface: Network interface to capture on (default: "any") + filter: BPF filter expression (e.g., "port 443", "host 10.0.0.1") (optional) + taskId: ECS task ID to capture traffic for a specific container's network namespace (optional) + containerName: Container name within the task (optional, uses first container if omitted) + region: AWS region where the instance runs (optional, auto-detected) + commandId: If provided, polls status of an existing capture instead of starting a new one + + Returns: + commandId for async polling, or capture results if already complete + """ + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + + if not re.match(r'^i-[0-9a-f]{8,17}$', instance_id): + return error_response(400, f'Invalid instanceId format: {instance_id}') + + duration = int(arguments.get('durationSeconds', 120)) + if duration < 10 or duration > 300: + return error_response(400, 'durationSeconds must be between 10 and 300') + + interface = arguments.get('interface', 'any') + if not re.match(r'^[a-zA-Z0-9\-\.]+$', interface): + return error_response(400, f'Invalid interface name: {interface}') + + bpf_filter = arguments.get('filter', '') + if bpf_filter and re.search(r'[;&|`$(){}]', bpf_filter): + return error_response(400, 'filter contains invalid characters') + + ecs_task_id = arguments.get('taskId', '').strip() + container_name = arguments.get('containerName', '').strip() + + # Check if this is a status poll for an existing command + command_id = arguments.get('commandId') + if command_id: + return _poll_tcpdump_status(command_id, instance_id, arguments) + + target_region = resolve_region(arguments, instance_id) + + try: + regional_ssm = get_regional_client('ssm', target_region) + except Exception as e: + return error_response(500, f'Failed to create SSM client for region {target_region}: {str(e)}') + + # Build the shell script that runs tcpdump and uploads to S3 + timestamp = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') + s3_prefix = f"tcpdump/{instance_id}/{timestamp}" + s3_key = f"{s3_prefix}/capture.pcap" + s3_key_txt = f"{s3_prefix}/capture_summary.txt" + s3_key_stats = f"{s3_prefix}/capture_stats.json" + s3_uri = f"s3://{LOGS_BUCKET}/{s3_key}" + s3_uri_txt = f"s3://{LOGS_BUCKET}/{s3_key_txt}" + s3_uri_stats = f"s3://{LOGS_BUCKET}/{s3_key_stats}" + + filter_clause = f' {bpf_filter}' if bpf_filter else '' + + use_nsenter = bool(ecs_task_id) + ns_label = '' + if ecs_task_id: + ns_label = f' (ECS task {ecs_task_id}' + (f' container {container_name}' if container_name else '') + ' namespace)' + + script = f"""#!/bin/bash +set -euo pipefail + +PCAP_FILE="/tmp/tcpdump_capture_{timestamp}.pcap" +TXT_FILE="/tmp/tcpdump_summary_{timestamp}.txt" +STATS_FILE="/tmp/tcpdump_stats_{timestamp}.json" + +# Check if tcpdump is available +if ! command -v tcpdump &>/dev/null; then + echo "ERROR: tcpdump not found. Installing..." + if command -v yum &>/dev/null; then + yum install -y tcpdump 2>/dev/null || {{ echo "FATAL: Failed to install tcpdump"; exit 1; }} + elif command -v apt-get &>/dev/null; then + apt-get update -qq && apt-get install -y tcpdump 2>/dev/null || {{ echo "FATAL: Failed to install tcpdump"; exit 1; }} + else + echo "FATAL: No package manager found to install tcpdump" + exit 1 + fi +fi + +NSENTER_PREFIX="" +""" + + # Add ECS task container PID resolution when taskId is provided + if ecs_task_id: + container_filter = f'--filter "name={container_name}"' if container_name else '' + script += f""" +# === Resolve ECS task "{ecs_task_id}" to container PID === +echo "Resolving ECS task to container PID..." +TARGET_PID="" + +# Method 1: ECS agent introspection endpoint (works on all ECS-optimized AMIs) +echo "Trying ECS agent introspection endpoint..." +TASK_META=$(curl -s http://localhost:51678/v1/tasks 2>/dev/null || true) +if [ -n "$TASK_META" ] && command -v python3 &>/dev/null; then + TARGET_DOCKER_ID=$(python3 -c " +import sys, json +try: + data = json.loads('''$TASK_META''') + tasks = data if isinstance(data, list) else data.get('Tasks', []) + for task in tasks: + task_arn = task.get('Arn', '') + # Match by full ARN or just the task ID suffix + if '{ecs_task_id}' in task_arn: + containers = task.get('Containers', []) + for c in containers: + container_name_filter = '{container_name}' + if container_name_filter and c.get('Name') != container_name_filter: + continue + docker_id = c.get('DockerId', '') + if docker_id: + print(docker_id) + sys.exit(0) + # If no container_name filter matched, take the first container + if containers: + docker_id = containers[0].get('DockerId', '') + if docker_id: + print(docker_id) + sys.exit(0) +except Exception as e: + print('', file=sys.stderr) +" 2>/dev/null || true) + if [ -n "$TARGET_DOCKER_ID" ]; then + echo "Found Docker container ID from ECS introspection: $TARGET_DOCKER_ID" + fi +fi + +# Method 2: docker ps with ECS task label +if [ -z "$TARGET_DOCKER_ID" ] && command -v docker &>/dev/null; then + echo "Trying docker ps with task label..." + TARGET_DOCKER_ID=$(docker ps --filter "label=com.amazonaws.ecs.task-arn" {container_filter} --format '{{{{.ID}}}} {{{{.Labels}}}}' 2>/dev/null | grep '{ecs_task_id}' | head -1 | awk '{{print $1}}') + if [ -z "$TARGET_DOCKER_ID" ]; then + # Broader search + TARGET_DOCKER_ID=$(docker ps --format '{{{{.ID}}}} {{{{.Labels}}}}' 2>/dev/null | grep '{ecs_task_id}' | head -1 | awk '{{print $1}}') + fi + if [ -n "$TARGET_DOCKER_ID" ]; then + echo "Found Docker container ID from docker ps: $TARGET_DOCKER_ID" + fi +fi + +# Get PID from Docker container ID +if [ -n "$TARGET_DOCKER_ID" ] && command -v docker &>/dev/null; then + TARGET_PID=$(docker inspect --format '{{{{.State.Pid}}}}' "$TARGET_DOCKER_ID" 2>/dev/null || true) + echo "Docker container $TARGET_DOCKER_ID -> PID $TARGET_PID" +fi + +# Method 3: ctr (containerd native CLI — for newer ECS AMIs using containerd without docker) +if [ -z "$TARGET_PID" ] || [ "$TARGET_PID" = "0" ]; then + CTR="" + for p in /usr/local/bin/ctr /usr/bin/ctr $(which ctr 2>/dev/null); do + if [ -x "$p" ]; then CTR="$p"; break; fi + done + if [ -n "$CTR" ]; then + echo "Trying ctr ($CTR) to find ECS task container..." + # ECS uses the 'moby' namespace in containerd (or 'default') + # ctr containers ls does NOT show task IDs — must inspect each container's labels + for NS in moby default; do + for cid in $($CTR -n $NS containers ls -q 2>/dev/null); do + INFO=$($CTR -n $NS containers info "$cid" 2>/dev/null || true) + if echo "$INFO" | grep -q "{ecs_task_id}"; then + echo "ctr: found container $cid in namespace $NS" + CTR_PID=$($CTR -n $NS task ls 2>/dev/null | grep "$cid" | awk '{{print $2}}') + if [ -n "$CTR_PID" ] && [ "$CTR_PID" != "0" ] && [ -e "/proc/$CTR_PID/ns/net" ]; then + TARGET_PID="$CTR_PID" + echo "ctr: resolved pid=$TARGET_PID" + break 2 + fi + fi + done + done + [ -z "$TARGET_PID" ] && echo "ctr: could not resolve ECS task '{ecs_task_id}'" + fi +fi + +if [ -z "$TARGET_PID" ] || [ "$TARGET_PID" = "0" ]; then + echo "FATAL: Could not resolve ECS task {ecs_task_id} to a container PID on this instance." + echo "Ensure the task is running on instance {instance_id}." + echo "" + echo "Debug info:" + echo " ECS agent introspection:" + curl -s http://localhost:51678/v1/tasks 2>/dev/null | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) + tasks = d if isinstance(d, list) else d.get('Tasks', []) + for t in tasks[:10]: + print(f' Task: {{t.get(\"Arn\",\"?\")}} Status: {{t.get(\"DesiredStatus\",\"?\")}}') +except: print(' (could not parse)') +" 2>/dev/null || echo " (endpoint not available)" + echo " Running docker containers:" + docker ps --format 'table {{{{.ID}}}}\\t{{{{.Names}}}}\\t{{{{.Status}}}}' 2>/dev/null | head -10 || echo " (docker not available)" + echo " ctr binary: ${{CTR:-not found}}" + echo " containerd socket: $(ls -la /run/containerd/containerd.sock 2>/dev/null || echo 'not found')" + exit 1 +fi + +echo "Resolved ECS task {ecs_task_id} -> PID $TARGET_PID" +if [ ! -e "/proc/$TARGET_PID/ns/net" ]; then + if [ ! -d "/proc/$TARGET_PID" ]; then + echo "FATAL: PID $TARGET_PID does not exist in /proc (container may have exited)" + else + echo "FATAL: PID $TARGET_PID exists but /proc/$TARGET_PID/ns/net is missing" + fi + exit 1 +fi +NSENTER_PREFIX="nsenter -n -t $TARGET_PID " +""" + + script += f""" +echo "Starting tcpdump{ns_label} on interface '{interface}' for {duration}s..." +echo "Filter: '{bpf_filter or 'none'}'" +echo "Output: $PCAP_FILE" + +# Run tcpdump with timeout (with optional nsenter) +timeout {duration} ${{NSENTER_PREFIX}}tcpdump -i {interface} -w "$PCAP_FILE" -c 100000{filter_clause} 2>&1 || true + +# Verify capture file exists and has data +if [ ! -f "$PCAP_FILE" ]; then + echo "FATAL: Capture file not created" + exit 1 +fi + +FILE_SIZE=$(stat -c%s "$PCAP_FILE" 2>/dev/null || stat -f%z "$PCAP_FILE" 2>/dev/null || echo "0") +echo "Capture complete. File size: $FILE_SIZE bytes" + +if [ "$FILE_SIZE" -eq 0 ]; then + echo "WARNING: Capture file is empty — no packets matched the filter" +fi + +# Decode pcap to human-readable text summary (first 5000 packets max) +echo "Decoding pcap to text summary..." +# Use -c 5000 instead of piping through head to avoid SIGPIPE under set -euo pipefail +tcpdump -nn -r "$PCAP_FILE" -c 5000 > "$TXT_FILE" 2>/dev/null || true +TXT_SIZE=$(stat -c%s "$TXT_FILE" 2>/dev/null || stat -f%z "$TXT_FILE" 2>/dev/null || echo "0") +PACKET_COUNT=$(wc -l < "$TXT_FILE" 2>/dev/null || echo "0") +echo "Decoded $PACKET_COUNT packets to text (txt_size=$TXT_SIZE)" + +# If decode produced empty output, log diagnostics and retry +if [ "$TXT_SIZE" -eq 0 ] || [ "$PACKET_COUNT" -eq 0 ]; then + echo "WARNING: Text decode produced empty output." + echo "Pcap file details:" + ls -la "$PCAP_FILE" 2>/dev/null || true + file "$PCAP_FILE" 2>/dev/null || true + # Retry with verbose stderr to diagnose + echo "Retry with stderr:" + tcpdump -nn -r "$PCAP_FILE" -c 10 2>&1 || true +fi + +# Generate stats JSON with protocol breakdown and top talkers (using Python for valid JSON) +echo "Generating capture statistics..." +if command -v python3 &>/dev/null; then + python3 - "$PCAP_FILE" "$STATS_FILE" << 'PYSTATS' +import subprocess, json, sys, re +from collections import Counter + +pcap, out = sys.argv[1], sys.argv[2] + +def tcpdump_count(extra_args=None): + cmd = ['tcpdump', '-nn', '-r', pcap] + (extra_args or []) + try: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + return len([l for l in r.stdout.strip().splitlines() if l]) + except Exception: + return 0 + +def tcpdump_lines(): + try: + r = subprocess.run(['tcpdump', '-nn', '-r', pcap], capture_output=True, text=True, timeout=60) + return [l for l in r.stdout.strip().splitlines() if l] + except Exception: + return [] + +lines = tcpdump_lines() +total = len(lines) + +ip_port_re = re.compile(r'^(\\d+\\.\\d+\\.\\d+\\.\\d+)\\.\\d+$') +src_counter, dst_counter = Counter(), Counter() +for line in lines: + parts = line.split() + if len(parts) >= 5: + m = ip_port_re.match(parts[2]) + if m: + src_counter[m.group(1)] += 1 + dst_raw = parts[4].rstrip(':') + m = ip_port_re.match(dst_raw) + if m: + dst_counter[m.group(1)] += 1 + +retrans = sum(1 for l in lines if 'retransmit' in l.lower() or 'retrans' in l.lower()) + +stats = {{ + "totalPackets": total, + "protocols": {{ + "tcp": tcpdump_count(['tcp']), + "udp": tcpdump_count(['udp']), + "icmp": tcpdump_count(['icmp']), + "arp": tcpdump_count(['arp']), + }}, + "ports": {{ + "dns_53": tcpdump_count(['port', '53']), + "http_80": tcpdump_count(['port', '80']), + "https_443": tcpdump_count(['port', '443']), + }}, + "tcpFlags": {{ + "syn": tcpdump_count(['tcp[tcpflags] & (tcp-syn) != 0']), + "rst": tcpdump_count(['tcp[tcpflags] & (tcp-rst) != 0']), + }}, + "possibleRetransmits": retrans, + "topSourceIPs": dict(src_counter.most_common(10)), + "topDestinationIPs": dict(dst_counter.most_common(10)), +}} + +with open(out, 'w') as f: + json.dump(stats, f, indent=2) +print(f"Stats generated: {{total}} packets") +PYSTATS +else + TOTAL=$(tcpdump -nn -r "$PCAP_FILE" 2>/dev/null | wc -l) + TCP_COUNT=$(tcpdump -nn -r "$PCAP_FILE" tcp 2>/dev/null | wc -l) + UDP_COUNT=$(tcpdump -nn -r "$PCAP_FILE" udp 2>/dev/null | wc -l) + ICMP_COUNT=$(tcpdump -nn -r "$PCAP_FILE" icmp 2>/dev/null | wc -l) + echo '{{"totalPackets":'$TOTAL',"protocols":{{"tcp":'$TCP_COUNT',"udp":'$UDP_COUNT',"icmp":'$ICMP_COUNT'}},"topSourceIPs":{{}},"topDestinationIPs":{{}}}}' > "$STATS_FILE" +fi +if [ ! -f "$STATS_FILE" ] || [ ! -s "$STATS_FILE" ]; then + echo '{{"error":"stats generation failed"}}' > "$STATS_FILE" +fi + +# Upload all artifacts to S3 +set +e +UPLOAD_FAILURES=0 + +echo "Uploading pcap to {s3_uri}..." +aws s3 cp "$PCAP_FILE" "{s3_uri}" --no-progress 2>&1 +if [ $? -eq 0 ]; then echo "UPLOAD_PCAP=ok"; else echo "UPLOAD_PCAP=failed"; UPLOAD_FAILURES=$((UPLOAD_FAILURES + 1)); fi + +echo "Uploading text summary to {s3_uri_txt}..." +aws s3 cp "$TXT_FILE" "{s3_uri_txt}" --quiet 2>&1 +if [ $? -eq 0 ]; then echo "UPLOAD_TXT=ok"; else echo "UPLOAD_TXT=failed"; UPLOAD_FAILURES=$((UPLOAD_FAILURES + 1)); fi + +echo "Uploading stats to {s3_uri_stats}..." +aws s3 cp "$STATS_FILE" "{s3_uri_stats}" --quiet 2>&1 +if [ $? -eq 0 ]; then echo "UPLOAD_STATS=ok"; else echo "UPLOAD_STATS=failed"; UPLOAD_FAILURES=$((UPLOAD_FAILURES + 1)); fi + +if [ "$UPLOAD_FAILURES" -gt 0 ]; then + echo "WARNING: $UPLOAD_FAILURES of 3 uploads failed. Ensure the instance IAM role has s3:PutObject permission to {LOGS_BUCKET}." +fi + +echo "S3_KEY={s3_key}" +echo "S3_KEY_TXT={s3_key_txt}" +echo "S3_KEY_STATS={s3_key_stats}" +echo "FILE_SIZE=$FILE_SIZE" +echo "PACKET_COUNT=$PACKET_COUNT" +echo "UPLOAD_FAILURES=$UPLOAD_FAILURES" + +# Inline the decoded text and stats in stdout so Lambda can parse them even if S3 upload failed +echo "===INLINE_STATS_BEGIN===" +cat "$STATS_FILE" 2>/dev/null || echo '{{"error":"stats file missing"}}' +echo "" +echo "===INLINE_STATS_END===" +echo "===INLINE_TXT_BEGIN===" +head -500 "$TXT_FILE" 2>/dev/null || echo "(no decoded text)" +echo "" +echo "===INLINE_TXT_END===" + +# Cleanup +rm -f "$PCAP_FILE" "$TXT_FILE" "$STATS_FILE" 2>/dev/null || true +echo "DONE" +exit 0 +""" + + try: + response = regional_ssm.send_command( + InstanceIds=[instance_id], + DocumentName='AWS-RunShellScript', + Parameters={ + 'commands': [script], + 'executionTimeout': [str(duration + 120)], + }, + TimeoutSeconds=duration + 180, + Comment=f'tcpdump capture for ECS instance {instance_id} ({duration}s)', + ) + + cmd_id = response['Command']['CommandId'] + + # Store metadata for status polling + try: + s3_client.put_object( + Bucket=LOGS_BUCKET, + Key=f"tcpdump-commands/{cmd_id}.json", + Body=json.dumps({ + 'commandId': cmd_id, + 'instanceId': instance_id, + 'region': target_region, + 's3Key': s3_key, + 's3KeyTxt': s3_key_txt, + 's3KeyStats': s3_key_stats, + 's3Prefix': s3_prefix, + 'durationSeconds': duration, + 'interface': interface, + 'filter': bpf_filter, + 'taskId': ecs_task_id or None, + 'containerName': container_name or None, + 'startedAt': timestamp, + }), + ) + except Exception: + pass # Non-fatal + + return success_response({ + 'message': f'tcpdump capture started ({duration}s){ns_label}', + 'commandId': cmd_id, + 'instanceId': instance_id, + 'region': target_region, + 'durationSeconds': duration, + 'interface': interface, + 'filter': bpf_filter or 'none', + 'taskId': ecs_task_id or None, + 'containerName': container_name or None, + 's3Key': s3_key, + 's3KeyTxt': s3_key_txt, + 's3KeyStats': s3_key_stats, + 's3Bucket': LOGS_BUCKET, + 'estimatedCompletionSeconds': duration + 30, + 'nextStep': f'Poll with tcpdump_capture(commandId="{cmd_id}", instanceId="{instance_id}") after ~{duration + 30}s. Once complete, use tcpdump_analyze(instanceId="{instance_id}", commandId="{cmd_id}") to read the decoded packet summary.', + 'task': { + 'taskId': cmd_id, + 'state': 'running', + 'message': f'tcpdump running for {duration}s on {interface}', + 'progress': 0, + }, + }) + + except Exception as e: + return error_response(500, f'Failed to start tcpdump: {str(e)}') + + +def _poll_tcpdump_status(command_id: str, instance_id: str, arguments: Dict) -> Dict: + """Poll the status of a tcpdump SSM Run Command.""" + + # Try to load stored metadata + metadata = {} + try: + meta_resp = s3_client.get_object( + Bucket=LOGS_BUCKET, + Key=f"tcpdump-commands/{command_id}.json", + ) + metadata = json.loads(meta_resp['Body'].read().decode('utf-8')) + except Exception: + pass + + target_region = metadata.get('region') or resolve_region(arguments, instance_id) + + try: + regional_ssm = get_regional_client('ssm', target_region) + result = regional_ssm.get_command_invocation( + CommandId=command_id, + InstanceId=instance_id, + ) + + status = result.get('Status', 'Unknown') + stdout = result.get('StandardOutputContent', '') + stderr = result.get('StandardErrorContent', '') + + # Parse output for S3 key and file size + s3_key = metadata.get('s3Key', '') + s3_key_txt = metadata.get('s3KeyTxt', '') + s3_key_stats = metadata.get('s3KeyStats', '') + file_size = 0 + packet_count = 0 + for line in stdout.split('\n'): + if line.startswith('S3_KEY='): + s3_key = line.split('=', 1)[1].strip() + elif line.startswith('S3_KEY_TXT='): + s3_key_txt = line.split('=', 1)[1].strip() + elif line.startswith('S3_KEY_STATS='): + s3_key_stats = line.split('=', 1)[1].strip() + elif line.startswith('FILE_SIZE='): + try: file_size = int(line.split('=', 1)[1].strip()) + except ValueError: pass + elif line.startswith('PACKET_COUNT='): + try: packet_count = int(line.split('=', 1)[1].strip()) + except ValueError: pass + + capture_completed = ( + 'Capture complete.' in stdout + or 'DONE' in stdout + or 'UPLOAD_PCAP=ok' in stdout + or 'UPLOAD_PCAP=failed' in stdout + or ('FILE_SIZE=' in stdout and 'S3_KEY=' in stdout) + ) + if 'FATAL:' in stdout: + capture_completed = False + + upload_failures = 0 + for line in stdout.split('\n'): + if line.startswith('UPLOAD_FAILURES='): + try: upload_failures = int(line.split('=', 1)[1].strip()) + except ValueError: pass + + # Extract inline stats and text from stdout + inline_stats = {} + inline_txt_lines = [] + if '===INLINE_STATS_BEGIN===' in stdout: + try: + stats_block = stdout.split('===INLINE_STATS_BEGIN===')[1].split('===INLINE_STATS_END===')[0].strip() + if stats_block: + inline_stats = json.loads(stats_block) + except (IndexError, json.JSONDecodeError): + pass + if '===INLINE_TXT_BEGIN===' in stdout: + try: + txt_block = stdout.split('===INLINE_TXT_BEGIN===')[1].split('===INLINE_TXT_END===')[0].strip() + if txt_block: + inline_txt_lines = txt_block.split('\n') + except IndexError: + pass + + if status in ('Success',) or (capture_completed and status == 'Failed'): + presigned_url = '' + try: + presigned_url = s3_client.generate_presigned_url( + 'get_object', + Params={'Bucket': LOGS_BUCKET, 'Key': s3_key}, + ExpiresIn=3600, + ) + except Exception: + pass + + # If S3 uploads failed from node, store inline data from Lambda + if (upload_failures > 0 or (status == 'Failed' and capture_completed)) and inline_stats: + try: + s3_client.put_object(Bucket=LOGS_BUCKET, Key=s3_key_stats, + Body=json.dumps(inline_stats, indent=2), ContentType='application/json') + except Exception: + pass + if (upload_failures > 0 or (status == 'Failed' and capture_completed)) and inline_txt_lines: + try: + s3_client.put_object(Bucket=LOGS_BUCKET, Key=s3_key_txt, + Body='\n'.join(inline_txt_lines), ContentType='text/plain') + except Exception: + pass + + warnings = [] + actual_failures = 0 + if upload_failures > 0 or (status == 'Failed' and capture_completed): + actual_failures = upload_failures if upload_failures > 0 else 3 + warnings.append(f'{actual_failures} of 3 S3 uploads failed from the node. Stats and text recovered via Lambda.') + if actual_failures == 3: + warnings.append('pcap file was NOT uploaded — add S3 PutObject permission to the instance IAM role.') + + response_data = { + 'commandId': command_id, + 'instanceId': instance_id, + 'status': 'completed' if not warnings else 'completed_with_warnings', + 's3Key': s3_key, 's3KeyTxt': s3_key_txt, 's3KeyStats': s3_key_stats, + 's3Bucket': LOGS_BUCKET, + 'fileSizeBytes': file_size, 'fileSizeHuman': format_bytes(file_size), + 'packetCount': packet_count, + 'presignedUrl': presigned_url, 'presignedUrlExpiresIn': '1 hour', + 'output': stdout[-2000:] if len(stdout) > 2000 else stdout, + 'nextStep': f'Use tcpdump_analyze(instanceId="{instance_id}", commandId="{command_id}") to read decoded packet data and statistics.', + 'task': {'taskId': command_id, 'state': 'completed', + 'message': f'tcpdump capture completed' + (f' ({actual_failures} S3 uploads failed — recovered via Lambda)' if warnings else f' — uploaded to s3://{LOGS_BUCKET}/{s3_key}'), + 'progress': 100}, + } + if warnings: + response_data['warnings'] = warnings + if inline_stats: + response_data['inlineStats'] = inline_stats + return success_response(response_data) + + elif status in ('InProgress', 'Pending', 'Delayed'): + elapsed = 0 + duration = metadata.get('durationSeconds', 120) + if metadata.get('startedAt'): + try: + start_dt = datetime.strptime(metadata['startedAt'], '%Y%m%dT%H%M%SZ') + elapsed = (datetime.utcnow() - start_dt).total_seconds() + except Exception: + pass + progress = min(95, int((elapsed / (duration + 30)) * 100)) if duration else 0 + + return success_response({ + 'commandId': command_id, + 'instanceId': instance_id, + 'status': 'in_progress', + 'elapsedSeconds': int(elapsed), + 'durationSeconds': duration, + 'nextStep': 'Poll again in 15-30 seconds', + 'task': {'taskId': command_id, 'state': 'running', + 'message': f'tcpdump capture in progress ({int(elapsed)}s / {duration}s)', + 'progress': progress}, + }) + + else: + return error_response(500, f'tcpdump command {status}', { + 'commandId': command_id, 'status': status, + 'stdout': stdout[-2000:] if stdout else '', + 'stderr': stderr[-2000:] if stderr else '', + 'statusDetails': result.get('StatusDetails', ''), + 'task': {'taskId': command_id, 'state': 'failed', + 'message': f'tcpdump command {status}: {stderr[:200] if stderr else "unknown error"}', + 'progress': 0}, + }) + + except Exception as e: + return error_response(500, f'Failed to poll tcpdump status: {str(e)}') + + +def tcpdump_analyze(arguments: Dict) -> Dict: + """ + Read and analyze a completed tcpdump capture from S3. + Returns decoded packet text, protocol statistics, and top talkers. + + Inputs: + instanceId: EC2 instance ID (required) + commandId: SSM Command ID from tcpdump_capture (optional — finds latest if omitted) + section: "summary" (decoded packets), "stats" (protocol breakdown), "all" (default) + maxPackets: Max decoded packet lines to return (default: 500, max: 3000) + filter: Text filter to apply on decoded lines (e.g., "SYN", "RST", "10.0.0.5") + + Returns: + Decoded packet text, protocol stats, top talkers, and anomaly indicators + """ + instance_id = arguments.get('instanceId') + if not instance_id: + return error_response(400, 'instanceId is required') + + command_id = arguments.get('commandId') + section = arguments.get('section', 'all') + max_packets = min(int(arguments.get('maxPackets', 500)), 3000) + text_filter = arguments.get('filter', '') + + # Find the capture metadata + metadata = {} + if command_id: + try: + meta_resp = s3_client.get_object( + Bucket=LOGS_BUCKET, + Key=f"tcpdump-commands/{command_id}.json", + ) + metadata = json.loads(meta_resp['Body'].read().decode('utf-8')) + except Exception: + pass + + # If no commandId, find the latest capture for this instance + if not metadata: + try: + list_resp = safe_s3_list_raw(LOGS_BUCKET, "tcpdump-commands/", max_keys=200) + candidates = [] + for obj in list_resp: + try: + r = s3_client.get_object(Bucket=LOGS_BUCKET, Key=obj['Key']) + m = json.loads(r['Body'].read().decode('utf-8')) + if m.get('instanceId') == instance_id: + candidates.append(m) + except Exception: + continue + if candidates: + candidates.sort(key=lambda x: x.get('startedAt', ''), reverse=True) + metadata = candidates[0] + except Exception: + pass + + if not metadata: + return error_response(404, f'No tcpdump capture found for {instance_id}. Run tcpdump_capture first.') + + s3_key_txt = metadata.get('s3KeyTxt', '') + s3_key_stats = metadata.get('s3KeyStats', '') + s3_key_pcap = metadata.get('s3Key', '') + + results = { + 'instanceId': instance_id, + 'commandId': metadata.get('commandId', command_id or 'unknown'), + 'captureInfo': { + 'interface': metadata.get('interface', 'unknown'), + 'filter': metadata.get('filter', 'none'), + 'durationSeconds': metadata.get('durationSeconds', 0), + 'startedAt': metadata.get('startedAt', 'unknown'), + 'taskId': metadata.get('taskId'), + 'containerName': metadata.get('containerName'), + }, + } + + # Read stats + if section in ('stats', 'all'): + stats = {} + if s3_key_stats: + try: + raw = safe_s3_read_raw(LOGS_BUCKET, s3_key_stats) + if raw: + stats = json.loads(raw.decode('utf-8') if isinstance(raw, bytes) else raw) + except (json.JSONDecodeError, Exception): + stats = {'error': 'Could not parse stats JSON'} + else: + stats = {'error': 'No stats file found — capture may still be in progress'} + + results['statistics'] = stats + + # Anomaly detection + anomalies = [] + if isinstance(stats, dict) and 'totalPackets' in stats: + total = stats.get('totalPackets', 0) + rst_count = stats.get('tcpFlags', {}).get('rst', 0) + syn_count = stats.get('tcpFlags', {}).get('syn', 0) + retrans = stats.get('possibleRetransmits', 0) + + if total > 0: + rst_pct = (rst_count / total) * 100 + if rst_pct > 5: + anomalies.append({ + 'type': 'high_rst_rate', + 'severity': 'warning' if rst_pct < 15 else 'critical', + 'message': f'{rst_pct:.1f}% of packets are TCP RST ({rst_count}/{total}) — possible connection rejection or firewall drops', + }) + if retrans > 0: + retrans_pct = (retrans / total) * 100 + anomalies.append({ + 'type': 'retransmissions', + 'severity': 'warning' if retrans_pct < 5 else 'critical', + 'message': f'{retrans} possible retransmissions ({retrans_pct:.1f}%) — network congestion or packet loss', + }) + if syn_count > 0 and rst_count > syn_count * 0.5: + anomalies.append({ + 'type': 'syn_rst_ratio', + 'severity': 'warning', + 'message': f'High RST-to-SYN ratio ({rst_count} RST vs {syn_count} SYN) — many connections being refused', + }) + icmp_count = stats.get('protocols', {}).get('icmp', 0) + if icmp_count > total * 0.1: + anomalies.append({ + 'type': 'high_icmp', + 'severity': 'info', + 'message': f'{icmp_count} ICMP packets ({(icmp_count/total)*100:.1f}%) — possible ping flood or unreachable destinations', + }) + results['anomalies'] = anomalies + + # Read decoded text summary + if section in ('summary', 'all'): + decoded_lines = [] + if s3_key_txt: + try: + raw = safe_s3_read_raw(LOGS_BUCKET, s3_key_txt) + if raw: + content = raw.decode('utf-8') if isinstance(raw, bytes) else raw + all_lines = content.split('\n') + + if text_filter: + pattern = re.compile(re.escape(text_filter), re.IGNORECASE) + all_lines = [l for l in all_lines if pattern.search(l)] + + total_lines = len(all_lines) + decoded_lines = all_lines[:max_packets] + + results['decodedPackets'] = { + 'lines': decoded_lines, + 'totalPackets': total_lines, + 'returnedPackets': len(decoded_lines), + 'truncated': total_lines > max_packets, + 'filter': text_filter or 'none', + } + else: + results['decodedPackets'] = {'error': 'Text summary file is empty or unreadable'} + except Exception as e: + results['decodedPackets'] = {'error': f'Failed to read text summary: {str(e)}'} + else: + results['decodedPackets'] = {'error': 'No text summary file found — capture may still be in progress'} + + # Presigned URL for pcap download + if s3_key_pcap: + try: + results['pcapDownloadUrl'] = s3_client.generate_presigned_url( + 'get_object', + Params={'Bucket': LOGS_BUCKET, 'Key': s3_key_pcap}, + ExpiresIn=3600, + ) + results['pcapDownloadUrlExpiresIn'] = '1 hour' + except Exception: + pass + + results['s3Bucket'] = LOGS_BUCKET + results['s3KeyPcap'] = s3_key_pcap + results['s3KeyTxt'] = s3_key_txt + results['s3KeyStats'] = s3_key_stats + + return success_response(results) diff --git a/mcp/aws-ecs-instance-log-mcp/tests/__init__.py b/mcp/aws-ecs-instance-log-mcp/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mcp/aws-ecs-instance-log-mcp/tests/conftest.py b/mcp/aws-ecs-instance-log-mcp/tests/conftest.py new file mode 100644 index 0000000..d2f58a6 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/tests/conftest.py @@ -0,0 +1,12 @@ +""" +Shared pytest configuration for ECS Instance Log MCP tests. +Sets required environment variables before any test module imports the Lambda. +""" +import os + +# These must be set before boto3 clients are created at module-level in the Lambda +os.environ.setdefault('AWS_DEFAULT_REGION', 'us-east-1') +os.environ.setdefault('AWS_REGION', 'us-east-1') +os.environ.setdefault('LOGS_BUCKET_NAME', 'test-bucket') +os.environ.setdefault('SSM_AUTOMATION_ROLE_ARN', 'arn:aws:iam::123456789012:role/test') +os.environ.setdefault('ALLOWED_REGIONS', 'us-east-1,us-west-2') diff --git a/mcp/aws-ecs-instance-log-mcp/tests/test_instance_validation.py b/mcp/aws-ecs-instance-log-mcp/tests/test_instance_validation.py new file mode 100644 index 0000000..83a538d --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/tests/test_instance_validation.py @@ -0,0 +1,123 @@ +""" +Property-based tests for Lambda ECS instance validation (Property 7). +Tests validate_ecs_instance() correctness with mocked EC2/ECS responses. +Mirrors eks-node-log-mcp/tests/test_instance_validation.py for ECS. +""" +import json +import pytest +from hypothesis import given, strategies as st, settings, assume + + +# ============================================================================= +# Property 7: Lambda ECS instance validation correctness +# ============================================================================= + +tag_key = st.text(min_size=1, max_size=50, alphabet='abcdefghijklmnopqrstuvwxyz./-:') +tag_value = st.text(min_size=0, max_size=20) +tag_entry = st.fixed_dictionaries({'Key': tag_key, 'Value': tag_value}) +tag_list = st.lists(tag_entry, min_size=0, max_size=10) + +ecs_cluster_name = st.text(min_size=1, max_size=30, alphabet='abcdefghijklmnopqrstuvwxyz0123456789-') + + +def has_ecs_tag(tags): + """Check if any tag indicates ECS membership.""" + for t in tags: + if t['Key'] == 'aws:ecs:clusterName': + return True + if t['Key'].startswith('ecs:cluster'): + return True + if t['Key'] == 'Name' and 'ecs' in t.get('Value', '').lower(): + return True + return False + + +def simulate_validate_ecs_instance(tags): + """Simulate the validate_ecs_instance tag-check logic (without ECS API fallback).""" + for tag in tags: + if tag['Key'] == 'aws:ecs:clusterName': + return None # Valid + if tag['Key'].startswith('ecs:cluster'): + return None + if tag['Key'] == 'Name' and 'ecs' in tag.get('Value', '').lower(): + return None + return {'statusCode': 403, 'body': json.dumps({'error': 'Not an ECS instance'})} + + +@given(tags=tag_list) +@settings(max_examples=100) +def test_ecs_validation_correctness(tags): + """validate_ecs_instance returns None iff ECS tag exists.""" + result = simulate_validate_ecs_instance(tags) + if has_ecs_tag(tags): + assert result is None + else: + assert result is not None + assert result['statusCode'] == 403 + + +@given(cluster_name=ecs_cluster_name) +@settings(max_examples=50) +def test_ecs_validation_accepts_tagged_instance(cluster_name): + """Instance with aws:ecs:clusterName tag is always accepted.""" + tags = [ + {'Key': 'aws:ecs:clusterName', 'Value': cluster_name}, + {'Key': 'Name', 'Value': 'my-ecs-instance'}, + ] + result = simulate_validate_ecs_instance(tags) + assert result is None + + +def test_ecs_validation_accepts_ecs_cluster_tag(): + """Instance with ecs:cluster-name tag is accepted.""" + tags = [ + {'Key': 'ecs:cluster-name', 'Value': 'my-cluster'}, + ] + result = simulate_validate_ecs_instance(tags) + assert result is None + + +def test_ecs_validation_accepts_ecs_name_tag(): + """Instance with 'ecs' in Name tag is accepted.""" + tags = [ + {'Key': 'Name', 'Value': 'my-ecs-worker-node'}, + ] + result = simulate_validate_ecs_instance(tags) + assert result is None + + +def test_ecs_validation_rejects_untagged_instance(): + """Instance without any ECS tag is rejected.""" + tags = [ + {'Key': 'Name', 'Value': 'my-instance'}, + {'Key': 'Environment', 'Value': 'production'}, + {'Key': 'kubernetes.io/cluster/my-cluster', 'Value': 'owned'}, # EKS tag, not ECS + ] + result = simulate_validate_ecs_instance(tags) + assert result is not None + assert result['statusCode'] == 403 + + +def test_ecs_validation_rejects_empty_tags(): + """Instance with no tags is rejected.""" + result = simulate_validate_ecs_instance([]) + assert result is not None + assert result['statusCode'] == 403 + + +@given(tags=st.lists( + st.fixed_dictionaries({ + 'Key': st.text(min_size=1, max_size=30, alphabet='abcdefghijklmnopqrstuvwxyz'), + 'Value': tag_value, + }), + min_size=0, max_size=5, +)) +@settings(max_examples=50) +def test_ecs_validation_rejects_non_ecs_tags(tags): + """Tags that don't match ECS patterns are always rejected.""" + # These tags use only lowercase alpha keys — can never match aws:ecs:clusterName or ecs:cluster* + # and Name values won't contain 'ecs' since values are also lowercase alpha + assume(not has_ecs_tag(tags)) + result = simulate_validate_ecs_instance(tags) + assert result is not None + assert result['statusCode'] == 403 diff --git a/mcp/aws-ecs-instance-log-mcp/tests/test_region_validation.py b/mcp/aws-ecs-instance-log-mcp/tests/test_region_validation.py new file mode 100644 index 0000000..3064b36 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/tests/test_region_validation.py @@ -0,0 +1,111 @@ +""" +Property-based tests for Lambda region validation (Properties 5, 6). +Tests ALLOWED_REGIONS parsing and validate_region() correctness. +Mirrors eks-node-log-mcp/tests/test_region_validation.py for ECS. +""" +import os +import sys +import json +import pytest +from hypothesis import given, strategies as st, settings, assume + +# Strategy: generate comma-separated region strings with edge cases +region_str = st.from_regex(r'[a-z]{2}-[a-z]+-[0-9]', fullmatch=True) +region_list = st.lists(region_str, min_size=0, max_size=5) + + +# ============================================================================= +# Property 5: Lambda ALLOWED_REGIONS parsing +# ============================================================================= + +@given(regions=region_list) +@settings(max_examples=100) +def test_allowed_regions_parsing_clean(regions): + """Clean comma-separated regions parse to exact set.""" + env_val = ','.join(regions) + parsed = set( + r.strip() for r in env_val.split(',') + if r.strip() + ) + expected = set(r for r in regions if r.strip()) + assert parsed == expected + + +@given(regions=st.lists(region_str, min_size=1, max_size=4)) +@settings(max_examples=100) +def test_allowed_regions_parsing_with_whitespace(regions): + """Regions with extra whitespace and empty segments parse correctly.""" + parts = [] + for r in regions: + parts.append(f' {r} ') + parts.append('') + parts.insert(0, '') + env_val = ','.join(parts) + + parsed = set( + r.strip() for r in env_val.split(',') + if r.strip() + ) + assert parsed == set(regions) + + +def test_allowed_regions_empty_defaults_to_aws_region(): + """Empty ALLOWED_REGIONS defaults to {AWS_REGION}.""" + parsed = set( + r.strip() for r in ''.split(',') + if r.strip() + ) or {'us-east-1'} + assert parsed == {'us-east-1'} + + +def test_allowed_regions_only_commas(): + """String of only commas produces empty set, falls back to default.""" + parsed = set( + r.strip() for r in ',,,'.split(',') + if r.strip() + ) or {'us-west-2'} + assert parsed == {'us-west-2'} + + +# ============================================================================= +# Property 6: Lambda region validation correctness +# ============================================================================= + +@given( + region=region_str, + allowed=st.frozensets(region_str, min_size=1, max_size=5), +) +@settings(max_examples=100) +def test_validate_region_membership(region, allowed): + """validate_region returns None iff region is in allowed set.""" + if region in allowed: + result = None + else: + result = {'statusCode': 403} + + if region in allowed: + assert result is None + else: + assert result is not None + assert result['statusCode'] == 403 + + +@given(allowed=st.frozensets(region_str, min_size=1, max_size=5)) +@settings(max_examples=100) +def test_validate_region_always_accepts_member(allowed): + """Any region that is a member of the allowed set is accepted.""" + for r in allowed: + assert r in allowed + + +@given( + region=region_str, + allowed=st.frozensets(region_str, min_size=1, max_size=3), +) +@settings(max_examples=100) +def test_validate_region_rejects_non_member(region, allowed): + """A region not in the allowed set is rejected with 403.""" + assume(region not in allowed) + result = {'statusCode': 403, 'body': json.dumps({'error': f"Region '{region}' is not permitted"})} + assert result['statusCode'] == 403 + assert region in result['body'] diff --git a/mcp/aws-ecs-instance-log-mcp/tests/test_tool_validation_wiring.py b/mcp/aws-ecs-instance-log-mcp/tests/test_tool_validation_wiring.py new file mode 100644 index 0000000..4222f3b --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/tests/test_tool_validation_wiring.py @@ -0,0 +1,135 @@ +""" +Unit tests for MCP tool function validation wiring. +Verifies that tool functions call region and ECS instance validation. +Mirrors eks-node-log-mcp/tests/test_tool_validation_wiring.py for ECS. +""" +import os +import sys +import json +import pytest +from unittest.mock import patch, MagicMock + +# Set required env vars before importing the module +os.environ.setdefault('LOGS_BUCKET_NAME', 'test-bucket') +os.environ.setdefault('SSM_AUTOMATION_ROLE_ARN', 'arn:aws:iam::123456789012:role/test') +os.environ.setdefault('AWS_REGION', 'us-east-1') +os.environ.setdefault('AWS_DEFAULT_REGION', 'us-east-1') +os.environ.setdefault('ALLOWED_REGIONS', 'us-east-1,us-west-2') + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src', 'lambda')) + + +class TestValidationFunctionsExist: + """Tests that all validation functions are defined in the module.""" + + def test_validation_functions_exist(self): + mod = __import__('ecs-log-automation') + assert callable(getattr(mod, 'validate_region', None)) + assert callable(getattr(mod, 'resolve_and_validate_region', None)) + assert callable(getattr(mod, 'validate_ecs_instance', None)) + assert callable(getattr(mod, '_parse_presigned_url_expiration', None)) + assert callable(getattr(mod, 'TimeWindowResolver', None)) or hasattr(mod, 'TimeWindowResolver') + + +class TestRegionValidation: + """Tests that validate_region works correctly.""" + + def test_validate_region_accepts_allowed(self): + mod = __import__('ecs-log-automation') + result = mod.validate_region('us-east-1') + assert result is None + + def test_validate_region_rejects_disallowed(self): + mod = __import__('ecs-log-automation') + result = mod.validate_region('ap-south-1') + assert result is not None + assert result['statusCode'] == 403 + body = json.loads(result['body']) + assert 'not permitted' in body['error'] + + def test_allowed_regions_parsed_from_env(self): + mod = __import__('ecs-log-automation') + assert 'us-east-1' in mod.ALLOWED_REGIONS + assert 'us-west-2' in mod.ALLOWED_REGIONS + + +class TestPresignedUrlExpiration: + """Tests that presigned URL expiration is configurable.""" + + def test_default_expiration_is_900(self): + mod = __import__('ecs-log-automation') + # Default is 900 for ECS (15 minutes) + assert mod.PRESIGNED_URL_EXPIRATION == 900 + + def test_parse_valid_value(self): + mod = __import__('ecs-log-automation') + with patch.dict(os.environ, {'PRESIGNED_URL_EXPIRATION_SECONDS': '120'}): + result = mod._parse_presigned_url_expiration() + assert result == 120 + + def test_parse_invalid_value_defaults(self): + mod = __import__('ecs-log-automation') + with patch.dict(os.environ, {'PRESIGNED_URL_EXPIRATION_SECONDS': 'abc'}): + result = mod._parse_presigned_url_expiration() + assert result == 900 + + def test_parse_zero_defaults(self): + mod = __import__('ecs-log-automation') + with patch.dict(os.environ, {'PRESIGNED_URL_EXPIRATION_SECONDS': '0'}): + result = mod._parse_presigned_url_expiration() + assert result == 900 + + def test_parse_negative_defaults(self): + mod = __import__('ecs-log-automation') + with patch.dict(os.environ, {'PRESIGNED_URL_EXPIRATION_SECONDS': '-5'}): + result = mod._parse_presigned_url_expiration() + assert result == 900 + + +class TestTimeWindowResolver: + """Tests that TimeWindowResolver works correctly.""" + + def test_default_window_is_10_minutes(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({}) + assert window['resolution_reason'].startswith('no incident time') + delta = window['window_end_utc'] - window['window_start_utc'] + assert 9 * 60 <= delta.total_seconds() <= 11 * 60 # ~10 minutes + + def test_explicit_window(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({ + 'start_time': '2025-01-15T10:00:00Z', + 'end_time': '2025-01-15T10:30:00Z', + }) + assert 'explicit' in window['resolution_reason'] + delta = window['window_end_utc'] - window['window_start_utc'] + assert delta.total_seconds() == 30 * 60 + + def test_incident_time_padding(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({ + 'incident_time': '2025-01-15T10:00:00Z', + }) + assert 'padding' in window['resolution_reason'] + delta = window['window_end_utc'] - window['window_start_utc'] + assert delta.total_seconds() == 10 * 60 # +/- 5 min = 10 min + + def test_max_window_clamped(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({ + 'start_time': '2025-01-01T00:00:00Z', + 'end_time': '2025-01-10T00:00:00Z', + }) + assert 'clamped' in window['resolution_reason'] + delta = window['window_end_utc'] - window['window_start_utc'] + assert delta.total_seconds() <= 24 * 3600 + + def test_swapped_start_end(self): + mod = __import__('ecs-log-automation') + window = mod.TimeWindowResolver.resolve({ + 'start_time': '2025-01-15T11:00:00Z', + 'end_time': '2025-01-15T10:00:00Z', + }) + assert 'swapped' in window['resolution_reason'] + assert window['window_start_utc'] < window['window_end_utc'] diff --git a/mcp/aws-ecs-instance-log-mcp/tsconfig.json b/mcp/aws-ecs-instance-log-mcp/tsconfig.json new file mode 100644 index 0000000..22a6ee1 --- /dev/null +++ b/mcp/aws-ecs-instance-log-mcp/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020"], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "outDir": "./lib" + }, + "include": ["src/**/*", "bin/**/*"], + "exclude": ["node_modules", "cdk.out"] +}