diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 4cf43fcbf..12224bc19 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -11,9 +11,32 @@ jobs: matrix: mode: ["server", "library"] environment: ["ci"] - e2e_group: [1, 2, 3] + # Config-aligned shards (@cfg_*). Packed small groups keep job count reasonable + # while avoiding mixed-config restarts inside large suites. + shard: + - name: default + tags: "not @skip and @cfg_default" + - name: authorized + tags: "not @skip and @cfg_authorized" + - name: mcp + tags: "not @skip and (@cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth)" + - name: rbac + tags: "not @skip and @cfg_rbac" + - name: skills + tags: "not @skip and (@cfg_skills or @cfg_skills_directory)" + - name: other + tags: "not @skip and (@cfg_rh_identity or @cfg_negative or @cfg_byok_pdf or @cfg_degraded or @cfg_unified)" + # Server-only; listed in shard (not matrix.include) so it expands with + # mode=server before any library jobs. include would append after library. + - name: tls + tags: "not @skip and @cfg_tls" + exclude: + - mode: library + shard: + name: tls + tags: "not @skip and @cfg_tls" - name: "E2E: ${{ matrix.mode }} mode / ${{ matrix.environment }} / group ${{ matrix.e2e_group }}" + name: "E2E: ${{ matrix.mode }} / ${{ matrix.environment }} / ${{ matrix.shard.name }}" env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -43,7 +66,7 @@ jobs: persist-credentials: ${{ github.event_name != 'pull_request_target' }} # Fetch submodules (required for lightspeed-providers) - submodules: ‘recursive’ + submodules: "recursive" - name: Verify actual git checkout result run: | @@ -112,7 +135,7 @@ jobs: echo "=== Configuration Summary ===" echo "Deployment mode: ${{ matrix.mode }}" echo "Environment: ${{ matrix.environment }}" - echo "E2E shard (Makefile test-e2e-tagged): @e2e_group_${{ matrix.e2e_group }} (with not @skip)" + echo "E2E shard (Makefile test-e2e-tagged): ${{ matrix.shard.tags }}" echo "Source config: tests/e2e/configs/run-${{ matrix.environment }}.yaml" echo "" echo "=== Configuration Preview ===" @@ -239,8 +262,8 @@ jobs: TERM: xterm-256color FORCE_COLOR: 1 E2E_DEPLOYMENT_MODE: ${{ matrix.mode }} - # Matches Makefile test-e2e-tagged / E2E_BEHAVE_TAG_EXPR (one @e2e_group_* per job). - E2E_BEHAVE_TAG_EXPR: "not @skip and @e2e_group_${{ matrix.e2e_group }}" + # Matches Makefile test-e2e-tagged / E2E_BEHAVE_TAG_EXPR (one @cfg_* pack per job). + E2E_BEHAVE_TAG_EXPR: "${{ matrix.shard.tags }}" run: | echo "Installing test dependencies..." pip install uv diff --git a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml index 3919ae64e..b522c6479 100644 --- a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml +++ b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml @@ -270,6 +270,10 @@ spec: value: "$(params.namespace)" - name: SNAPSHOT value: $(params.SNAPSHOT) + - name: OTEL_SDK_DISABLED + value: "true" + - name: OTEL_ANONYMIZATION_SECRET + value: "lightspeed-stack-otel-anonymization-dev-default" image: registry.access.redhat.com/ubi9/ubi-minimal script: | set +e diff --git a/.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml b/.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml index b075142c7..bd25670f6 100644 --- a/.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml +++ b/.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml @@ -396,6 +396,10 @@ spec: value: "$(params.vllm-api-key)" - name: VLLM_MODEL value: "$(params.model)" + - name: OTEL_SDK_DISABLED + value: "true" + - name: OTEL_ANONYMIZATION_SECRET + value: "lightspeed-stack-otel-anonymization-dev-default" image: registry.access.redhat.com/ubi9/ubi-minimal script: | set +e diff --git a/Makefile b/Makefile index 223acf7a0..46077ace5 100644 --- a/Makefile +++ b/Makefile @@ -162,11 +162,11 @@ test-e2e: ## Run end to end tests for the service test-e2e-local: ## Run end to end tests for the service (no script wrapper) uv run behave --color --format pretty --tags=-skip -D dump_errors=true @tests/e2e/test_list.txt -# Tag-based subsets (@e2e_group_* on feature files). Default runs all groups; override for one shard, e.g. -# E2E_BEHAVE_TAG_EXPR='not @skip and @e2e_group_2' make test-e2e-tagged-local -E2E_BEHAVE_TAG_EXPR ?= not @skip and (e2e_group_1 or e2e_group_2 or e2e_group_3) +# Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g. +# E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local +E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified) -test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @e2e_group_*) +test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*) script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt" test-e2e-tagged-local: ## Same as test-e2e-tagged without script wrapper diff --git a/README.md b/README.md index 556c59a08..78119846f 100644 --- a/README.md +++ b/README.md @@ -918,7 +918,7 @@ test-unit Run the unit tests test-integration Run integration tests tests test-e2e Run end to end tests for the service test-e2e-local Run end to end tests for the service (no script wrapper) -test-e2e-tagged Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @e2e_group_*) +test-e2e-tagged Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*) test-e2e-tagged-local Same as test-e2e-tagged without script wrapper benchmarks Run benchmarks check-types-src Check type hints in sources only diff --git a/docker-compose-library.yaml b/docker-compose-library.yaml index f4fe486b6..336737ba0 100755 --- a/docker-compose-library.yaml +++ b/docker-compose-library.yaml @@ -65,6 +65,7 @@ services: - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-} - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-} - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-} + - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default} - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-true} healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/liveness"] diff --git a/docker-compose.yaml b/docker-compose.yaml index 18e43be2d..e57eb39c8 100755 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -105,6 +105,7 @@ services: - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-} - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-} - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-} + - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default} - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-true} depends_on: llama-stack: diff --git a/docs/testing/e2e_testing.md b/docs/testing/e2e_testing.md index 6bddfd3bb..c1d38fcea 100644 --- a/docs/testing/e2e_testing.md +++ b/docs/testing/e2e_testing.md @@ -214,7 +214,7 @@ You can put several tags on one scenario. To document why a scenario is skipped, ## Configuration Files -- **Lightspeed-stack**: Under `tests/e2e/configuration/server-mode/` and `library-mode/`. Switched via `switch_config()` and copied into the container's config path (or applied via ConfigMap in Prow). Names like `lightspeed-stack.yaml`, `lightspeed-stack-auth-noop-token.yaml`, `lightspeed-stack-rbac.yaml`, etc. +- **Lightspeed-stack**: Under `tests/e2e/configuration/server-mode/` and `library-mode/`. Switched via `switch_config()` and copied into the container's config path (or applied via ConfigMap in Prow). Bootstrap: `lightspeed-stack.yaml`; variants: `lightspeed-stack-default.yaml`, `lightspeed-stack-authorized.yaml`, `lightspeed-stack-rbac.yaml`, etc. (see `tests/e2e/configuration/grouped/README.md`). - **Llama Stack**: Under `tests/e2e/configs/` (e.g. `run-ci.yaml`). Used by the Llama Stack container; not switched by Behave step-by-step, but the stack is started with the appropriate run config. See `tests/e2e/configuration/README.md` for a short description of each config. diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml index f8aa35caf..298eac71d 100644 --- a/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml @@ -55,7 +55,11 @@ spec: key: id optional: true - name: KV_RAG_PATH - value: "/app-root/src/.llama/storage/rag/kv_store.db" + value: "/app-root/.e2e-rag-work/kv_store.db" + - name: OTEL_SDK_DISABLED + value: "true" + - name: OTEL_ANONYMIZATION_SECRET + value: "lightspeed-stack-otel-anonymization-dev-default" - name: VLLM_MODEL valueFrom: secretKeyRef: diff --git a/tests/e2e/configuration/README.md b/tests/e2e/configuration/README.md index 3edc5d852..b36efa599 100644 --- a/tests/e2e/configuration/README.md +++ b/tests/e2e/configuration/README.md @@ -54,8 +54,27 @@ Both server-mode and library-mode default configurations include: 3. **User Data Collection** - Enabled for feedback and transcripts testing -### Special-Purpose Configurations +### Config variants (`@cfg_*` tags) + +Named configs merge compatible options and map to `@cfg_*` Behave tags so CI +can shard by config (fewer restarts per job). See +[grouped/README.md](grouped/README.md). + +| Config | Purpose | +|--------|---------| +| `lightspeed-stack-default.yaml` | Default + inline RAG | +| `lightspeed-stack-authorized.yaml` | Authorization (noop-token) | +| `lightspeed-stack-negative.yaml` | No cache + invalid feedback storage | +| `lightspeed-stack-rbac.yaml` | RBAC (jwk-token auth) | +| `lightspeed-stack-rh-identity.yaml` | RH Identity header auth | +| `lightspeed-stack-skills.yaml` / `-skills-directory.yaml` | Agent skills | +| `lightspeed-stack-mcp.yaml` | All valid MCP servers (`mcp_servers_api`) | +| `lightspeed-stack-mcp-invalid.yaml` | Invalid MCP file token | +| `lightspeed-stack-mcp-api-auth.yaml` | MCP server API auth | +| `lightspeed-stack-mcp-{file,kubernetes,client,oauth}-auth.yaml` | Per-auth MCP (`mcp.feature`) | +| `lightspeed-stack-byok-pdf.yaml` | BYOK PDF (library only) | +| `lightspeed-stack-tls.yaml` / `-degraded.yaml` | TLS / degraded mode (server only) | + +Bootstrap default: `lightspeed-stack.yaml`. Environment-specific: +`lightspeed-stack-rhelai.yaml`, `lightspeed-stack-rhoai.yaml`. -- `lightspeed-stack-auth-noop-token.yaml` - For authorization testing -- `lightspeed-stack-invalid-feedback-storage.yaml` - For negative feedback testing -- `lightspeed-stack-no-cache.yaml` - For cache-disabled scenarios diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-auth-noop-token.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-authorized.yaml similarity index 70% rename from tests/e2e/configuration/library-mode/lightspeed-stack-auth-noop-token.yaml rename to tests/e2e/configuration/library-mode/lightspeed-stack-authorized.yaml index ca8b4476c..c049163bf 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-auth-noop-token.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-authorized.yaml @@ -1,3 +1,11 @@ +# @cfg_authorized +# Safe merge: auth-noop-token + shields from default. +# Intentionally NO mcp_servers: query/streaming_query/responses/tools all call +# check_mcp_auth(); a static MCP entry would probe mock-mcp on every authorized +# suite request and amplify flake. +# Intentionally NO pii-redaction shields: streaming_query compares +# token-stream text to turn_complete; redaction only applies to the complete +# message and would make that assertion fail (v2 vs v[NUM]). name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-byok-pdf.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-byok-pdf.yaml index da9924d84..873aced40 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-byok-pdf.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-byok-pdf.yaml @@ -1,3 +1,6 @@ +# @cfg_byok_pdf +# Kept separate: dedicated vector store + rag.inline=[pdf-field-notes] so +# retrieval is not mixed with e2e-test-docs (Paul Graham) chunks. name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-inline-rag.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-default.yaml similarity index 68% rename from tests/e2e/configuration/library-mode/lightspeed-stack-inline-rag.yaml rename to tests/e2e/configuration/library-mode/lightspeed-stack-default.yaml index f73c4d9c3..811b30efd 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-inline-rag.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-default.yaml @@ -1,3 +1,8 @@ +# @cfg_default +# Safe merge: lightspeed-stack.yaml + lightspeed-stack-inline-rag.yaml +# Same byok rag_id (e2e-test-docs) with both rag.tool and rag.inline. +# /v1/rags partial asserts still match. No MCP (would break info tools checks +# and add check_mcp_auth to every call). No PDF BYOK (separate retrieval store). name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 @@ -39,5 +44,15 @@ byok_rag: score_multiplier: 1.0 rag: + tool: + - e2e-test-docs inline: - e2e-test-docs + +shields: + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-api-auth.yaml similarity index 80% rename from tests/e2e/configuration/library-mode/lightspeed-stack-mcp-auth.yaml rename to tests/e2e/configuration/library-mode/lightspeed-stack-mcp-api-auth.yaml index 553dc0eae..d9ea761b1 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-auth.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-api-auth.yaml @@ -1,3 +1,7 @@ +# @cfg_mcp_api_auth +# From: lightspeed-stack-mcp-auth.yaml +# Kept out of @cfg_authorized: static MCP would make check_mcp_auth run on +# every query/responses call in the authorized suite. name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-invalid.yaml similarity index 82% rename from tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml rename to tests/e2e/configuration/library-mode/lightspeed-stack-mcp-invalid.yaml index bdfe7e194..b32feb157 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-invalid.yaml @@ -1,3 +1,6 @@ +# @cfg_mcp_invalid +# From: lightspeed-stack-invalid-mcp-file-auth.yaml +# Must stay isolated: tools/query expect 401 when the only MCP server has a bad token. name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml index ecb71cf23..90761a733 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml @@ -1,3 +1,9 @@ +# @cfg_mcp +# Merges: lightspeed-stack-mcp.yaml + lightspeed-stack-mcp-file-auth.yaml + +# lightspeed-stack-mcp-client-auth.yaml + lightspeed-stack-mcp-oauth-auth.yaml + +# lightspeed-stack-mcp-kubernetes-auth.yaml (valid tokens only) +# All four MCP auth flavours share one process; scenarios select a server by name. +# Keep @cfg_mcp_invalid separate (bad file token must be the only mcp-file entry). name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 @@ -36,4 +42,4 @@ mcp_servers: - name: "mcp-client" url: "http://mock-mcp:3000" authorization_headers: - Authorization: "client" \ No newline at end of file + Authorization: "client" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-feedback-storage.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-negative.yaml similarity index 58% rename from tests/e2e/configuration/library-mode/lightspeed-stack-invalid-feedback-storage.yaml rename to tests/e2e/configuration/library-mode/lightspeed-stack-negative.yaml index 16edc3ddf..c8fb98607 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-feedback-storage.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-negative.yaml @@ -1,3 +1,8 @@ +# @cfg_negative +# Merges: lightspeed-stack-no-cache.yaml + lightspeed-stack-invalid-feedback-storage.yaml +# Compatible: both use noop-with-token; query works with cache=None; conversations +# v2 asserts "cache not configured"; feedback asserts store failure at /invalid. +# Intentionally no conversation_cache and invalid feedback_storage. name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 @@ -18,5 +23,7 @@ user_data_collection: transcripts_enabled: true transcripts_storage: "/tmp/data/transcripts" +# NO conversation_cache — cache-disabled + empty MCP list scenarios + authentication: module: "noop-with-token" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-no-cache.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-no-cache.yaml deleted file mode 100644 index 464770f41..000000000 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-no-cache.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - use_as_library_client: true - # Unified mode: run.yaml (materialized per provider by CI/the harness) - # is consumed as the synthesis profile instead of the legacy two-file path. - config: - profile: run.yaml -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" - -# NO conversation_cache configured - for testing error handling - -authentication: - module: "noop-with-token" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-rbac.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-rbac.yaml index 4461d60f5..cce3e07ed 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-rbac.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-rbac.yaml @@ -1,3 +1,5 @@ +# @cfg_rbac +# From: lightspeed-stack-rbac.yaml (auth module jwk-token is incompatible with other groups) name: Lightspeed Core Service (RBAC E2E Tests - Library Mode) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-auth-rh-identity.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-rh-identity.yaml similarity index 87% rename from tests/e2e/configuration/library-mode/lightspeed-stack-auth-rh-identity.yaml rename to tests/e2e/configuration/library-mode/lightspeed-stack-rh-identity.yaml index 433e77fec..ba4ebd482 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-auth-rh-identity.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-rh-identity.yaml @@ -1,3 +1,5 @@ +# @cfg_rh_identity +# From: lightspeed-stack-auth-rh-identity.yaml (auth module incompatible with other groups) name: Lightspeed Core Service (LCS) - RH Identity Auth service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-skills-directory.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-skills-directory.yaml index a6c9c5cb8..1e0d8981c 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-skills-directory.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-skills-directory.yaml @@ -1,3 +1,5 @@ +# @cfg_skills_directory +# Directory discovery (echo + summarize). Separate from @cfg_skills. name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-skills.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-skills.yaml index c35f56300..982a1309c 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-skills.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-skills.yaml @@ -1,3 +1,6 @@ +# @cfg_skills +# Echo-only path. Not merged with skills-directory: @SkillsConfig asserts +# exact list_skills tool_results content with echo alone. name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-auth-noop-token.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-authorized.yaml similarity index 83% rename from tests/e2e/configuration/server-mode/lightspeed-stack-auth-noop-token.yaml rename to tests/e2e/configuration/server-mode/lightspeed-stack-authorized.yaml index 49ee71d59..dba5ea60b 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-auth-noop-token.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-authorized.yaml @@ -1,3 +1,7 @@ +# @cfg_authorized +# Safe merge: auth-noop-token + shields. No MCP (see library twin comment). +# Based on auth-noop-token. Intentionally NO mcp_servers / pii-redaction shields +# (see library twin: streaming token vs turn_complete comparison). name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-inline-rag.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-default.yaml similarity index 65% rename from tests/e2e/configuration/server-mode/lightspeed-stack-inline-rag.yaml rename to tests/e2e/configuration/server-mode/lightspeed-stack-default.yaml index 1a2850162..1581097db 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-inline-rag.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-default.yaml @@ -1,3 +1,8 @@ +# @cfg_default +# Safe merge: lightspeed-stack.yaml + lightspeed-stack-inline-rag.yaml +# Same byok rag_id (e2e-test-docs) with both rag.tool and rag.inline. +# /v1/rags partial asserts still match. No MCP (would break info tools checks +# and add check_mcp_auth to every call). No PDF BYOK (separate retrieval store). name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 @@ -37,5 +42,15 @@ byok_rag: score_multiplier: 1.0 rag: + tool: + - e2e-test-docs inline: - e2e-test-docs + +shields: + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-degraded-mode.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-degraded.yaml similarity index 89% rename from tests/e2e/configuration/server-mode/lightspeed-stack-degraded-mode.yaml rename to tests/e2e/configuration/server-mode/lightspeed-stack-degraded.yaml index 1435b9fc0..390d0548c 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-degraded-mode.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-degraded.yaml @@ -1,3 +1,5 @@ +# @cfg_degraded +# From: lightspeed-stack-degraded-mode.yaml (server-mode only) name: Lightspeed Core Service (LCS) - Degraded Mode Test service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-api-auth.yaml similarity index 86% rename from tests/e2e/configuration/server-mode/lightspeed-stack-mcp-auth.yaml rename to tests/e2e/configuration/server-mode/lightspeed-stack-mcp-api-auth.yaml index a158cd661..a18e03fb1 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-api-auth.yaml @@ -1,3 +1,5 @@ +# @cfg_mcp_api_auth +# From: lightspeed-stack-mcp-auth.yaml (kept out of authorized — see library twin) name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-invalid.yaml similarity index 80% rename from tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml rename to tests/e2e/configuration/server-mode/lightspeed-stack-mcp-invalid.yaml index ffaee6f21..804b936e3 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-invalid.yaml @@ -1,3 +1,6 @@ +# @cfg_mcp_invalid +# From: lightspeed-stack-invalid-mcp-file-auth.yaml +# Must stay isolated: tools/query expect 401 when the only MCP server has a bad token. name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml index e78a705d1..0f144bc65 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml @@ -1,3 +1,9 @@ +# @cfg_mcp +# Merges: lightspeed-stack-mcp.yaml + lightspeed-stack-mcp-file-auth.yaml + +# lightspeed-stack-mcp-client-auth.yaml + lightspeed-stack-mcp-oauth-auth.yaml + +# lightspeed-stack-mcp-kubernetes-auth.yaml (valid tokens only) +# All four MCP auth flavours share one process; scenarios select a server by name. +# Keep @cfg_mcp_invalid separate (bad file token must be the only mcp-file entry). name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 @@ -34,4 +40,4 @@ mcp_servers: - name: "mcp-client" url: "http://mock-mcp:3000" authorization_headers: - Authorization: "client" \ No newline at end of file + Authorization: "client" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-feedback-storage.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-negative.yaml similarity index 63% rename from tests/e2e/configuration/server-mode/lightspeed-stack-invalid-feedback-storage.yaml rename to tests/e2e/configuration/server-mode/lightspeed-stack-negative.yaml index eb6ba2054..f9792607b 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-feedback-storage.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-negative.yaml @@ -1,3 +1,8 @@ +# @cfg_negative +# Merges: lightspeed-stack-no-cache.yaml + lightspeed-stack-invalid-feedback-storage.yaml +# Compatible: both use noop-with-token; query works with cache=None; conversations +# v2 asserts "cache not configured"; feedback asserts store failure at /invalid. +# Intentionally no conversation_cache and invalid feedback_storage. name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 @@ -21,5 +26,7 @@ user_data_collection: transcripts_enabled: true transcripts_storage: "/tmp/data/transcripts" +# NO conversation_cache — cache-disabled + empty MCP list scenarios + authentication: module: "noop-with-token" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-no-cache.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-no-cache.yaml deleted file mode 100644 index 6c8f31438..000000000 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-no-cache.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Lightspeed Core Service (LCS) -service: - host: 0.0.0.0 - port: 8080 - auth_enabled: false - workers: 1 - color_log: true - access_log: true -llama_stack: - # Uses a remote llama-stack service - # The instance would have already been started with a llama-stack-run.yaml file - use_as_library_client: false - # Alternative for "as library use" - # use_as_library_client: true - # library_client_config_path: - url: http://${env.E2E_LLAMA_HOSTNAME}:8321 - api_key: xyzzy -user_data_collection: - feedback_enabled: true - feedback_storage: "/tmp/data/feedback" - transcripts_enabled: true - transcripts_storage: "/tmp/data/transcripts" - -# NO conversation_cache configured - for testing error handling - -authentication: - module: "noop-with-token" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-rbac.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-rbac.yaml index ea5bce5f3..ca966cc0f 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-rbac.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-rbac.yaml @@ -1,3 +1,5 @@ +# @cfg_rbac +# From: lightspeed-stack-rbac.yaml (auth module jwk-token is incompatible with other groups) name: Lightspeed Core Service (RBAC E2E Tests) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-auth-rh-identity.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-rh-identity.yaml similarity index 84% rename from tests/e2e/configuration/server-mode/lightspeed-stack-auth-rh-identity.yaml rename to tests/e2e/configuration/server-mode/lightspeed-stack-rh-identity.yaml index e2b468cf0..0853969fe 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-auth-rh-identity.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-rh-identity.yaml @@ -1,3 +1,5 @@ +# @cfg_rh_identity +# From: lightspeed-stack-auth-rh-identity.yaml (auth module incompatible with other groups) name: Lightspeed Core Service (LCS) - RH Identity Auth service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-skills-directory.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-skills-directory.yaml index 0ae7888c7..1910bd307 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-skills-directory.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-skills-directory.yaml @@ -1,3 +1,4 @@ +# @cfg_skills_directory name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-skills.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-skills.yaml index 387d03856..1f5cbce72 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-skills.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-skills.yaml @@ -1,3 +1,4 @@ +# @cfg_skills name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-tls.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-tls.yaml index fd45ea744..a1e73fe77 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-tls.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-tls.yaml @@ -1,3 +1,5 @@ +# @cfg_tls +# From: lightspeed-stack-tls.yaml (server-mode only; special inference provider) name: Lightspeed Core Service (LCS) service: host: 0.0.0.0 diff --git a/tests/e2e/features/authorized_noop.feature b/tests/e2e/features/authorized_noop.feature index e24934f71..51b37ae37 100644 --- a/tests/e2e/features/authorized_noop.feature +++ b/tests/e2e/features/authorized_noop.feature @@ -1,4 +1,4 @@ -@e2e_group_1 +@cfg_default Feature: Authorized endpoint API tests for the noop authentication module Background: @@ -6,7 +6,7 @@ Feature: Authorized endpoint API tests for the noop authentication module And The system is in default state And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack.yaml configuration + And The service uses the lightspeed-stack-default.yaml configuration And The service is restarted Scenario: Check if the authorized endpoint works fine when user_id and auth header are not provided diff --git a/tests/e2e/features/authorized_noop_token.feature b/tests/e2e/features/authorized_noop_token.feature index e8f75d2f8..c8f10d0af 100644 --- a/tests/e2e/features/authorized_noop_token.feature +++ b/tests/e2e/features/authorized_noop_token.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @Authorized +@cfg_authorized @Authorized Feature: Authorized endpoint API tests for the noop-with-token authentication module Background: @@ -7,7 +7,7 @@ Feature: Authorized endpoint API tests for the noop-with-token authentication mo And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Scenario: Check if the authorized endpoint works when user_id is not provided diff --git a/tests/e2e/features/authorized_rh_identity.feature b/tests/e2e/features/authorized_rh_identity.feature index 3e196527d..9a01bc729 100644 --- a/tests/e2e/features/authorized_rh_identity.feature +++ b/tests/e2e/features/authorized_rh_identity.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @RHIdentity +@cfg_rh_identity @RHIdentity Feature: Authorized endpoint API tests for the rh-identity authentication module Background: @@ -6,7 +6,7 @@ Feature: Authorized endpoint API tests for the rh-identity authentication module And The system is in default state And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-rh-identity.yaml configuration + And The service uses the lightspeed-stack-rh-identity.yaml configuration And The service is restarted Scenario: Request fails when identity field is missing diff --git a/tests/e2e/features/byok_pdf.feature b/tests/e2e/features/byok_pdf.feature index 71245b9c8..930314e2c 100644 --- a/tests/e2e/features/byok_pdf.feature +++ b/tests/e2e/features/byok_pdf.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @skip-in-server-mode +@cfg_byok_pdf @skip-in-server-mode Feature: BYOK PDF support tests # Validates that a vector store built from a PDF by rag-content's `pdf` diff --git a/tests/e2e/features/conversation_cache_v2.feature b/tests/e2e/features/conversation_cache_v2.feature index ff32f8d59..b96bfebe7 100644 --- a/tests/e2e/features/conversation_cache_v2.feature +++ b/tests/e2e/features/conversation_cache_v2.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @Authorized +@Authorized Feature: Conversation Cache V2 API tests Background: @@ -6,7 +6,7 @@ Feature: Conversation Cache V2 API tests And The system is in default state And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted @@ -16,7 +16,8 @@ Feature: Conversation Cache V2 API tests # BUG: Test without no_tools to expose AttributeError with empty vector database # TODO: Remove @skip when bug is fixed (empty vector DB causes 500 error) - @skip + + @skip @cfg_authorized Scenario: V2 conversations endpoint WITHOUT no_tools (known bug - empty vector DB) Given REST API service prefix is /v1 And I use "query" to ask question with authorization header @@ -30,7 +31,7 @@ Feature: Conversation Cache V2 API tests Then The status code of the response is 200 And The conversation with conversation_id from above is returned - + @cfg_authorized Scenario: V2 conversations endpoint finds the correct conversation when it exists Given REST API service prefix is /v1 And I use "query" to ask question with authorization header @@ -71,6 +72,7 @@ Feature: Conversation Cache V2 API tests # V2 Conversation GET by ID Endpoint Tests # ==================================================================== + @cfg_authorized Scenario: V2 conversations/{conversation_id} endpoint finds conversation with full metadata Given REST API service prefix is /v1 And I use "query" to ask question with authorization header @@ -133,7 +135,7 @@ Feature: Conversation Cache V2 API tests } """ - + @cfg_authorized Scenario: V2 conversations/{conversation_id} GET endpoint fails when conversation_id is malformed Given REST API service prefix is /v2 When I use REST API conversation endpoint with conversation_id "abcdef" using HTTP GET method @@ -148,27 +150,18 @@ Feature: Conversation Cache V2 API tests } """ - + @cfg_authorized Scenario: V2 conversations/{conversation_id} GET endpoint fails when conversation does not exist Given REST API service prefix is /v2 When I use REST API conversation endpoint with conversation_id "12345678-abcd-0000-0123-456789abcdef" using HTTP GET method Then The status code of the response is 404 And The body of the response contains Conversation not found - @NoCacheConfig - Scenario: Check conversations/{conversation_id} fails when cache not configured - Given The service uses the lightspeed-stack-no-cache.yaml configuration - And The service is restarted - And REST API service prefix is /v2 - When I access REST API endpoint "conversations" using HTTP GET method - Then The status code of the response is 500 - And The body of the response contains Conversation cache not configured - - # ==================================================================== # V2 Conversation DELETE Endpoint Tests # ==================================================================== + @cfg_authorized Scenario: V2 conversations DELETE endpoint removes the correct conversation Given REST API service prefix is /v1 And I use "query" to ask question with authorization header @@ -189,14 +182,14 @@ Feature: Conversation Cache V2 API tests Then The status code of the response is 404 And The body of the response contains Conversation not found - + @cfg_authorized Scenario: V2 conversations/{conversation_id} DELETE endpoint fails when conversation_id is malformed Given REST API service prefix is /v2 When I use REST API conversation endpoint with conversation_id "abcdef" using HTTP DELETE method Then The status code of the response is 400 And The body of the response contains Invalid conversation ID format - + @cfg_authorized Scenario: V2 conversations DELETE endpoint fails when the conversation does not exist Given REST API service prefix is /v2 When I use REST API conversation endpoint with conversation_id "12345678-abcd-0000-0123-456789abcdef" using HTTP DELETE method @@ -210,6 +203,7 @@ Feature: Conversation Cache V2 API tests # V2 Conversation PUT (Update Topic Summary) Endpoint Tests # ==================================================================== + @cfg_authorized Scenario: V2 conversations PUT endpoint successfully updates topic summary Given REST API service prefix is /v1 And I use "query" to ask question with authorization header @@ -231,7 +225,7 @@ Feature: Conversation Cache V2 API tests And The conversation with conversation_id from above is returned And The conversation topic_summary is "Kubernetes Deployment Strategies" - + @cfg_authorized Scenario: V2 conversations PUT endpoint fails when conversation_id is malformed Given REST API service prefix is /v2 When I use REST API conversation endpoint with conversation_id "invalid-id" and topic_summary "Updated Summary" using HTTP PUT method @@ -246,14 +240,14 @@ Feature: Conversation Cache V2 API tests } """ - + @cfg_authorized Scenario: V2 conversations PUT endpoint fails when conversation does not exist Given REST API service prefix is /v2 When I use REST API conversation endpoint with conversation_id "12345678-abcd-0000-0123-456789abcdef" and topic_summary "Updated Summary" using HTTP PUT method Then The status code of the response is 404 And The body of the response contains Conversation not found - + @cfg_authorized Scenario: V2 conversations PUT endpoint fails with empty topic summary (422) Given REST API service prefix is /v1 And I use "query" to ask question with authorization header @@ -266,3 +260,12 @@ Feature: Conversation Cache V2 API tests When I use REST API conversation endpoint with conversation_id from above and empty topic_summary using HTTP PUT method Then The status code of the response is 422 And The body of the response contains String should have at least 1 character + + @NoCacheConfig @cfg_negative + Scenario: Check conversations/{conversation_id} fails when cache not configured + Given The service uses the lightspeed-stack-negative.yaml configuration + And The service is restarted + And REST API service prefix is /v2 + When I access REST API endpoint "conversations" using HTTP GET method + Then The status code of the response is 500 + And The body of the response contains Conversation cache not configured diff --git a/tests/e2e/features/conversations.feature b/tests/e2e/features/conversations.feature index b017f407f..58adce29d 100644 --- a/tests/e2e/features/conversations.feature +++ b/tests/e2e/features/conversations.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @Authorized +@cfg_authorized @Authorized Feature: conversations endpoint API tests Background: @@ -7,7 +7,7 @@ Feature: conversations endpoint API tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted diff --git a/tests/e2e/features/degraded_mode_startup.feature b/tests/e2e/features/degraded_mode_startup.feature index 80d1040e9..6b24f3f8c 100644 --- a/tests/e2e/features/degraded_mode_startup.feature +++ b/tests/e2e/features/degraded_mode_startup.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @skip-in-library-mode @Authorized +@cfg_degraded @skip-in-library-mode @Authorized Feature: Degraded mode startup End-to-end scenarios that test LCORE startup behavior when llama-stack @@ -14,7 +14,7 @@ Feature: Degraded mode startup And the Lightspeed stack configuration directory is "tests/e2e/configuration" Scenario: Degraded mode metric is set to 0.0 when started with llama-stack - Given The service uses the lightspeed-stack-degraded-mode.yaml configuration + Given The service uses the lightspeed-stack-degraded.yaml configuration And The service is restarted When I access endpoint "metrics" using HTTP GET method Then The status code of the response is 200 @@ -22,7 +22,7 @@ Feature: Degraded mode startup Scenario: Degraded mode metric is set to 1.0 when started without llama-stack Given The llama-stack connection is disrupted - And The service uses the lightspeed-stack-degraded-mode.yaml configuration + And The service uses the lightspeed-stack-degraded.yaml configuration And The service is restarted When I access endpoint "metrics" using HTTP GET method Then The status code of the response is 200 @@ -30,11 +30,11 @@ Feature: Degraded mode startup Scenario: Readiness endpoint reports degraded state when started without llama-stack Given The llama-stack connection is disrupted - And The service uses the lightspeed-stack-degraded-mode.yaml configuration + And The service uses the lightspeed-stack-degraded.yaml configuration And The service is restarted When I access endpoint "readiness" using HTTP GET method - Then The status code of the response is 503 + Then The status code of the response is 200 And The body of the response, ignoring the "providers" field, is the following """ - {"ready": false, "reason": "Cannot connect to backend service", "overall_status": "unhealthy", "impacts": ["LLM inference unavailable", "Provider health checks unavailable"]} + {"ready": true, "reason": "Service running in degraded mode", "overall_status": "degraded", "impacts": ["LLM inference unavailable", "RAG functionality unavailable", "Agent tools unavailable"]} """ diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 47982fce7..015ac2526 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -243,6 +243,8 @@ def before_scenario(context: Context, scenario: Scenario) -> None: context.scenario_lightspeed_override_active = False context.lightspeed_stack_skip_restart = False + # Reset force-restart from a prior disrupt/MCP reset scenario. + context.force_lightspeed_restart_after_mcp_config_reset = False # Clear shield unregister state from previous scenarios (see ``shields_are_disabled_for_scenario``). for _attr in ( @@ -279,18 +281,11 @@ def _dump_pod_logs_on_failure( def after_scenario(context: Context, scenario: Scenario) -> None: """Run after each scenario is run. - Perform per-scenario teardown: restore scenario-specific configuration and, - in server mode, attempt to restart and verify the Llama Stack container if - it was previously running. + Perform per-scenario teardown: failure logs (Prow) and shield re-register. If ``configure_service`` applied a non-baseline YAML during the scenario - (``context.scenario_lightspeed_override_active``), copies - ``context.feature_config`` back and restarts lightspeed-stack. - - When not running in library mode and the context indicates the Llama Stack - was running before the scenario, this function attempts to start the - llama-stack container and polls its health endpoint until it becomes - healthy or a timeout is reached. + (``context.scenario_lightspeed_override_active``), clears that flag only; + the next ``The service uses ...`` step re-applies config as needed. Parameters: ---------- @@ -312,10 +307,6 @@ def after_scenario(context: Context, scenario: Scenario) -> None: if getattr(context, "scenario_lightspeed_override_active", False): context.scenario_lightspeed_override_active = False - feature_cfg = getattr(context, "feature_config", None) - if feature_cfg: - switch_config(feature_cfg) - restart_container("lightspeed-stack") # Re-register shield if ``Given shields are disabled for this scenario`` unregistered it. if getattr(context, "shields_disabled_for_scenario", False): @@ -407,7 +398,7 @@ def _restore_llama_stack() -> None: # Wait for the service to be healthy print("Restoring Llama Stack connection...") - max_attempts = 24 + max_attempts = 60 for attempt in range(max_attempts): try: result = subprocess.run( @@ -457,6 +448,8 @@ def before_feature(context: Context, feature: Feature) -> None: Per-feature setup that is not expressed in Gherkin. Lightspeed YAML is applied in feature Backgrounds via ``configure_service``. + Does not reset the applied-config basename tracker (skip-restart across features). + Records monotonic start time on ``feature`` for duration logging in ``after_feature`` (includes scenarios and feature teardown). @@ -466,7 +459,8 @@ def before_feature(context: Context, feature: Feature) -> None: ``E2E_FLAKY_MAX_ATTEMPTS`` environment variable. """ setattr(feature, _E2E_FEATURE_PERF_START_ATTR, time.perf_counter()) - reset_active_lightspeed_stack_config_basename() + context.feature_config = None + context.scenario_lightspeed_override_active = False context.active_lightspeed_stack_config_basename = None # One real Llama disruption per feature (module-level flag; survives context resets) reset_llama_stack_disrupt_once_tracking() @@ -489,10 +483,20 @@ def before_feature(context: Context, feature: Feature) -> None: delattr(context, _attr) +def _restore_config_after_feature_enabled() -> bool: + """Return True when legacy per-feature bootstrap restore/restart is requested.""" + return os.getenv("E2E_RESTORE_CONFIG_AFTER_FEATURE", "0").strip().lower() in { + "1", + "true", + "yes", + } + + def after_feature(context: Context, feature: Feature) -> None: """Run after each feature file is exercised. - Perform feature-level teardown: restore any modified configuration and, + Perform feature-level teardown: restore bootstrap configuration when + ``E2E_RESTORE_CONFIG_AFTER_FEATURE=1``, otherwise keep the active config; when ``context.feedback_e2e_conversation_cleanup`` is set by feedback steps, delete tracked feedback test conversations. """ @@ -513,14 +517,19 @@ def after_feature(context: Context, feature: Feature) -> None: # Restore Lightspeed Stack config if the generic configure_service step switched it. # This cleanup intentionally runs for any feature (not tag-gated) - any feature that - # leaves a backup file will trigger config restoration and container restarts. + # leaves a backup file will trigger config restoration and container restarts when + # E2E_RESTORE_CONFIG_AFTER_FEATURE=1; otherwise the backup is dropped only. backup_path = "lightspeed-stack.yaml.backup" if os.path.exists(backup_path): - switch_config(backup_path) - remove_config_backup(backup_path) - if not context.is_library_mode: - restart_container("llama-stack") - restart_container("lightspeed-stack") + if _restore_config_after_feature_enabled(): + switch_config(backup_path) + remove_config_backup(backup_path) + if not context.is_library_mode: + restart_container("llama-stack") + restart_container("lightspeed-stack") + reset_active_lightspeed_stack_config_basename() + else: + remove_config_backup(backup_path) # Clean up any proxy servers left from the last scenario if hasattr(context, "tunnel_proxy") or hasattr(context, "interception_proxy"): diff --git a/tests/e2e/features/faiss.feature b/tests/e2e/features/faiss.feature index bec4f128e..cd5fd4b6a 100644 --- a/tests/e2e/features/faiss.feature +++ b/tests/e2e/features/faiss.feature @@ -1,4 +1,4 @@ -@e2e_group_1 @Authorized +@cfg_authorized @Authorized Feature: FAISS support tests Background: @@ -7,7 +7,7 @@ Feature: FAISS support tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Scenario: check if vector store is registered diff --git a/tests/e2e/features/feedback.feature b/tests/e2e/features/feedback.feature index 3fa16c16e..3d863fc13 100644 --- a/tests/e2e/features/feedback.feature +++ b/tests/e2e/features/feedback.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @Feedback +@Feedback Feature: feedback endpoint API tests @@ -8,9 +8,10 @@ Feature: feedback endpoint API tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted + @cfg_authorized Scenario: Check if enabling the feedback is working When The feedback is enabled Then The status code of the response is 200 @@ -23,7 +24,8 @@ Feature: feedback endpoint API tests } } """ - + + @cfg_authorized Scenario: Check if disabling the feedback is working When The feedback is disabled Then The status code of the response is 200 @@ -37,6 +39,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if toggling the feedback with incorrect attribute name fails When I update feedback status with """ @@ -62,6 +65,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if getting feedback status returns true when feedback is enabled And The feedback is enabled When I retreive the current feedback status @@ -76,6 +80,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if getting feedback status returns false when feedback is disabled And The feedback is disabled When I retreive the current feedback status @@ -90,6 +95,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if feedback endpoint is not working when feedback is disabled And A new conversation is initialized And The feedback is disabled @@ -113,6 +119,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if feedback endpoint fails when required fields are not specified And The feedback is enabled When I submit the following feedback without specifying conversation ID @@ -153,6 +160,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if feedback endpoint is working when sentiment is negative And A new conversation is initialized And The feedback is enabled @@ -173,6 +181,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if feedback endpoint is working when sentiment is positive And A new conversation is initialized And The feedback is enabled @@ -193,6 +202,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if feedback submission fails when invalid sentiment is passed And A new conversation is initialized And The feedback is enabled @@ -218,6 +228,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if feedback submission fails when nonexisting conversation ID is passed And The feedback is enabled When I submit the following feedback for nonexisting conversation "12345678-abcd-0000-0123-456789abcdef" @@ -240,6 +251,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if feedback submission fails when conversation belongs to a different user And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva # Create a conversation as a different user (via user_id query param for noop_with_token) @@ -258,6 +270,7 @@ Feature: feedback endpoint API tests Then The status code of the response is 403 And The body of the response contains User does not have permission to perform this action + @cfg_authorized Scenario: Check if feedback endpoint fails when only empty string user_feedback is provided Given The system is in default state And A new conversation is initialized @@ -285,33 +298,7 @@ Feature: feedback endpoint API tests } """ -@InvalidFeedbackStorageConfig - Scenario: Check if feedback submittion fails when invalid feedback storage path is configured - Given The service uses the lightspeed-stack-invalid-feedback-storage.yaml configuration - And The service is restarted - And The system is in default state - And The feedback is enabled - And A new conversation is initialized - When I submit the following feedback for the conversation created before - """ - { - "llm_response": "Sample Response", - "sentiment": -1, - "user_feedback": "Not satisfied with the response quality", - "user_question": "Sample Question" - } - """ - Then The status code of the response is 500 - And The body of the response is the following - """ - { - "detail": { - "response": "Failed to store feedback", - "cause": "Failed to store feedback at directory: /invalid" - } - } - """ - + @cfg_authorized Scenario: Check if sequential feedback status toggling maintains consistency When The feedback is enabled Then The status code of the response is 200 @@ -331,6 +318,7 @@ Feature: feedback endpoint API tests } """ + @cfg_authorized Scenario: Check if submitting duplicate feedback succeeds And A new conversation is initialized And The feedback is enabled @@ -366,3 +354,30 @@ Feature: feedback endpoint API tests "response": "feedback received" } """ + +@InvalidFeedbackStorageConfig @cfg_negative + Scenario: Check if feedback submittion fails when invalid feedback storage path is configured + Given The service uses the lightspeed-stack-negative.yaml configuration + And The service is restarted + And The system is in default state + And The feedback is enabled + And A new conversation is initialized + When I submit the following feedback for the conversation created before + """ + { + "llm_response": "Sample Response", + "sentiment": -1, + "user_feedback": "Not satisfied with the response quality", + "user_question": "Sample Question" + } + """ + Then The status code of the response is 500 + And The body of the response is the following + """ + { + "detail": { + "response": "Failed to store feedback", + "cause": "Failed to store feedback at directory: /invalid" + } + } + """ diff --git a/tests/e2e/features/health.feature b/tests/e2e/features/health.feature index 1563cad97..4bfa16664 100644 --- a/tests/e2e/features/health.feature +++ b/tests/e2e/features/health.feature @@ -1,4 +1,4 @@ -@e2e_group_2 +@cfg_default Feature: REST API tests @@ -7,7 +7,7 @@ Feature: REST API tests And The system is in default state And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack.yaml configuration + And The service uses the lightspeed-stack-default.yaml configuration And The service is restarted diff --git a/tests/e2e/features/http_401_unauthorized.feature b/tests/e2e/features/http_401_unauthorized.feature index d33076277..806b0ef1d 100644 --- a/tests/e2e/features/http_401_unauthorized.feature +++ b/tests/e2e/features/http_401_unauthorized.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @Authorized @Feedback @RHIdentity @RBAC +@Authorized @Feedback @RHIdentity @RBAC Feature: HTTP 401 Unauthorized Aggregates end-to-end scenarios that assert a 401 response when authentication @@ -13,8 +13,12 @@ Feature: HTTP 401 Unauthorized # --- query / streaming_query --- +# --- @cfg_authorized --- + + + @cfg_authorized Scenario: Check if LLM responds to sent question with error when not authenticated - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I use "query" to ask question """ @@ -31,8 +35,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Check if LLM responds to sent question with error when bearer token is missing - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I use "query" to ask question """ @@ -41,8 +47,10 @@ Feature: HTTP 401 Unauthorized Then The status code of the response is 401 And The body of the response contains No Authorization header found + + @cfg_authorized Scenario: Check if LLM responds to sent question with error when not authenticated (streaming_query) - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I use "streaming_query" to ask question """ @@ -61,8 +69,10 @@ Feature: HTTP 401 Unauthorized # --- conversations --- + + @cfg_authorized Scenario: Check if conversations endpoint fails when the auth header is not present - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And I use "query" to ask question with authorization header @@ -84,8 +94,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Check if conversations/{conversation_id} endpoint fails when the auth header is not present - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And I use "query" to ask question with authorization header @@ -109,8 +121,10 @@ Feature: HTTP 401 Unauthorized # --- FAISS --- + + @cfg_authorized Scenario: Check if rags endpoints responds with error when not authenticated - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I access REST API endpoint rags using HTTP GET method Then The status code of the response is 401 @@ -124,10 +138,30 @@ Feature: HTTP 401 Unauthorized } """ + # --- skills --- + + @cfg_authorized + Scenario: Skills list returns 401 when not authenticated + Given The service uses the lightspeed-stack-authorized.yaml configuration + And The service is restarted + When I access REST API endpoint "skills" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "No Authorization header found" + } + } + """ + # --- prompts --- + + @cfg_authorized Scenario: Prompts list returns 401 when not authenticated - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I access REST API endpoint "prompts" using HTTP GET method Then The status code of the response is 401 @@ -141,8 +175,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Prompts create returns 401 when not authenticated - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I access REST API endpoint "prompts" using HTTP POST method """ @@ -151,8 +187,10 @@ Feature: HTTP 401 Unauthorized Then The status code of the response is 401 And The body of the response contains No Authorization header found + + @cfg_authorized Scenario: Prompts get by id returns 401 when not authenticated - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I access REST API endpoint "prompts/pmpt_5c76d7f7c633ef97477adeb2f642150d8d08e8a6526e9909" using HTTP GET method Then The status code of the response is 401 @@ -166,8 +204,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Prompts update returns 401 when not authenticated - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I access REST API endpoint "prompts/pmpt_5c76d7f7c633ef97477adeb2f642150d8d08e8a6526e9909" using HTTP PUT method """ @@ -176,8 +216,10 @@ Feature: HTTP 401 Unauthorized Then The status code of the response is 401 And The body of the response contains No Authorization header found + + @cfg_authorized Scenario: Prompts delete returns 401 when not authenticated - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I access REST API endpoint "prompts/pmpt_5c76d7f7c633ef97477adeb2f642150d8d08e8a6526e9909" using HTTP DELETE method Then The status code of the response is 401 @@ -193,8 +235,10 @@ Feature: HTTP 401 Unauthorized # --- authorized (noop token) --- + + @cfg_authorized Scenario: Check if the authorized endpoint fails when user_id and auth header are not provided - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I access endpoint "authorized" using HTTP POST method """ @@ -211,8 +255,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Check if the authorized endpoint works with proper user_id but bearer token is not present - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I access endpoint "authorized" using HTTP POST method with user_id "test_user" Then The status code of the response is 401 @@ -226,8 +272,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Check if the authorized endpoint works when auth token is malformed - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I access endpoint "authorized" using HTTP POST method with user_id "test_user" Then The status code of the response is 401 @@ -243,8 +291,10 @@ Feature: HTTP 401 Unauthorized # --- rlsapi v1 --- + + @cfg_authorized Scenario: Request without authorization returns 401 (rlsapi infer) - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I use "infer" to ask question """ @@ -261,8 +311,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Request with empty bearer token returns 401 (rlsapi infer) - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I use "infer" to ask question """ @@ -271,26 +323,12 @@ Feature: HTTP 401 Unauthorized Then The status code of the response is 401 And The body of the response contains No Authorization header found - # --- rh-identity --- - - Scenario: Request fails when x-rh-identity header is missing (rh-identity) - Given The service uses the lightspeed-stack-auth-rh-identity.yaml configuration - And The service is restarted - And I remove the auth header - When I access endpoint "authorized" using HTTP POST method - """ - {"placeholder":"abc"} - """ - Then The status code of the response is 401 - And The body of the response is the following - """ - {"detail": "Missing x-rh-identity header"} - """ - # --- RBAC --- + + @cfg_authorized Scenario: Request without token returns 401 (RBAC) - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I remove the auth header When I access REST API endpoint "models" using HTTP GET method @@ -305,17 +343,12 @@ Feature: HTTP 401 Unauthorized } """ - Scenario: Request with malformed Authorization header returns 401 (RBAC) - Given The service uses the lightspeed-stack-rbac.yaml configuration - And The service is restarted - And I set the Authorization header to NotBearer sometoken - When I access REST API endpoint "models" using HTTP GET method - Then The status code of the response is 401 # --- conversation cache v2 --- + @cfg_authorized Scenario: V2 conversations endpoint fails when auth header is not present - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given REST API service prefix is /v2 And I remove the auth header @@ -331,8 +364,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: V2 conversations/{conversation_id} endpoint fails when auth header is not present - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given REST API service prefix is /v1 And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva @@ -356,8 +391,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: V2 conversations/{conversation_id} DELETE endpoint fails when auth header is not present - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given REST API service prefix is /v1 And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva @@ -381,8 +418,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: V2 conversations PUT endpoint fails when auth header is not present - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given REST API service prefix is /v1 And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva @@ -408,8 +447,10 @@ Feature: HTTP 401 Unauthorized # --- responses --- + + @cfg_authorized Scenario: Responses returns error when not authenticated - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given The system is in default state When I use "responses" to ask question @@ -427,8 +468,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Responses returns error when bearer token is missing - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given The system is in default state And I set the Authorization header to Bearer @@ -441,8 +484,10 @@ Feature: HTTP 401 Unauthorized # --- responses streaming --- + + @cfg_authorized Scenario: Streaming responses returns error when not authenticated - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I use "responses" to ask question """ @@ -459,8 +504,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Streaming responses returns error when bearer token is missing - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted When I use "responses" to ask question """ @@ -471,8 +518,10 @@ Feature: HTTP 401 Unauthorized # --- feedback --- + + @cfg_authorized Scenario: Check if feedback endpoint is not working when not authorized - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And A new conversation is initialized @@ -497,8 +546,10 @@ Feature: HTTP 401 Unauthorized } """ + + @cfg_authorized Scenario: Check if update feedback status endpoint is not working when not authorized - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I remove the auth header When The feedback is enabled @@ -511,4 +562,34 @@ Feature: HTTP 401 Unauthorized "cause": "No Authorization header found" } } - """ \ No newline at end of file + """ +# --- @cfg_rh_identity --- + + @cfg_rh_identity + Scenario: Request fails when x-rh-identity header is missing (rh-identity) + Given The service uses the lightspeed-stack-rh-identity.yaml configuration + And The service is restarted + And I remove the auth header + When I access endpoint "authorized" using HTTP POST method + """ + {"placeholder":"abc"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + {"detail": "Missing x-rh-identity header"} + """ + + # --- RBAC --- + + +# --- @cfg_rbac --- + + @cfg_rbac + Scenario: Request with malformed Authorization header returns 401 (RBAC) + Given The service uses the lightspeed-stack-rbac.yaml configuration + And The service is restarted + And I set the Authorization header to NotBearer sometoken + When I access REST API endpoint "models" using HTTP GET method + Then The status code of the response is 401 + diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index 9922c3d96..c5724fda7 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -1,4 +1,4 @@ -@e2e_group_3 +@cfg_default Feature: Info tests @@ -7,7 +7,7 @@ Feature: Info tests And The system is in default state And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack.yaml configuration + And The service uses the lightspeed-stack-default.yaml configuration And The service is restarted Scenario: Check if the OpenAPI endpoint works as expected diff --git a/tests/e2e/features/inline_rag.feature b/tests/e2e/features/inline_rag.feature index d22a02ce1..762f44d0f 100644 --- a/tests/e2e/features/inline_rag.feature +++ b/tests/e2e/features/inline_rag.feature @@ -1,4 +1,4 @@ -@e2e_group_3 +@cfg_default Feature: Inline RAG (BYOK) support tests Background: @@ -7,7 +7,7 @@ Feature: Inline RAG (BYOK) support tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-inline-rag.yaml configuration + And The service uses the lightspeed-stack-default.yaml configuration And The service is restarted Scenario: Check if inline RAG source is registered diff --git a/tests/e2e/features/llama_stack_disrupted.feature b/tests/e2e/features/llama_stack_disrupted.feature index 5d63e82c6..6e0c2e68d 100644 --- a/tests/e2e/features/llama_stack_disrupted.feature +++ b/tests/e2e/features/llama_stack_disrupted.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @skip-in-library-mode @Authorized +@skip-in-library-mode @Authorized Feature: Llama Stack connection disrupted End-to-end scenarios that stop the Llama Stack container (or simulate disconnect) and @@ -13,9 +13,13 @@ Feature: Llama Stack connection disrupted And the Lightspeed stack configuration directory is "tests/e2e/configuration" - # --- lightspeed-stack.yaml (aligned with health, info, models, …) --- + # --- lightspeed-stack-default.yaml (aligned with health, info, models, …) --- + +# --- @cfg_default --- + + @cfg_default Scenario: Check if models endpoint reports error when llama-stack is unreachable - Given The service uses the lightspeed-stack.yaml configuration + Given The service uses the lightspeed-stack-default.yaml configuration And The service is restarted Given The system is in default state And The llama-stack connection is disrupted @@ -26,8 +30,10 @@ Feature: Llama Stack connection disrupted {"detail": {"response": "Unable to connect to OGX", "cause": "Connection error."}} """ + + @cfg_default Scenario: Check if service report proper readiness state when llama stack is not available - Given The service uses the lightspeed-stack.yaml configuration + Given The service uses the lightspeed-stack-default.yaml configuration And The service is restarted Given The system is in default state And The llama-stack connection is disrupted @@ -38,8 +44,10 @@ Feature: Llama Stack connection disrupted {"ready": false, "reason": "Cannot connect to backend service", "overall_status": "unhealthy", "impacts": ["LLM inference unavailable", "Provider health checks unavailable"]} """ + + @cfg_default Scenario: Check if service report proper liveness state even when llama stack is not available - Given The service uses the lightspeed-stack.yaml configuration + Given The service uses the lightspeed-stack-default.yaml configuration And The service is restarted Given The system is in default state And The llama-stack connection is disrupted @@ -50,8 +58,10 @@ Feature: Llama Stack connection disrupted {"alive": true} """ + + @cfg_default Scenario: Check if info endpoint reports error when llama-stack connection is not working - Given The service uses the lightspeed-stack.yaml configuration + Given The service uses the lightspeed-stack-default.yaml configuration And The service is restarted And The llama-stack connection is disrupted When I access REST API endpoint "info" using HTTP GET method @@ -61,8 +71,12 @@ Feature: Llama Stack connection disrupted {"detail": {"response": "Unable to connect to OGX", "cause": "Connection error."}} """ + +# --- @cfg_default (noop auth; tools list needs no bearer) --- + + @cfg_default Scenario: Check if tools endpoint reports error when llama-stack is unreachable - Given The service uses the lightspeed-stack.yaml configuration + Given The service uses the lightspeed-stack-default.yaml configuration And The service is restarted And The llama-stack connection is disrupted When I access REST API endpoint "tools" using HTTP GET method @@ -73,10 +87,12 @@ Feature: Llama Stack connection disrupted """ - # --- lightspeed-stack-auth-noop-token.yaml (aligned with query, responses, conversations, …) --- + # --- lightspeed-stack-authorized.yaml (aligned with query, responses, conversations, …) --- + + @cfg_authorized Scenario: Check if LLM responds for query request with error for inability to connect to llama-stack Given Llama Stack is restarted - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And The llama-stack connection is disrupted @@ -87,8 +103,10 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Responses returns error when unable to connect to llama-stack - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Given The system is in default state And The llama-stack connection is disrupted @@ -100,8 +118,10 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Streaming responses returns error when unable to connect to llama-stack - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And The llama-stack connection is disrupted @@ -112,8 +132,10 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Check if rags endpoint fails when llama-stack is unavailable - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And The llama-stack connection is disrupted @@ -121,8 +143,10 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Check if prompts list endpoint fails when llama-stack is unavailable - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And The llama-stack connection is disrupted @@ -130,8 +154,10 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Check if prompts create endpoint fails when llama-stack is unavailable - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And The llama-stack connection is disrupted @@ -142,8 +168,10 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Check if prompts get by id endpoint fails when llama-stack is unavailable - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And The llama-stack connection is disrupted @@ -151,8 +179,10 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Check if prompts update endpoint fails when llama-stack is unavailable - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And The llama-stack connection is disrupted @@ -163,8 +193,10 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Check if prompts delete endpoint fails when llama-stack is unavailable - Given The service uses the lightspeed-stack-auth-noop-token.yaml configuration + Given The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And The llama-stack connection is disrupted @@ -172,9 +204,11 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Check if conversations/{conversation_id} GET endpoint fails when llama-stack is unavailable Given Llama Stack is restarted - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And I use "query" to ask question with authorization header @@ -188,9 +222,11 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Check if conversations/{conversation_id} DELETE endpoint fails when llama-stack is unavailable Given Llama Stack is restarted - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And I use "query" to ask question with authorization header @@ -204,9 +240,11 @@ Feature: Llama Stack connection disrupted Then The status code of the response is 503 And The body of the response contains Unable to connect to OGX + + @cfg_authorized Scenario: Check conversations/{conversation_id} works when llama-stack is down Given Llama Stack is restarted - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 @@ -226,9 +264,13 @@ Feature: Llama Stack connection disrupted And The conversation history has correct metadata And The conversation uses model {MODEL} and provider {PROVIDER} + +# --- still @cfg_authorized (noop-with-token; not RBAC) --- + + @cfg_authorized Scenario: V2 conversations DELETE endpoint works even when llama-stack is down Given Llama Stack is restarted - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 @@ -253,7 +295,8 @@ Feature: Llama Stack connection disrupted # --- lightspeed-stack-rbac.yaml (aligned with rbac.feature / rlsapi_v1_errors.feature) --- - @RBAC + + @RBAC @cfg_rbac Scenario: Returns 503 when llama-stack connection is broken Given Llama Stack is restarted And The service uses the lightspeed-stack-rbac.yaml configuration diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature index d1c722b81..4f66fcb5b 100644 --- a/tests/e2e/features/mcp.feature +++ b/tests/e2e/features/mcp.feature @@ -1,4 +1,3 @@ -@e2e_group_2 Feature: MCP tests Background: @@ -7,9 +6,10 @@ Feature: MCP tests And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" +# Per-auth single-server configs (@cfg_mcp). Cannot share one multi-server YAML: +# check_mcp_auth probes every configured MCP server. lightspeed-stack-mcp.yaml is for mcp_servers_api. -# File-based (valid token) — lightspeed-stack-mcp-file-auth.yaml - @MCPFileAuthConfig + @MCPFileAuthConfig @cfg_mcp Scenario: Check if tools endpoint succeeds when MCP file-based auth token is passed Given MCP configuration is reset for a new scenario And The service uses the lightspeed-stack-mcp-file-auth.yaml configuration @@ -19,10 +19,10 @@ Feature: MCP tests Then The status code of the response is 200 And The body of the response contains mcp-file - @MCPFileAuthConfig @flaky + + @MCPFileAuthConfig @flaky @cfg_mcp Scenario: Check if query endpoint succeeds when MCP file-based auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-file-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-file-auth.yaml configuration And The service is restarted And The mcp-file mcp server Authorization header is set to "/tmp/mcp-token" And I capture the current token metrics @@ -36,10 +36,10 @@ Feature: MCP tests | Hello | And The token metrics have increased - @MCPFileAuthConfig @flaky + + @MCPFileAuthConfig @flaky @cfg_mcp Scenario: Check if streaming_query endpoint succeeds when MCP file-based auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-file-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-file-auth.yaml configuration And The service is restarted And The mcp-file mcp server Authorization header is set to "/tmp/mcp-token" And I capture the current token metrics @@ -54,71 +54,10 @@ Feature: MCP tests | Hello | And The token metrics have increased -# File-based (invalid token) — lightspeed-stack-invalid-mcp-file-auth.yaml - @InvalidMCPFileAuthConfig - Scenario: Check if tools endpoint reports error when MCP file-based invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-invalid-mcp-file-auth.yaml configuration - And The service is restarted - And The mcp-file mcp server Authorization header is set to "/tmp/invalid-mcp-token" - When I access REST API endpoint "tools" using HTTP GET method - Then The status code of the response is 401 - And The body of the response is the following - """ - { - "detail": { - "response": "Missing or invalid credentials provided by client", - "cause": "MCP server at http://mock-mcp:3000 requires OAuth" - } - } - """ - - @InvalidMCPFileAuthConfig - Scenario: Check if query endpoint reports error when MCP file-based invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-invalid-mcp-file-auth.yaml configuration - And The service is restarted - And The mcp-file mcp server Authorization header is set to "/tmp/invalid-mcp-token" - When I use "query" to ask question - """ - {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 401 - And The body of the response is the following - """ - { - "detail": { - "response": "Missing or invalid credentials provided by client", - "cause": "MCP server at http://mock-mcp:3000 requires OAuth" - } - } - """ - @InvalidMCPFileAuthConfig - Scenario: Check if streaming_query endpoint reports error when MCP file-based invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-invalid-mcp-file-auth.yaml configuration - And The service is restarted - And The mcp-file mcp server Authorization header is set to "/tmp/invalid-mcp-token" - When I use "streaming_query" to ask question - """ - {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 401 - And The body of the response is the following - """ - { - "detail": { - "response": "Missing or invalid credentials provided by client", - "cause": "MCP server at http://mock-mcp:3000 requires OAuth" - } - } - """ - -# Kubernetes — lightspeed-stack-mcp-kubernetes-auth.yaml (success paths then invalid token) - @MCPKubernetesAuthConfig + @MCPKubernetesAuthConfig @cfg_mcp Scenario: Check if tools endpoint succeeds when MCP kubernetes auth token is passed - Given MCP configuration is reset for a new scenario + Given MCP configuration is reset for a new scenario And The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration And The service is restarted And I set the Authorization header to Bearer kubernetes-test-token @@ -126,10 +65,10 @@ Feature: MCP tests Then The status code of the response is 200 And The body of the response contains mcp-kubernetes - @MCPKubernetesAuthConfig @flaky + + @MCPKubernetesAuthConfig @flaky @cfg_mcp Scenario: Check if query endpoint succeeds when MCP kubernetes auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration And The service is restarted And I set the Authorization header to Bearer kubernetes-test-token And I capture the current token metrics @@ -143,10 +82,10 @@ Feature: MCP tests | Hello | And The token metrics have increased - @MCPKubernetesAuthConfig @flaky + + @MCPKubernetesAuthConfig @flaky @cfg_mcp Scenario: Check if streaming_query endpoint succeeds when MCP kubernetes auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration And The service is restarted And I set the Authorization header to Bearer kubernetes-test-token And I capture the current token metrics @@ -161,10 +100,10 @@ Feature: MCP tests | Hello | And The token metrics have increased - @MCPKubernetesAuthConfig + + @MCPKubernetesAuthConfig @cfg_mcp Scenario: Check if tools endpoint reports error when MCP kubernetes invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration And The service is restarted And I set the Authorization header to Bearer kubernetes-invalid-token When I access REST API endpoint "tools" using HTTP GET method @@ -179,10 +118,10 @@ Feature: MCP tests } """ - @MCPKubernetesAuthConfig + + @MCPKubernetesAuthConfig @cfg_mcp Scenario: Check if query endpoint reports error when MCP kubernetes invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration And The service is restarted And I set the Authorization header to Bearer kubernetes-invalid-token When I use "query" to ask question with authorization header @@ -200,10 +139,10 @@ Feature: MCP tests } """ - @MCPKubernetesAuthConfig + + @MCPKubernetesAuthConfig @cfg_mcp Scenario: Check if streaming_query endpoint reports error when MCP kubernetes invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-kubernetes-auth.yaml configuration And The service is restarted And I set the Authorization header to Bearer kubernetes-invalid-token When I use "streaming_query" to ask question with authorization header @@ -221,10 +160,10 @@ Feature: MCP tests } """ -# Client-provided — lightspeed-stack-mcp-clientauth.yaml -@MCPClientAuthConfig + +@MCPClientAuthConfig @cfg_mcp Scenario: Check if tools endpoint succeeds when MCP client-provided auth token is passed - Given MCP configuration is reset for a new scenario + Given MCP configuration is reset for a new scenario And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to @@ -235,10 +174,10 @@ Feature: MCP tests Then The status code of the response is 200 And The body of the response contains mcp-client - @MCPClientAuthConfig @flaky + + @MCPClientAuthConfig @flaky @cfg_mcp Scenario: Check if query endpoint succeeds when MCP client-provided auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -255,10 +194,10 @@ Feature: MCP tests | Hello | And The token metrics have increased - @MCPClientAuthConfig @flaky + + @MCPClientAuthConfig @flaky @cfg_mcp Scenario: Check if streaming_query endpoint succeeds when MCP client-provided auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -276,19 +215,19 @@ Feature: MCP tests | Hello | And The token metrics have increased - @MCPClientAuthConfig + + @MCPClientAuthConfig @cfg_mcp Scenario: Check if tools endpoint succeeds by skipping when MCP client-provided auth token is omitted - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 200 And The body of the response does not contain mcp-client - @MCPClientAuthConfig @flaky + + @MCPClientAuthConfig @flaky @cfg_mcp Scenario: Check if query endpoint succeeds by skipping when MCP client-provided auth token is omitted - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted And I capture the current token metrics When I use "query" to ask question @@ -302,10 +241,10 @@ Feature: MCP tests | Hello | And The token metrics have increased - @MCPClientAuthConfig @flaky + + @MCPClientAuthConfig @flaky @cfg_mcp Scenario: Check if streaming_query endpoint succeeds by skipping when MCP client-provided auth token is omitted - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted And I capture the current token metrics When I use "streaming_query" to ask question @@ -320,10 +259,10 @@ Feature: MCP tests | Hello | And The token metrics have increased - @MCPClientAuthConfig + + @MCPClientAuthConfig @cfg_mcp Scenario: Check if tools endpoint reports error when MCP client-provided invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -341,10 +280,10 @@ Feature: MCP tests } """ - @MCPClientAuthConfig + + @MCPClientAuthConfig @cfg_mcp Scenario: Check if query endpoint reports error when MCP client-provided invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -365,10 +304,10 @@ Feature: MCP tests } """ - @MCPClientAuthConfig + + @MCPClientAuthConfig @cfg_mcp Scenario: Check if streaming_query endpoint reports error when MCP client-provided invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -389,10 +328,10 @@ Feature: MCP tests } """ -# OAuth — lightspeed-stack-mcp-oauth-auth.yaml (valid token, then unauthenticated, then invalid token) - @MCPOAuthAuthConfig + + @MCPOAuthAuthConfig @cfg_mcp Scenario: Check if tools endpoint succeeds when MCP OAuth auth token is passed - Given MCP configuration is reset for a new scenario + Given MCP configuration is reset for a new scenario And The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to @@ -403,10 +342,10 @@ Feature: MCP tests Then The status code of the response is 200 And The body of the response contains mcp-oauth - @MCPOAuthAuthConfig @flaky + + @MCPOAuthAuthConfig @flaky @cfg_mcp Scenario: Check if query endpoint succeeds when MCP OAuth auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -423,10 +362,10 @@ Feature: MCP tests | Hello | And The token metrics have increased - @MCPOAuthAuthConfig @flaky + + @MCPOAuthAuthConfig @flaky @cfg_mcp Scenario: Check if streaming_query endpoint succeeds when MCP OAuth auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -444,10 +383,10 @@ Feature: MCP tests | Hello | And The token metrics have increased - @MCPOAuthAuthConfig + + @MCPOAuthAuthConfig @cfg_mcp Scenario: Check if tools endpoint reports error when MCP OAuth requires authentication - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration And The service is restarted When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 401 @@ -462,10 +401,10 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - @MCPOAuthAuthConfig + + @MCPOAuthAuthConfig @cfg_mcp Scenario: Check if query endpoint reports error when MCP OAuth requires authentication - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration And The service is restarted When I use "query" to ask question """ @@ -483,10 +422,10 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - @MCPOAuthAuthConfig + + @MCPOAuthAuthConfig @cfg_mcp Scenario: Check if streaming_query endpoint reports error when MCP OAuth requires authentication - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration And The service is restarted When I use "streaming_query" to ask question """ @@ -504,10 +443,10 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - @MCPOAuthAuthConfig + + @MCPOAuthAuthConfig @cfg_mcp Scenario: Check if tools endpoint reports error when MCP OAuth invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -526,10 +465,10 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - @MCPOAuthAuthConfig + + @MCPOAuthAuthConfig @cfg_mcp Scenario: Check if query endpoint reports error when MCP OAuth invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -551,10 +490,10 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - @MCPOAuthAuthConfig + + @MCPOAuthAuthConfig @cfg_mcp Scenario: Check if streaming_query endpoint reports error when MCP OAuth invalid auth token is passed - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration + Given The service uses the lightspeed-stack-mcp-oauth-auth.yaml configuration And The service is restarted And I set the "MCP-HEADERS" header to """ @@ -576,11 +515,79 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" + + @cfg_mcp Scenario: Check if MCP client auth options endpoint is working - Given MCP configuration is reset for a new scenario - And The service uses the lightspeed-stack-mcp.yaml configuration + Given MCP configuration is reset for a new scenario + And The service uses the lightspeed-stack-mcp-client-auth.yaml configuration And The service is restarted When I access REST API endpoint "mcp-auth/client-options" using HTTP GET method Then The status code of the response is 200 And The body of the response has proper client auth options structure And The response contains server "mcp-client" with client auth header "Authorization" + +# Invalid MCP file token uses lightspeed-stack-mcp-invalid.yaml (@cfg_mcp_invalid) + + @InvalidMCPFileAuthConfig @cfg_mcp_invalid + Scenario: Check if tools endpoint reports error when MCP file-based invalid auth token is passed + Given MCP configuration is reset for a new scenario + And The service uses the lightspeed-stack-mcp-invalid.yaml configuration + And The service is restarted + And The mcp-file mcp server Authorization header is set to "/tmp/invalid-mcp-token" + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + } + } + """ + + + @InvalidMCPFileAuthConfig @cfg_mcp_invalid + Scenario: Check if query endpoint reports error when MCP file-based invalid auth token is passed + Given MCP configuration is reset for a new scenario + And The service uses the lightspeed-stack-mcp-invalid.yaml configuration + And The service is restarted + And The mcp-file mcp server Authorization header is set to "/tmp/invalid-mcp-token" + When I use "query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + } + } + """ + + + @InvalidMCPFileAuthConfig @cfg_mcp_invalid + Scenario: Check if streaming_query endpoint reports error when MCP file-based invalid auth token is passed + Given MCP configuration is reset for a new scenario + And The service uses the lightspeed-stack-mcp-invalid.yaml configuration + And The service is restarted + And The mcp-file mcp server Authorization header is set to "/tmp/invalid-mcp-token" + When I use "streaming_query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3000 requires OAuth" + } + } + """ + + diff --git a/tests/e2e/features/mcp_servers_api.feature b/tests/e2e/features/mcp_servers_api.feature index cb3e85dc3..2d7715ad4 100644 --- a/tests/e2e/features/mcp_servers_api.feature +++ b/tests/e2e/features/mcp_servers_api.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @MCP +@cfg_mcp @MCP Feature: MCP Server Management API tests Tests for the dynamic MCP server management endpoints: diff --git a/tests/e2e/features/mcp_servers_api_auth.feature b/tests/e2e/features/mcp_servers_api_auth.feature index 54e0d7201..26bebbae3 100644 --- a/tests/e2e/features/mcp_servers_api_auth.feature +++ b/tests/e2e/features/mcp_servers_api_auth.feature @@ -1,4 +1,4 @@ -@e2e_group_1 @MCPServerAPIAuth +@cfg_mcp_api_auth @MCPServerAPIAuth Feature: MCP Server Management API authentication tests Tests that the MCP server management endpoints enforce authentication @@ -9,7 +9,7 @@ Feature: MCP Server Management API authentication tests And The system is in default state And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-mcp-auth.yaml configuration + And The service uses the lightspeed-stack-mcp-api-auth.yaml configuration And The service is restarted Scenario: List MCP servers returns 401 without auth token diff --git a/tests/e2e/features/mcp_servers_api_no_config.feature b/tests/e2e/features/mcp_servers_api_no_config.feature index 6dfe0e7b0..41d68e802 100644 --- a/tests/e2e/features/mcp_servers_api_no_config.feature +++ b/tests/e2e/features/mcp_servers_api_no_config.feature @@ -1,8 +1,8 @@ -@e2e_group_1 @MCPNoConfig +@cfg_negative @MCPNoConfig Feature: MCP Server API tests without configured MCP servers Tests that the MCP server management endpoints work correctly - when no MCP servers are configured in lightspeed-stack.yaml. + when no MCP servers are configured (lightspeed-stack-negative.yaml). Background: Given The service is started locally @@ -10,7 +10,7 @@ Feature: MCP Server API tests without configured MCP servers And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" And I set the Authorization header to Bearer mcp-e2e-no-config-token - And The service uses the lightspeed-stack-no-cache.yaml configuration + And The service uses the lightspeed-stack-negative.yaml configuration And The service is restarted Scenario: List MCP servers returns empty list when none configured diff --git a/tests/e2e/features/models.feature b/tests/e2e/features/models.feature index 804f0183a..01dcea9c9 100644 --- a/tests/e2e/features/models.feature +++ b/tests/e2e/features/models.feature @@ -1,4 +1,4 @@ -@e2e_group_2 +@cfg_default Feature: Models endpoint tests @@ -7,7 +7,7 @@ Feature: Models endpoint tests And The system is in default state And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack.yaml configuration + And The service uses the lightspeed-stack-default.yaml configuration And The service is restarted diff --git a/tests/e2e/features/opentelemetry.feature b/tests/e2e/features/opentelemetry.feature index c236c9016..8d8f79a59 100644 --- a/tests/e2e/features/opentelemetry.feature +++ b/tests/e2e/features/opentelemetry.feature @@ -1,4 +1,4 @@ -@e2e_group_1 @OTel @skip +@cfg_authorized @OTel @skip Feature: OpenTelemetry observability tests Background: @@ -9,7 +9,7 @@ Feature: OpenTelemetry observability tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted diff --git a/tests/e2e/features/prompts.feature b/tests/e2e/features/prompts.feature index 7ea93dd80..f9a278c7f 100644 --- a/tests/e2e/features/prompts.feature +++ b/tests/e2e/features/prompts.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @Authorized +@cfg_authorized @Authorized Feature: Prompts API endpoint tests Background: @@ -7,7 +7,7 @@ Feature: Prompts API endpoint tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted # --- 200 OK --- diff --git a/tests/e2e/features/proxy.feature b/tests/e2e/features/proxy.feature index 00fde258a..c93d66390 100644 --- a/tests/e2e/features/proxy.feature +++ b/tests/e2e/features/proxy.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @skip-in-library-mode @skip-in-prow +@cfg_default @skip-in-library-mode @skip-in-prow Feature: Proxy and TLS networking tests for Llama Stack providers Verify that the Lightspeed Stack works correctly when Llama Stack's @@ -14,7 +14,7 @@ Feature: Proxy and TLS networking tests for Llama Stack providers And The system is in default state And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack.yaml configuration + And The service uses the lightspeed-stack-default.yaml configuration And The service is restarted And The original Llama Stack config is restored if modified diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index f6a20ef04..c0207315c 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @Authorized +@cfg_authorized @Authorized Feature: Query endpoint API tests Background: @@ -7,7 +7,7 @@ Feature: Query endpoint API tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted @flaky diff --git a/tests/e2e/features/rbac.feature b/tests/e2e/features/rbac.feature index 07d711ddb..41c3b834b 100644 --- a/tests/e2e/features/rbac.feature +++ b/tests/e2e/features/rbac.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @RBAC +@cfg_rbac @RBAC Feature: Role-Based Access Control (RBAC) Comprehensive tests for role-based access control to ensure diff --git a/tests/e2e/features/responses.feature b/tests/e2e/features/responses.feature index 59672e2fe..8908629d5 100644 --- a/tests/e2e/features/responses.feature +++ b/tests/e2e/features/responses.feature @@ -1,4 +1,4 @@ -@e2e_group_1 @Authorized +@cfg_authorized @Authorized Feature: Responses endpoint API tests Background: @@ -7,7 +7,7 @@ Feature: Responses endpoint API tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted diff --git a/tests/e2e/features/responses_streaming.feature b/tests/e2e/features/responses_streaming.feature index ddd1129ae..b733ea70c 100644 --- a/tests/e2e/features/responses_streaming.feature +++ b/tests/e2e/features/responses_streaming.feature @@ -1,4 +1,4 @@ -@e2e_group_1 @Authorized +@cfg_authorized @Authorized Feature: Responses endpoint streaming API tests # Same coverage as ``responses.feature`` with ``stream=true`` (SSE for success paths; @@ -9,7 +9,7 @@ Feature: Responses endpoint streaming API tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Scenario: Streaming responses returns 200 for minimal request diff --git a/tests/e2e/features/rest_api.feature b/tests/e2e/features/rest_api.feature index a40bfd248..9a4b67363 100644 --- a/tests/e2e/features/rest_api.feature +++ b/tests/e2e/features/rest_api.feature @@ -1,4 +1,4 @@ -@e2e_group_1 +@cfg_default Feature: REST API tests @@ -7,7 +7,7 @@ Feature: REST API tests And The system is in default state And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack.yaml configuration + And The service uses the lightspeed-stack-default.yaml configuration And The service is restarted Scenario: Check if the OpenAPI endpoint works as expected diff --git a/tests/e2e/features/rlsapi_v1.feature b/tests/e2e/features/rlsapi_v1.feature index 31190b454..ef14186b1 100644 --- a/tests/e2e/features/rlsapi_v1.feature +++ b/tests/e2e/features/rlsapi_v1.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @Authorized +@cfg_authorized @Authorized Feature: rlsapi v1 /infer endpoint API tests Background: @@ -7,7 +7,7 @@ Feature: rlsapi v1 /infer endpoint API tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Scenario: Basic inference with minimal request (question only) diff --git a/tests/e2e/features/rlsapi_v1_errors.feature b/tests/e2e/features/rlsapi_v1_errors.feature index fe4fedd41..9ab9a05f8 100644 --- a/tests/e2e/features/rlsapi_v1_errors.feature +++ b/tests/e2e/features/rlsapi_v1_errors.feature @@ -1,4 +1,4 @@ -@e2e_group_1 @RBAC +@cfg_rbac @RBAC Feature: rlsapi v1 /infer endpoint error response tests Tests for error conditions on the rlsapi v1 /infer endpoint including diff --git a/tests/e2e/features/skills.feature b/tests/e2e/features/skills.feature index 3914a8872..9d7613a4b 100644 --- a/tests/e2e/features/skills.feature +++ b/tests/e2e/features/skills.feature @@ -1,4 +1,3 @@ -@e2e_group_2 Feature: Agent skills tests Background: @@ -7,9 +6,9 @@ Feature: Agent skills tests And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - # --- Skill tools registration --- +# Echo skill path (@cfg_skills) - @SkillsConfig + @SkillsConfig @cfg_skills Scenario: Skill tools are registered when skills are configured Given The service uses the lightspeed-stack-skills.yaml configuration And MCP configuration is reset for a new scenario @@ -138,49 +137,10 @@ Feature: Agent skills tests } """ - Scenario: Skill tools are not registered when no skills are configured - Given The service uses the lightspeed-stack.yaml configuration - And MCP configuration is reset for a new scenario - And The service is restarted - When I access REST API endpoint "tools" using HTTP GET method - Then The status code of the response is 200 - And The body of the response is the following - """ - { - "tools": [ - { - "identifier": "insert_into_memory", - "description": "Insert documents into memory", - "parameters": [], - "provider_id": "file-search", - "toolgroup_id": "builtin::file_search", - "server_source": "builtin", - "type": "tool" - }, - { - "identifier": "file_search", - "description": "Search files for relevant information", - "parameters": [ - { - "name": "query", - "description": "The query to search for. Can be a natural language sentence or keywords.", - "parameter_type": "string", - "required": true, - "default": null - } - ], - "provider_id": "file-search", - "toolgroup_id": "builtin::file_search", - "server_source": "builtin", - "type": "tool" - } - ] - } - """ # --- Skill discovery --- - @SkillsConfig + @SkillsConfig @cfg_skills Scenario: LLM can discover skills via list_skills tool using query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -211,7 +171,8 @@ Feature: Agent skills tests """ And The token metrics have increased - @SkillsConfig + + @SkillsConfig @cfg_skills Scenario: LLM can discover skills via list_skills tool using streaming_query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -245,7 +206,8 @@ Feature: Agent skills tests # --- Skill activation --- - @SkillsConfig @flaky + + @SkillsConfig @flaky @cfg_skills Scenario: LLM can Load a skill and use its instructions via query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -279,7 +241,8 @@ Feature: Agent skills tests """ And The token metrics have increased - @SkillsConfig @flaky + + @SkillsConfig @flaky @cfg_skills Scenario: LLM can load a skill and use its instructions via streaming_query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -317,7 +280,8 @@ Feature: Agent skills tests # --- Skill resource loading --- - @SkillsConfig + + @SkillsConfig @cfg_skills Scenario: LLM can load a skill reference file via read_skill_resource tool using query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -352,7 +316,8 @@ Feature: Agent skills tests """ And The token metrics have increased - @SkillsConfig + + @SkillsConfig @cfg_skills Scenario: LLM can load a skill reference file via read_skill_resource tool using streaming_query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -390,7 +355,8 @@ Feature: Agent skills tests # --- Error handling: unknown skill --- - @SkillsConfig @skip + + @SkillsConfig @skip @cfg_skills Scenario: load_skill returns error for unknown skill name via query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -422,7 +388,8 @@ Feature: Agent skills tests """ - @SkillsConfig @skip + + @SkillsConfig @skip @cfg_skills Scenario: load_skill returns error for unknown skill name via streaming_query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -455,7 +422,8 @@ Feature: Agent skills tests """ # --- Error handling: missing resource --- - @SkillsConfig @skip + + @SkillsConfig @skip @cfg_skills Scenario: read_skill_resource returns error for nonexistent resource file via query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -487,7 +455,8 @@ Feature: Agent skills tests ] """ - @SkillsConfig @skip + + @SkillsConfig @skip @cfg_skills Scenario: read_skill_resource returns error for nonexistent resource file via streaming_query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -523,7 +492,8 @@ Feature: Agent skills tests # --- Context management: deduplication --- - @SkillsConfig @skip + + @SkillsConfig @skip @cfg_skills Scenario: Duplicate skill activation in same conversation returns already-loaded note via query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -584,70 +554,11 @@ Feature: Agent skills tests """ - # --- Multiple skills --- - - @SkillsMultiConfig - Scenario: Skills directory path discovers all skills in subdirectories via query endpoint - Given The service uses the lightspeed-stack-skills-directory.yaml configuration - And The service is restarted - When I use "query" to ask question - """ - {"query": "List all available skills using the list_skills tool.", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The body of the "tool_calls" field of the response is the following - """ - [ - { - "name": "list_skills", - "type": "function_call" - } - ] - """ - And The body of the "tool_results" field of the response is the following - """ - [ - { - "status": "success", - "content": "{\"echo\":\"Echo back the user's input exactly as provided. Use when a user asks to echo, repeat, or mirror text.\",\"summarize\":\"Summarize text into a concise single-sentence overview. Use when a user asks to summarize, condense, or shorten text.\"}", - "type": "function_call_output" - } - ] - """ - - @SkillsMultiConfig - Scenario: Skills directory path discovers all skills in subdirectories via streaming_query endpoint - Given The service uses the lightspeed-stack-skills-directory.yaml configuration - And The service is restarted - When I use "streaming_query" to ask question - """ - {"query": "List all available skills using the list_skills tool.", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - When I wait for the response to be completed - Then The status code of the response is 200 - And The body of the "tool_calls" field of the response is the following - """ - [ - { - "name": "list_skills", - "type": "function_call" - } - ] - """ - And The body of the "tool_results" field of the response is the following - """ - [ - { - "status": "success", - "content": "{\"echo\":\"Echo back the user's input exactly as provided. Use when a user asks to echo, repeat, or mirror text.\",\"summarize\":\"Summarize text into a concise single-sentence overview. Use when a user asks to summarize, condense, or shorten text.\"}", - "type": "function_call_output" - } - ] - """ - # --- Full progressive disclosure flow --- + @SkillsConfig @skip # TODO: This test is too flaky (should be run on demand) + @SkillsConfig @skip @cfg_skills Scenario: LLM completes list_skills then load_skill then read_skill_resource via query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -706,7 +617,9 @@ Feature: Agent skills tests """ + @SkillsConfig @skip # TODO: This test is too flaky (should be run on demand) + @SkillsConfig @skip @cfg_skills Scenario: LLM completes list_skills then load_skill then read_skill_resource via streaming_query endpoint Given The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted @@ -764,3 +677,110 @@ Feature: Agent skills tests } ] """ + +# --- Multiple skills --- +# --- @cfg_skills_directory: skills/ discovers echo+summarize --- + + @SkillsMultiConfig @cfg_skills_directory + Scenario: Skills directory path discovers all skills in subdirectories via query endpoint + Given The service uses the lightspeed-stack-skills-directory.yaml configuration + And The service is restarted + When I use "query" to ask question + """ + {"query": "List all available skills using the list_skills tool.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the "tool_calls" field of the response is the following + """ + [ + { + "name": "list_skills", + "type": "function_call" + } + ] + """ + And The body of the "tool_results" field of the response is the following + """ + [ + { + "status": "success", + "content": "{\"echo\":\"Echo back the user's input exactly as provided. Use when a user asks to echo, repeat, or mirror text.\",\"summarize\":\"Summarize text into a concise single-sentence overview. Use when a user asks to summarize, condense, or shorten text.\"}", + "type": "function_call_output" + } + ] + """ + + + @SkillsMultiConfig @cfg_skills_directory + Scenario: Skills directory path discovers all skills in subdirectories via streaming_query endpoint + Given The service uses the lightspeed-stack-skills-directory.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question + """ + {"query": "List all available skills using the list_skills tool.", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The body of the "tool_calls" field of the response is the following + """ + [ + { + "name": "list_skills", + "type": "function_call" + } + ] + """ + And The body of the "tool_results" field of the response is the following + """ + [ + { + "status": "success", + "content": "{\"echo\":\"Echo back the user's input exactly as provided. Use when a user asks to echo, repeat, or mirror text.\",\"summarize\":\"Summarize text into a concise single-sentence overview. Use when a user asks to summarize, condense, or shorten text.\"}", + "type": "function_call_output" + } + ] + """ + +# --- @cfg_default: skills disabled --- + + @cfg_default + Scenario: Skill tools are not registered when no skills are configured + Given The service uses the lightspeed-stack-default.yaml configuration + And MCP configuration is reset for a new scenario + And The service is restarted + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response is the following + """ + { + "tools": [ + { + "identifier": "insert_into_memory", + "description": "Insert documents into memory", + "parameters": [], + "provider_id": "file-search", + "toolgroup_id": "builtin::file_search", + "server_source": "builtin", + "type": "tool" + }, + { + "identifier": "file_search", + "description": "Search files for relevant information", + "parameters": [ + { + "name": "query", + "description": "The query to search for. Can be a natural language sentence or keywords.", + "parameter_type": "string", + "required": true, + "default": null + } + ], + "provider_id": "file-search", + "toolgroup_id": "builtin::file_search", + "server_source": "builtin", + "type": "tool" + } + ] + } + """ + diff --git a/tests/e2e/features/smoketests.feature b/tests/e2e/features/smoketests.feature index b7cb61e86..d85ecac36 100644 --- a/tests/e2e/features/smoketests.feature +++ b/tests/e2e/features/smoketests.feature @@ -1,4 +1,4 @@ -@e2e_group_3 +@cfg_default Feature: Smoke tests @@ -7,7 +7,7 @@ Feature: Smoke tests And The system is in default state And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack.yaml configuration + And The service uses the lightspeed-stack-default.yaml configuration And The service is restarted diff --git a/tests/e2e/features/steps/common.py b/tests/e2e/features/steps/common.py index f0b2ccc2f..ec405701b 100644 --- a/tests/e2e/features/steps/common.py +++ b/tests/e2e/features/steps/common.py @@ -16,8 +16,9 @@ ) # Behave may clear user attributes on ``context`` between scenarios; keep the -# last applied config basename here so Background can skip re-applying the same -# YAML across scenarios in one feature. Mutate the dict entry (no global). +# last applied config basename here so ``The service uses ...`` can skip +# re-applying the same YAML across scenarios and across feature files in one +# job (CI shards are config-aligned). Mutate the dict entry (no global). _active_lightspeed_stack_config_basename: dict[str, Optional[str]] = {"basename": None} # Behave clears user attributes on ``context`` between scenarios; store @@ -26,10 +27,19 @@ def reset_active_lightspeed_stack_config_basename() -> None: - """Reset before each feature; see ``environment.before_feature``.""" + """Clear the applied-config basename tracker. + + Used when ``E2E_RESTORE_CONFIG_AFTER_FEATURE=1`` restores bootstrap YAML so + the next configure step does not skip-restart against a stale basename. + """ _active_lightspeed_stack_config_basename["basename"] = None +def get_active_lightspeed_stack_config_basename() -> Optional[str]: + """Return the last applied Lightspeed config basename, if any.""" + return _active_lightspeed_stack_config_basename["basename"] + + def get_llama_stack_hostname() -> str: """Return the Llama Stack hostname surviving per-scenario context clearing.""" return _llama_stack_endpoint["hostname"] @@ -87,13 +97,15 @@ def configure_service(context: Context, config_name: str) -> None: returns immediately: no backup, no copy, and sets ``context.lightspeed_stack_skip_restart`` so the next ``The service is restarted`` step can no-op—except after ``MCP configuration is reset for a new - scenario`` (library mode clears embedded Llama Stack storage), in which case + scenario`` or llama-stack disruption, in which case the restart is not skipped so Lightspeed reloads config and MCP state stays consistent. When the basename differs from the last apply, creates the backup on first use, copies the YAML, updates ``context.feature_config`` / override flags, and - stores the basename for the next check. Cleared in ``before_feature`` so a - new feature file always applies at least once. + stores the basename for the next check. Basename is kept across feature + files so consecutive features that share a ``@cfg_*`` YAML skip restart (CI + shards are config-aligned). Set ``E2E_RESTORE_CONFIG_AFTER_FEATURE=1`` to + restore bootstrap after each feature (legacy; forces re-apply next). Build path from ``lightspeed_stack_config_directory`` (directory step), defaulting base to ``tests/e2e/configuration`` if that step was omitted; then @@ -104,13 +116,11 @@ def configure_service(context: Context, config_name: str) -> None: Parameters: ---------- context (Context): Behave context. - config_name (str): Config filename (e.g. lightspeed-stack-inline-rag.yaml). + config_name (str): Config filename (e.g. lightspeed-stack-default.yaml). """ config_name = config_name.strip() if _active_lightspeed_stack_config_basename["basename"] == config_name: - # ``MCP configuration is reset for a new scenario`` may have run (library: - # clear ``~/.llama``). The next restart must not be skipped or SQLite - # handles / MCP state diverges from the running process. + # MCP reset or llama disrupt: do not skip the next restart. if getattr(context, "force_lightspeed_restart_after_mcp_config_reset", False): context.lightspeed_stack_skip_restart = False context.force_lightspeed_restart_after_mcp_config_reset = False diff --git a/tests/e2e/features/steps/feedback.py b/tests/e2e/features/steps/feedback.py index 0c7b5084a..704277e02 100644 --- a/tests/e2e/features/steps/feedback.py +++ b/tests/e2e/features/steps/feedback.py @@ -168,7 +168,5 @@ def _lightspeed_yaml_path(context: Context, filename: str) -> str: @given("An invalid feedback storage path is configured") # type: ignore[reportCallIssue] def configure_invalid_feedback_storage_path(context: Context) -> None: """Set an invalid feedback storage path and restart the container.""" - switch_config( - _lightspeed_yaml_path(context, "lightspeed-stack-invalid-feedback-storage.yaml") - ) + switch_config(_lightspeed_yaml_path(context, "lightspeed-stack-negative.yaml")) restart_container("lightspeed-stack") diff --git a/tests/e2e/features/steps/health.py b/tests/e2e/features/steps/health.py index dd5243c5a..e4e059ef5 100644 --- a/tests/e2e/features/steps/health.py +++ b/tests/e2e/features/steps/health.py @@ -34,6 +34,12 @@ def reset_llama_stack_disrupt_once_tracking() -> None: _llama_stack_was_running["value"] = False +def _force_lightspeed_restart_after_llama_disrupt(context: Context) -> None: + """Do not skip the next Lightspeed restart after Llama is disrupted.""" + context.force_lightspeed_restart_after_mcp_config_reset = True + context.lightspeed_stack_skip_restart = False + + @given("The llama-stack connection is disrupted") def llama_stack_connection_broken(context: Context) -> None: """Break llama_stack connection by stopping the container. @@ -63,6 +69,7 @@ def llama_stack_connection_broken(context: Context) -> None: """ if _llama_stack_disrupt_once["applied"]: print("Llama Stack disruption skipped (already applied once this feature)") + _force_lightspeed_restart_after_llama_disrupt(context) return # Store original state for restoration (only on the real disruption path). @@ -78,6 +85,7 @@ def llama_stack_connection_broken(context: Context) -> None: context.llama_stack_was_running = was_running _llama_stack_was_running["value"] = was_running _llama_stack_disrupt_once["applied"] = True + _force_lightspeed_restart_after_llama_disrupt(context) return # Docker-based disruption @@ -108,3 +116,4 @@ def llama_stack_connection_broken(context: Context) -> None: return _llama_stack_disrupt_once["applied"] = True + _force_lightspeed_restart_after_llama_disrupt(context) diff --git a/tests/e2e/features/streaming_query.feature b/tests/e2e/features/streaming_query.feature index 332ed0dde..9ba0f68a8 100644 --- a/tests/e2e/features/streaming_query.feature +++ b/tests/e2e/features/streaming_query.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @Authorized +@cfg_authorized @Authorized Feature: streaming_query endpoint API tests Background: @@ -7,7 +7,7 @@ Feature: streaming_query endpoint API tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Scenario: Check if streaming_query response in tokens matches the full response diff --git a/tests/e2e/features/tls-ca.feature b/tests/e2e/features/tls-ca.feature index 75eb9e49b..7ebb9e51a 100644 --- a/tests/e2e/features/tls-ca.feature +++ b/tests/e2e/features/tls-ca.feature @@ -1,4 +1,4 @@ -@e2e_group_1 @skip-in-library-mode @skip-in-prow +@cfg_tls @skip-in-library-mode @skip-in-prow Feature: TLS configuration — CA certificate verification Validate Llama Stack NetworkConfig.tls CA trust settings against the mock HTTPS inference provider (standard TLS port). diff --git a/tests/e2e/features/tls-mtls.feature b/tests/e2e/features/tls-mtls.feature index faceec491..053144bb6 100644 --- a/tests/e2e/features/tls-mtls.feature +++ b/tests/e2e/features/tls-mtls.feature @@ -1,4 +1,4 @@ -@e2e_group_1 @skip-in-library-mode @skip-in-prow +@cfg_tls @skip-in-library-mode @skip-in-prow Feature: TLS configuration — mutual TLS authentication Validate Llama Stack NetworkConfig.tls client certificate settings against the mock HTTPS inference provider (mTLS port). diff --git a/tests/e2e/features/tls-tlsv13.feature b/tests/e2e/features/tls-tlsv13.feature index b660e30a1..692b8fc1d 100644 --- a/tests/e2e/features/tls-tlsv13.feature +++ b/tests/e2e/features/tls-tlsv13.feature @@ -1,4 +1,4 @@ -@e2e_group_1 @skip-in-library-mode @skip-in-prow +@cfg_tls @skip-in-library-mode @skip-in-prow Feature: TLS configuration — TLS minimum version 1.3 Validate Llama Stack NetworkConfig.tls min_version TLSv1.3 against the mock HTTPS inference provider. diff --git a/tests/e2e/features/unified-mode-boot.feature b/tests/e2e/features/unified-mode-boot.feature index ef7b34dca..7c1413818 100644 --- a/tests/e2e/features/unified-mode-boot.feature +++ b/tests/e2e/features/unified-mode-boot.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @skip +@cfg_unified @skip Feature: Unified mode configuration boot Background: diff --git a/tests/e2e/features/unified-mode-legacy.feature b/tests/e2e/features/unified-mode-legacy.feature index 0bee463ec..407c43e6a 100644 --- a/tests/e2e/features/unified-mode-legacy.feature +++ b/tests/e2e/features/unified-mode-legacy.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @skip +@cfg_unified @skip Feature: Legacy two-file configuration during deprecation window Background: diff --git a/tests/e2e/features/unified-mode-migration.feature b/tests/e2e/features/unified-mode-migration.feature index 4ccaf7725..b5a2d06eb 100644 --- a/tests/e2e/features/unified-mode-migration.feature +++ b/tests/e2e/features/unified-mode-migration.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @skip +@cfg_unified @skip Feature: Legacy to unified configuration migration Background: diff --git a/tests/e2e/features/unified-mode-synthesis.feature b/tests/e2e/features/unified-mode-synthesis.feature index 57fd20aeb..52b4254fa 100644 --- a/tests/e2e/features/unified-mode-synthesis.feature +++ b/tests/e2e/features/unified-mode-synthesis.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @skip +@cfg_unified @skip Feature: Unified mode configuration synthesis Background: diff --git a/tests/e2e/features/unified-mode-validation.feature b/tests/e2e/features/unified-mode-validation.feature index a298e89f2..02fa8c4cb 100644 --- a/tests/e2e/features/unified-mode-validation.feature +++ b/tests/e2e/features/unified-mode-validation.feature @@ -1,4 +1,4 @@ -@e2e_group_2 @skip +@cfg_unified @skip Feature: Unified mode configuration validation Background: diff --git a/tests/e2e/features/vector_stores.feature b/tests/e2e/features/vector_stores.feature index 44620fcf1..55946115f 100644 --- a/tests/e2e/features/vector_stores.feature +++ b/tests/e2e/features/vector_stores.feature @@ -1,4 +1,4 @@ -@e2e_group_3 @VectorStores +@cfg_authorized @VectorStores Feature: vector stores API endpoint tests @@ -8,7 +8,7 @@ Feature: vector stores API endpoint tests And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c And REST API service prefix is /v1 And the Lightspeed stack configuration directory is "tests/e2e/configuration" - And The service uses the lightspeed-stack-auth-noop-token.yaml configuration + And The service uses the lightspeed-stack-authorized.yaml configuration And The service is restarted Scenario: List vector stores returns 200 diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index ad32a1a84..2a5d97f05 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -4,37 +4,38 @@ features/info.feature features/models.feature features/rest_api.feature features/smoketests.feature +features/inline_rag.feature +features/proxy.feature +features/llama_stack_disrupted.feature features/authorized_noop_token.feature -features/conversation_cache_v2.feature features/conversations.feature -features/prompts.feature features/faiss.feature -features/inline_rag.feature -features/byok_pdf.feature -features/vector_stores.feature -features/feedback.feature +features/opentelemetry.feature +features/prompts.feature features/query.feature features/responses.feature features/responses_streaming.feature features/rlsapi_v1.feature features/streaming_query.feature +features/vector_stores.feature +features/conversation_cache_v2.feature +features/feedback.feature features/http_401_unauthorized.feature -features/authorized_rh_identity.feature features/rbac.feature features/rlsapi_v1_errors.feature -features/llama_stack_disrupted.feature -features/mcp.feature +features/skills.feature +features/authorized_rh_identity.feature features/mcp_servers_api.feature +features/mcp.feature features/mcp_servers_api_auth.feature features/mcp_servers_api_no_config.feature -features/proxy.feature +features/byok_pdf.feature features/tls-ca.feature features/tls-mtls.feature features/tls-tlsv13.feature -features/opentelemetry.feature +features/degraded_mode_startup.feature features/unified-mode-boot.feature features/unified-mode-legacy.feature features/unified-mode-validation.feature features/unified-mode-migration.feature features/unified-mode-synthesis.feature -features/skills.feature \ No newline at end of file diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index dddc04558..6a4f5a0dc 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -62,15 +62,20 @@ def run_e2e_ops( ) -def wait_for_pod_health(pod_name: str, max_attempts: int = 20) -> None: +def wait_for_pod_health(pod_name: str, max_attempts: int = 60) -> None: """Wait for pod to be ready in OpenShift/Prow environment. - Generous number of attempts to account for OpenTelemetry instrumentation - initialization overhead during service startup. + Default 60 attempts to account for OpenTelemetry instrumentation + initialization overhead and slower pod rollouts. """ actual_pod_name = get_pod_name(pod_name) try: - result = run_e2e_ops("wait-for-pod", [actual_pod_name, str(max_attempts)]) + # Subprocess timeout must cover e2e-ops poll budget (attempts × 3s). + result = run_e2e_ops( + "wait-for-pod", + [actual_pod_name, str(max_attempts)], + timeout=max(180, max_attempts * 3 + 60), + ) print(result.stdout, end="") if result.returncode != 0: print(result.stderr, end="") diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index 0597c4846..548d3e6a4 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -64,6 +64,11 @@ def is_prow_environment() -> bool: E2E_HTTP_TRANSIENT_MAX_ATTEMPTS: int = 3 E2E_HTTP_TRANSIENT_DELAY_S: float = 0.5 +# Override via E2E_CONTAINER_HEALTH_MAX_ATTEMPTS (default 60). +E2E_CONTAINER_HEALTH_MAX_ATTEMPTS: int = int( + os.getenv("E2E_CONTAINER_HEALTH_MAX_ATTEMPTS", "60") +) + def request_with_transient_retry( **kwargs: Any, @@ -181,7 +186,10 @@ def validate_json(message: Any, schema: Any) -> None: assert False, "The provided schema is faulty:" + str(e) -def wait_for_container_health(container_name: str, max_attempts: int = 20) -> None: +def wait_for_container_health( + container_name: str, + max_attempts: Optional[int] = None, +) -> None: """Wait for container to be healthy. Polls a Docker container until its health status becomes `healthy` or the @@ -192,8 +200,8 @@ def wait_for_container_health(container_name: str, max_attempts: int = 20) -> No inspect errors or timeouts are ignored and retried; the function returns after the container is observed healthy or after all attempts complete. - OpenTelemetry instrumentation adds initialization overhead, so the default - has been set to 20 attempts (40 seconds) to prevent timeouts. + OpenTelemetry instrumentation adds initialization overhead; default attempts + come from ``E2E_CONTAINER_HEALTH_MAX_ATTEMPTS`` (60). Returns: ------- @@ -202,8 +210,11 @@ def wait_for_container_health(container_name: str, max_attempts: int = 20) -> No Parameters: ---------- container_name (str): Docker container name or ID to check. - max_attempts (int): Maximum number of health check attempts (default 20). + max_attempts (int | None): Maximum health check attempts (default from env). """ + if max_attempts is None: + max_attempts = E2E_CONTAINER_HEALTH_MAX_ATTEMPTS + if is_prow_environment(): wait_for_pod_health(container_name, max_attempts) return @@ -235,7 +246,7 @@ def wait_for_container_health(container_name: str, max_attempts: int = 20) -> No print( f"Could not confirm Docker health=healthy for {container_name} " - f"after {max_attempts} attempts" + f"after {max_attempts} attempts (~{max_attempts * 2}s)" ) @@ -473,7 +484,7 @@ def restart_container(container_name: str) -> None: # (~45-60s vs ~10s in server mode). OpenTelemetry instrumentation adds # initialization overhead. Use a generous attempt count so MCP-auth scenarios # that restart the container don't time out. - wait_for_container_health(container_name, max_attempts=20) + wait_for_container_health(container_name) if container_name == "llama-stack": from tests.e2e.features.steps.health import ( @@ -484,7 +495,7 @@ def restart_container(container_name: str) -> None: def wait_for_lightspeed_stack_http_ready( - max_attempts: int = 40, + max_attempts: int = 80, delay_s: float = 1.5, ) -> None: """Block until Lightspeed Stack accepts HTTP on the host-mapped port.