LCORE-3537: E2E tests rework

.github/workflows/e2e_tests.yaml

@@ -11,9 +11,32 @@ jobs:
       matrix:
         mode: ["server", "library"]
         environment: ["ci"]
-        e2e_group: [1, 2, 3]
+        # Config-aligned shards (@cfg_*). Packed small groups keep job count reasonable
+        # while avoiding mixed-config restarts inside large suites.
+        shard:
+          - name: default
+            tags: "not @skip and @cfg_default"
+          - name: authorized
+            tags: "not @skip and @cfg_authorized"
+          - name: mcp
+            tags: "not @skip and (@cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth)"
+          - name: rbac
+            tags: "not @skip and @cfg_rbac"
+          - name: skills
+            tags: "not @skip and (@cfg_skills or @cfg_skills_directory)"
+          - name: other
+            tags: "not @skip and (@cfg_rh_identity or @cfg_negative or @cfg_byok_pdf or @cfg_degraded or @cfg_unified)"
+          # Server-only; listed in shard (not matrix.include) so it expands with
+          # mode=server before any library jobs. include would append after library.
+          - name: tls
+            tags: "not @skip and @cfg_tls"
+        exclude:
+          - mode: library
+            shard:
+              name: tls
+              tags: "not @skip and @cfg_tls"
 
-    name: "E2E: ${{ matrix.mode }} mode / ${{ matrix.environment }} / group ${{ matrix.e2e_group }}"
+    name: "E2E: ${{ matrix.mode }} / ${{ matrix.environment }} / ${{ matrix.shard.name }}"
 
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -43,7 +66,7 @@ jobs:
           persist-credentials: ${{ github.event_name != 'pull_request_target' }}
 
           # Fetch submodules (required for lightspeed-providers)
-          submodules: ‘recursive’
+          submodules: "recursive"
 
       - name: Verify actual git checkout result
         run: |
@@ -112,7 +135,7 @@ jobs:
           echo "=== Configuration Summary ==="
           echo "Deployment mode: ${{ matrix.mode }}"
           echo "Environment: ${{ matrix.environment }}"
-          echo "E2E shard (Makefile test-e2e-tagged): @e2e_group_${{ matrix.e2e_group }} (with not @skip)"
+          echo "E2E shard (Makefile test-e2e-tagged): ${{ matrix.shard.tags }}"
           echo "Source config: tests/e2e/configs/run-${{ matrix.environment }}.yaml"
           echo ""
           echo "=== Configuration Preview ==="
@@ -239,8 +262,8 @@ jobs:
           TERM: xterm-256color
           FORCE_COLOR: 1
           E2E_DEPLOYMENT_MODE: ${{ matrix.mode }}
-          # Matches Makefile test-e2e-tagged / E2E_BEHAVE_TAG_EXPR (one @e2e_group_* per job).
-          E2E_BEHAVE_TAG_EXPR: "not @skip and @e2e_group_${{ matrix.e2e_group }}"
+          # Matches Makefile test-e2e-tagged / E2E_BEHAVE_TAG_EXPR (one @cfg_* pack per job).
+          E2E_BEHAVE_TAG_EXPR: "${{ matrix.shard.tags }}"
         run: |
           echo "Installing test dependencies..."
           pip install uv

.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml

@@ -270,6 +270,10 @@ spec:
                 value: "$(params.namespace)"
               - name: SNAPSHOT
                 value: $(params.SNAPSHOT)
+              - name: OTEL_SDK_DISABLED
+                value: "true"
+              - name: OTEL_ANONYMIZATION_SECRET
+                value: "lightspeed-stack-otel-anonymization-dev-default"
             image: registry.access.redhat.com/ubi9/ubi-minimal
             script: |
               set +e

.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml

@@ -396,6 +396,10 @@ spec:
                 value: "$(params.vllm-api-key)"
               - name: VLLM_MODEL
                 value: "$(params.model)"
+              - name: OTEL_SDK_DISABLED
+                value: "true"
+              - name: OTEL_ANONYMIZATION_SECRET
+                value: "lightspeed-stack-otel-anonymization-dev-default"
             image: registry.access.redhat.com/ubi9/ubi-minimal
             script: |
               set +e

Makefile

@@ -162,11 +162,11 @@ test-e2e: ## Run end to end tests for the service
 test-e2e-local: ## Run end to end tests for the service (no script wrapper)
 	uv run behave --color --format pretty --tags=-skip -D dump_errors=true @tests/e2e/test_list.txt
 
-# Tag-based subsets (@e2e_group_* on feature files). Default runs all groups; override for one shard, e.g.
-#   E2E_BEHAVE_TAG_EXPR='not @skip and @e2e_group_2' make test-e2e-tagged-local
-E2E_BEHAVE_TAG_EXPR ?= not @skip and (e2e_group_1 or e2e_group_2 or e2e_group_3)
+# Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g.
+#   E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local
+E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified)
 
-test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @e2e_group_*)
+test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*)
 	script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt"
 
 test-e2e-tagged-local: ## Same as test-e2e-tagged without script wrapper

README.md

@@ -920,7 +920,7 @@ test-unit                         Run the unit tests
 test-integration                  Run integration tests tests
 test-e2e                          Run end to end tests for the service
 test-e2e-local                    Run end to end tests for the service (no script wrapper)
-test-e2e-tagged                   Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @e2e_group_*)
+test-e2e-tagged                   Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*)
 test-e2e-tagged-local             Same as test-e2e-tagged without script wrapper
 benchmarks                        Run benchmarks
 check-types-src                   Check type hints in sources only

docker-compose-library.yaml

@@ -65,6 +65,7 @@ services:
       - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-}
       - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-}
       - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-}
+      - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default}
       - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-true}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8080/liveness"]

docker-compose.yaml

@@ -107,6 +107,7 @@ services:
       - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-}
       - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-}
       - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-}
+      - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default}
       - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-true}
     depends_on:
         llama-stack:

docs/testing/e2e_testing.md

@@ -214,7 +214,7 @@ You can put several tags on one scenario. To document why a scenario is skipped,
 
 ## Configuration Files
 
-- **Lightspeed-stack**: Under `tests/e2e/configuration/server-mode/` and `library-mode/`. Switched via `switch_config()` and copied into the container's config path (or applied via ConfigMap in Prow). Names like `lightspeed-stack.yaml`, `lightspeed-stack-auth-noop-token.yaml`, `lightspeed-stack-rbac.yaml`, etc.
+- **Lightspeed-stack**: Under `tests/e2e/configuration/server-mode/` and `library-mode/`. Switched via `switch_config()` and copied into the container's config path (or applied via ConfigMap in Prow). Bootstrap: `lightspeed-stack.yaml`; variants: `lightspeed-stack-default.yaml`, `lightspeed-stack-authorized.yaml`, `lightspeed-stack-rbac.yaml`, etc. (see `tests/e2e/configuration/grouped/README.md`).
 - **Llama Stack**: Under `tests/e2e/configs/` (e.g. `run-ci.yaml`). Used by the Llama Stack container; not switched by Behave step-by-step, but the stack is started with the appropriate run config.
 
 See `tests/e2e/configuration/README.md` for a short description of each config.

tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml

@@ -57,6 +57,10 @@ spec:
         # Unused for server-mode FAISS (llama pod owns the fixture); keep out of ~/.llama.
         - name: KV_RAG_PATH
           value: "/app-root/.e2e-rag-work/kv_store.db"
+        - name: OTEL_SDK_DISABLED
+          value: "true"
+        - name: OTEL_ANONYMIZATION_SECRET
+          value: "lightspeed-stack-otel-anonymization-dev-default"
         - name: VLLM_MODEL
           valueFrom:
             secretKeyRef:

tests/e2e/configuration/README.md

@@ -54,8 +54,27 @@ Both server-mode and library-mode default configurations include:
 
 3. **User Data Collection** - Enabled for feedback and transcripts testing
 
-### Special-Purpose Configurations
+### Config variants (`@cfg_*` tags)
+
+Named configs merge compatible options and map to `@cfg_*` Behave tags so CI
+can shard by config (fewer restarts per job). See
+[grouped/README.md](grouped/README.md).
+
+| Config | Purpose |
+|--------|---------|
+| `lightspeed-stack-default.yaml` | Default + inline RAG |
+| `lightspeed-stack-authorized.yaml` | Authorization (noop-token) |
+| `lightspeed-stack-negative.yaml` | No cache + invalid feedback storage |
+| `lightspeed-stack-rbac.yaml` | RBAC (jwk-token auth) |
+| `lightspeed-stack-rh-identity.yaml` | RH Identity header auth |
+| `lightspeed-stack-skills.yaml` / `-skills-directory.yaml` | Agent skills |
+| `lightspeed-stack-mcp.yaml` | All valid MCP servers (`mcp_servers_api`) |
+| `lightspeed-stack-mcp-invalid.yaml` | Invalid MCP file token |
+| `lightspeed-stack-mcp-api-auth.yaml` | MCP server API auth |
+| `lightspeed-stack-mcp-{file,kubernetes,client,oauth}-auth.yaml` | Per-auth MCP (`mcp.feature`) |
+| `lightspeed-stack-byok-pdf.yaml` | BYOK PDF (library only) |
+| `lightspeed-stack-tls.yaml` / `-degraded.yaml` | TLS / degraded mode (server only) |
+
+Bootstrap default: `lightspeed-stack.yaml`. Environment-specific:
+`lightspeed-stack-rhelai.yaml`, `lightspeed-stack-rhoai.yaml`.
 
-- `lightspeed-stack-auth-noop-token.yaml` - For authorization testing
-- `lightspeed-stack-invalid-feedback-storage.yaml` - For negative feedback testing
-- `lightspeed-stack-no-cache.yaml` - For cache-disabled scenarios

tests/e2e/configuration/library-mode/lightspeed-stack-auth-noop-token.yaml → tests/e2e/configuration/library-mode/lightspeed-stack-authorized.yaml

@@ -1,3 +1,11 @@
+# @cfg_authorized
+# Safe merge: auth-noop-token + shields from default.
+# Intentionally NO mcp_servers: query/streaming_query/responses/tools all call
+# check_mcp_auth(); a static MCP entry would probe mock-mcp on every authorized
+# suite request and amplify flake.
+# Intentionally NO pii-redaction shields: streaming_query compares
+# token-stream text to turn_complete; redaction only applies to the complete
+# message and would make that assertion fail (v2 vs v[NUM]).
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0

tests/e2e/configuration/library-mode/lightspeed-stack-byok-pdf.yaml

@@ -1,3 +1,6 @@
+# @cfg_byok_pdf
+# Kept separate: dedicated vector store + rag.inline=[pdf-field-notes] so
+# retrieval is not mixed with e2e-test-docs (Paul Graham) chunks.
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0

tests/e2e/configuration/library-mode/lightspeed-stack-inline-rag.yaml → tests/e2e/configuration/library-mode/lightspeed-stack-default.yaml

@@ -1,3 +1,8 @@
+# @cfg_default
+# Safe merge: lightspeed-stack.yaml + lightspeed-stack-inline-rag.yaml
+# Same byok rag_id (e2e-test-docs) with both rag.tool and rag.inline.
+# /v1/rags partial asserts still match. No MCP (would break info tools checks
+# and add check_mcp_auth to every call). No PDF BYOK (separate retrieval store).
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0
@@ -39,5 +44,15 @@ byok_rag:
     score_multiplier: 1.0
 
 rag:
+  tool:
+    - e2e-test-docs
   inline:
     - e2e-test-docs
+
+shields:
+  - name: pii-redaction
+    provider_id: redaction
+    config:
+      rules:
+        - pattern: '\d+'
+          replacement: '[NUM]'

tests/e2e/configuration/library-mode/lightspeed-stack-mcp-auth.yaml → tests/e2e/configuration/library-mode/lightspeed-stack-mcp-api-auth.yaml

@@ -1,3 +1,7 @@
+# @cfg_mcp_api_auth
+# From: lightspeed-stack-mcp-auth.yaml
+# Kept out of @cfg_authorized: static MCP would make check_mcp_auth run on
+# every query/responses call in the authorized suite.
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0

tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml → tests/e2e/configuration/library-mode/lightspeed-stack-mcp-invalid.yaml

@@ -1,3 +1,6 @@
+# @cfg_mcp_invalid
+# From: lightspeed-stack-invalid-mcp-file-auth.yaml
+# Must stay isolated: tools/query expect 401 when the only MCP server has a bad token.
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0

tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml

@@ -1,3 +1,9 @@
+# @cfg_mcp
+# Merges: lightspeed-stack-mcp.yaml + lightspeed-stack-mcp-file-auth.yaml +
+#         lightspeed-stack-mcp-client-auth.yaml + lightspeed-stack-mcp-oauth-auth.yaml +
+#         lightspeed-stack-mcp-kubernetes-auth.yaml (valid tokens only)
+# All four MCP auth flavours share one process; scenarios select a server by name.
+# Keep @cfg_mcp_invalid separate (bad file token must be the only mcp-file entry).
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0
@@ -36,4 +42,4 @@ mcp_servers:
   - name: "mcp-client"
     url: "http://mock-mcp:3000"
     authorization_headers:
-      Authorization: "client"
+      Authorization: "client"

tests/e2e/configuration/library-mode/lightspeed-stack-invalid-feedback-storage.yaml → tests/e2e/configuration/library-mode/lightspeed-stack-negative.yaml

@@ -1,3 +1,8 @@
+# @cfg_negative
+# Merges: lightspeed-stack-no-cache.yaml + lightspeed-stack-invalid-feedback-storage.yaml
+# Compatible: both use noop-with-token; query works with cache=None; conversations
+# v2 asserts "cache not configured"; feedback asserts store failure at /invalid.
+# Intentionally no conversation_cache and invalid feedback_storage.
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0
@@ -18,5 +23,7 @@ user_data_collection:
   transcripts_enabled: true
   transcripts_storage: "/tmp/data/transcripts"
 
+# NO conversation_cache — cache-disabled + empty MCP list scenarios
+
 authentication:
   module: "noop-with-token"

tests/e2e/configuration/library-mode/lightspeed-stack-rbac.yaml

@@ -1,3 +1,5 @@
+# @cfg_rbac
+# From: lightspeed-stack-rbac.yaml (auth module jwk-token is incompatible with other groups)
 name: Lightspeed Core Service (RBAC E2E Tests - Library Mode)
 service:
   host: 0.0.0.0

tests/e2e/configuration/library-mode/lightspeed-stack-auth-rh-identity.yaml → tests/e2e/configuration/library-mode/lightspeed-stack-rh-identity.yaml

@@ -1,3 +1,5 @@
+# @cfg_rh_identity
+# From: lightspeed-stack-auth-rh-identity.yaml (auth module incompatible with other groups)
 name: Lightspeed Core Service (LCS) - RH Identity Auth
 service:
   host: 0.0.0.0

tests/e2e/configuration/library-mode/lightspeed-stack-skills-directory.yaml

@@ -1,3 +1,5 @@
+# @cfg_skills_directory
+# Directory discovery (echo + summarize). Separate from @cfg_skills.
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0

tests/e2e/configuration/library-mode/lightspeed-stack-skills.yaml

@@ -1,3 +1,6 @@
+# @cfg_skills
+# Echo-only path. Not merged with skills-directory: @SkillsConfig asserts
+# exact list_skills tool_results content with echo alone.
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0

tests/e2e/configuration/server-mode/lightspeed-stack-auth-noop-token.yaml → tests/e2e/configuration/server-mode/lightspeed-stack-authorized.yaml

@@ -1,3 +1,7 @@
+# @cfg_authorized
+# Safe merge: auth-noop-token + shields. No MCP (see library twin comment).
+# Based on auth-noop-token. Intentionally NO mcp_servers / pii-redaction shields
+# (see library twin: streaming token vs turn_complete comparison).
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0

tests/e2e/configuration/server-mode/lightspeed-stack-inline-rag.yaml → tests/e2e/configuration/server-mode/lightspeed-stack-default.yaml

@@ -1,3 +1,8 @@
+# @cfg_default
+# Safe merge: lightspeed-stack.yaml + lightspeed-stack-inline-rag.yaml
+# Same byok rag_id (e2e-test-docs) with both rag.tool and rag.inline.
+# /v1/rags partial asserts still match. No MCP (would break info tools checks
+# and add check_mcp_auth to every call). No PDF BYOK (separate retrieval store).
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0
@@ -37,5 +42,15 @@ byok_rag:
     score_multiplier: 1.0
 
 rag:
+  tool:
+    - e2e-test-docs
   inline:
     - e2e-test-docs
+
+shields:
+  - name: pii-redaction
+    provider_id: redaction
+    config:
+      rules:
+        - pattern: '\d+'
+          replacement: '[NUM]'

tests/e2e/configuration/server-mode/lightspeed-stack-degraded-mode.yaml → tests/e2e/configuration/server-mode/lightspeed-stack-degraded.yaml

@@ -1,3 +1,5 @@
+# @cfg_degraded
+# From: lightspeed-stack-degraded-mode.yaml (server-mode only)
 name: Lightspeed Core Service (LCS) - Degraded Mode Test
 service:
   host: 0.0.0.0

tests/e2e/configuration/server-mode/lightspeed-stack-mcp-auth.yaml → tests/e2e/configuration/server-mode/lightspeed-stack-mcp-api-auth.yaml

@@ -1,3 +1,5 @@
+# @cfg_mcp_api_auth
+# From: lightspeed-stack-mcp-auth.yaml (kept out of authorized — see library twin)
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0

tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml → tests/e2e/configuration/server-mode/lightspeed-stack-mcp-invalid.yaml

@@ -1,3 +1,6 @@
+# @cfg_mcp_invalid
+# From: lightspeed-stack-invalid-mcp-file-auth.yaml
+# Must stay isolated: tools/query expect 401 when the only MCP server has a bad token.
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0

tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml

@@ -1,3 +1,9 @@
+# @cfg_mcp
+# Merges: lightspeed-stack-mcp.yaml + lightspeed-stack-mcp-file-auth.yaml +
+#         lightspeed-stack-mcp-client-auth.yaml + lightspeed-stack-mcp-oauth-auth.yaml +
+#         lightspeed-stack-mcp-kubernetes-auth.yaml (valid tokens only)
+# All four MCP auth flavours share one process; scenarios select a server by name.
+# Keep @cfg_mcp_invalid separate (bad file token must be the only mcp-file entry).
 name: Lightspeed Core Service (LCS)
 service:
   host: 0.0.0.0
@@ -34,4 +40,4 @@ mcp_servers:
   - name: "mcp-client"
     url: "http://mock-mcp:3000"
     authorization_headers:
-      Authorization: "client"
+      Authorization: "client"