timeouts

docker-compose-library.yaml

@@ -67,10 +67,12 @@ services:
       - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-}
       - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-true}
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8080/liveness"]
+      # /readiness checks providers + default model; library mode also boots the
+      # embedded stack, so allow a long grace period before counting failures.
+      test: ["CMD", "curl", "-f", "http://localhost:8080/readiness"]
       interval: 10s   # how often to run the check
       timeout: 5s    # how long to wait before considering it failed
-      retries: 3      # how many times to retry before marking as unhealthy
+      retries: 5      # how many times to retry before marking as unhealthy
       start_period: 15s # time to wait before starting checks (increased for library initialization)
 
   # Mock JWKS server for RBAC E2E tests

docker-compose.yaml

@@ -116,11 +116,13 @@ services:
     networks:
       - lightspeednet
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8080/liveness"]
+      # /readiness checks providers + default model; give Llama/providers time to
+      # finish registering before failures count toward unhealthy.
+      test: ["CMD", "curl", "-f", "http://localhost:8080/readiness"]
       interval: 10s   # how often to run the check
       timeout: 5s    # how long to wait before considering it failed
-      retries: 3      # how many times to retry before marking as unhealthy
-      start_period: 5s # time to wait before starting checks
+      retries: 5      # how many times to retry before marking as unhealthy
+      start_period: 60s # ignore failures while providers/models come up after restart
 
   # Mock JWKS server for RBAC E2E tests
   mock-jwks:

tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml

@@ -67,14 +67,16 @@ spec:
       command: ["/bin/sh", "-c", "mkdir -p /tmp/data && exec /app-root/entrypoint.sh"]
       ports:
         - containerPort: 8080
-      # TCP probes avoid HTTP/auth. LCS + Llama handshake and large images can take 60–120s before :8080 listens;
-      # aggressive liveness was killing the container (connection refused) and breaking port-forward sandboxes.
+      # Readiness waits for providers/default model (/readiness). Liveness stays TCP so a
+      # slow provider handshake does not kill the pod (connection refused during boot).
       readinessProbe:
-        tcpSocket:
+        httpGet:
+          path: /readiness
           port: 8080
         initialDelaySeconds: 20
         periodSeconds: 5
-        failureThreshold: 30
+        timeoutSeconds: 5
+        failureThreshold: 36  # ~3 min after initialDelay for provider/model registration
       livenessProbe:
         tcpSocket:
           port: 8080

tests/e2e/features/environment.py

@@ -1,10 +1,11 @@
 """Code to be called before and after certain events during testing.
 
-Currently four events have been registered:
+Currently five events have been registered:
 1. before_all
 2. before_feature
 3. before_scenario
 4. after_scenario
+5. after_feature
 """
 
 import os
@@ -34,7 +35,6 @@
 )
 from tests.e2e.utils.llama_stack_utils import register_shield
 from tests.e2e.utils.prow_utils import (
-    restart_pod,
     restore_llama_stack_pod,
     run_e2e_ops,
 )
@@ -285,30 +285,21 @@ def _dump_pod_logs_on_failure(
 def after_scenario(context: Context, scenario: Scenario) -> None:
     """Run after each scenario is run.
 
-    Perform per-scenario teardown: restore scenario-specific configuration and,
-    in server mode, attempt to restart and verify the Llama Stack container if
-    it was previously running.
+    Per-scenario teardown only:
 
-    If ``configure_service`` applied a non-baseline YAML during the scenario
-    (``context.scenario_lightspeed_override_active``), copies
-    ``context.feature_config`` back and restarts lightspeed-stack.
+    - If ``configure_service`` applied a non-baseline YAML
+      (``context.scenario_lightspeed_override_active``), copy
+      ``context.feature_config`` back and restart lightspeed-stack.
+    - Re-register the llama-guard shield when a scenario disabled it.
 
-    When not running in library mode and the context indicates the Llama Stack
-    was running before the scenario, this function attempts to start the
-    llama-stack container and polls its health endpoint until it becomes
-    healthy or a timeout is reached.
+    Llama Stack disruption recovery runs in ``after_feature``, not here.
 
     Parameters:
     ----------
         context (Context): Behave test context. Expected attributes used here include:
             - feature_config: path to the feature-level configuration to restore.
             - scenario_lightspeed_override_active: set by ``configure_service``
               when a scenario switches YAML after Background.
-            - is_library_mode (bool): whether tests run in library mode.
-            - llama_stack_was_running (bool, optional): whether llama-stack was
-              running before the scenario.
-            - hostname_llama, port_llama (str/int, optional): host and port
-              used for the llama-stack health check.
         scenario (Scenario): Behave scenario (unused; shield restore uses context flags).
     """
     if is_prow_environment():
@@ -370,48 +361,26 @@ def _print_llama_stack_diagnostics() -> None:
     print("--- end diagnostics ---")
 
 
-def _restore_llama_stack() -> None:
-    """Restore Llama Stack connection after disruption."""
+def _ensure_llama_stack_running() -> None:
+    """Bring Llama Stack back after disruption (soft-fail; teardown must not abort the suite).
+
+    On Prow, recreates the Llama pod. On Docker, ``docker start`` and polls
+    in-container ``/v1/health``. Does not restart lightspeed-stack; callers
+    decide that after config restore so Llama is only brought up once.
+    """
     if is_prow_environment():
-        # Recreate llama pod, then restart LCS so in-process clients reconnect (Llama IP/pod changed).
         try:
             restore_llama_stack_pod()
+            reset_llama_stack_disrupt_once_tracking()
+            print("✓ Prow: Llama Stack restored")
         except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
             print(f"Warning: Could not restore Llama Stack pod on Prow: {e}")
-            return
-        last_lcs_err: Optional[
-            subprocess.CalledProcessError | subprocess.TimeoutExpired
-        ] = None
-        for attempt in range(1, 4):
-            try:
-                restart_pod("lightspeed-stack")
-                print(
-                    "✓ Prow: Llama Stack restored and lightspeed-stack restarted "
-                    "for clean reconnect"
-                )
-                reset_llama_stack_disrupt_once_tracking()
-                return
-            except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
-                last_lcs_err = e
-                print(
-                    f"Warning: lightspeed-stack restart after Llama restore "
-                    f"attempt {attempt}/3 failed: {e}"
-                )
-                if attempt < 3:
-                    time.sleep(5)
-        print(
-            "Warning: Could not restart lightspeed-stack after Llama restore "
-            f"after 3 attempts: {last_lcs_err}"
-        )
         return
 
     try:
-        # Start the llama-stack container again
         subprocess.run(
             ["docker", "start", "llama-stack"], check=True, capture_output=True
         )
-
-        # Wait for the service to be healthy
         print("Restoring Llama Stack connection...")
         max_attempts = 24
         for attempt in range(max_attempts):
@@ -432,7 +401,7 @@ def _restore_llama_stack() -> None:
                 if result.returncode == 0:
                     print("✓ Llama Stack connection restored successfully")
                     reset_llama_stack_disrupt_once_tracking()
-                    break
+                    return
             except subprocess.TimeoutExpired:
                 print(
                     f"⏱ Health check timed out on attempt {attempt + 1}/{max_attempts}"
@@ -444,9 +413,9 @@ def _restore_llama_stack() -> None:
                     f"(attempt {attempt + 1}/{max_attempts})"
                 )
                 time.sleep(2)
-            else:
-                print("Warning: Llama Stack may not be fully healthy after restoration")
-                _print_llama_stack_diagnostics()
+
+        print("Warning: Llama Stack may not be fully healthy after restoration")
+        _print_llama_stack_diagnostics()
 
     except subprocess.CalledProcessError as e:
         print(f"Warning: Could not restore Llama Stack connection: {e}")
@@ -457,6 +426,44 @@ def _restore_llama_stack() -> None:
         _print_llama_stack_diagnostics()
 
 
+def _restore_lightspeed_config_backup() -> bool:
+    """Restore ``lightspeed-stack.yaml`` from backup if present.
+
+    Returns:
+        True when a backup was applied and removed.
+    """
+    backup_path = "lightspeed-stack.yaml.backup"
+    if not os.path.exists(backup_path):
+        return False
+    switch_config(backup_path)
+    remove_config_backup(backup_path)
+    return True
+
+
+def _restart_lightspeed_after_prow_llama_restore() -> None:
+    """Soft-fail LCS restart so Prow clients reconnect after a Llama pod change."""
+    last_lcs_err: Optional[
+        subprocess.CalledProcessError | subprocess.TimeoutExpired
+    ] = None
+    for attempt in range(1, 4):
+        try:
+            restart_container("lightspeed-stack")
+            print("✓ Prow: lightspeed-stack restarted after Llama disruption restore")
+            return
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+            last_lcs_err = e
+            print(
+                f"Warning: lightspeed-stack restart after Llama restore "
+                f"attempt {attempt}/3 failed: {e}"
+            )
+            if attempt < 3:
+                time.sleep(5)
+    print(
+        "Warning: Could not restart lightspeed-stack after Llama restore "
+        f"after 3 attempts: {last_lcs_err}"
+    )
+
+
 def before_feature(context: Context, feature: Feature) -> None:
     """Run before each feature file is exercised.
 
@@ -498,17 +505,15 @@ def before_feature(context: Context, feature: Feature) -> None:
 def after_feature(context: Context, feature: Feature) -> None:
     """Run after each feature file is exercised.
 
-    Perform feature-level teardown: restore any modified configuration and,
-    when ``context.feedback_e2e_conversation_cleanup`` is set by feedback steps,
-    delete tracked feedback test conversations.
-    """
-    # Restore Llama Stack FIRST (before any lightspeed-stack restart).
-    # Read from module-level state — Behave clears custom context attributes
-    # between scenarios, so context.llama_stack_was_running is unreliable here.
-    if get_llama_stack_was_running():
-        _restore_llama_stack()
-        reset_llama_stack_was_running()
+    Teardown order (avoids start-then-restart of Llama):
 
+    1. Feedback conversation cleanup (while the feature's LCS config is still active).
+    2. Restore ``lightspeed-stack.yaml`` from backup when present.
+    3. Bring Llama up **once** when needed (disrupted and/or config restored).
+    4. Restart lightspeed-stack **once** when config was restored, or on Prow after
+       a Llama disruption (clients must reconnect to a new pod).
+    5. Stop any leftover proxy servers; log feature duration.
+    """
     if getattr(context, "feedback_e2e_conversation_cleanup", False):
         token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva"
         for conversation_id in getattr(context, "feedback_conversations", []):
@@ -517,16 +522,27 @@ def after_feature(context: Context, feature: Feature) -> None:
             response = requests.delete(url, headers=headers, timeout=10)
             assert response.status_code == 200, f"{url} returned {response.status_code}"
 
-    # Restore Lightspeed Stack config if the generic configure_service step switched it.
-    # This cleanup intentionally runs for any feature (not tag-gated) - any feature that
-    # leaves a backup file will trigger config restoration and container restarts.
-    backup_path = "lightspeed-stack.yaml.backup"
-    if os.path.exists(backup_path):
-        switch_config(backup_path)
-        remove_config_backup(backup_path)
-        if not context.is_library_mode:
+    # Module-level flag — Behave clears custom context attrs between scenarios.
+    llama_was_disrupted = get_llama_stack_was_running()
+    if llama_was_disrupted:
+        reset_llama_stack_was_running()
+
+    # Restore host/ConfigMap YAML before bouncing containers so a single
+    # Llama start/restart sees the baseline enrichment config.
+    config_restored = _restore_lightspeed_config_backup()
+
+    if not context.is_library_mode and (llama_was_disrupted or config_restored):
+        if config_restored:
+            # ``docker restart`` starts a stopped container; picks up restored YAML.
             restart_container("llama-stack")
+        else:
+            # Disrupt-only (no backup): soft-fail so teardown does not abort the suite.
+            _ensure_llama_stack_running()
+
+    if config_restored:
         restart_container("lightspeed-stack")
+    elif llama_was_disrupted and is_prow_environment():
+        _restart_lightspeed_after_prow_llama_restore()
 
     # Clean up any proxy servers left from the last scenario
     if hasattr(context, "tunnel_proxy") or hasattr(context, "interception_proxy"):

tests/e2e/utils/utils.py

@@ -468,12 +468,15 @@ def restart_container(container_name: str) -> None:
         print(f"Failed to restart container {container_name}: {e}")
         raise
 
-    # Wait for container to be healthy.
-    # Library mode embeds llama-stack, so the container takes longer to start
-    # (~45-60s vs ~10s in server mode). OpenTelemetry instrumentation adds
-    # initialization overhead. Use a generous attempt count so MCP-auth scenarios
-    # that restart the container don't time out.
-    wait_for_container_health(container_name, max_attempts=20)
+    # Wait for container health. Lightspeed compose probes /readiness with a long
+    # start_period (providers/models); allow enough poll time to cover that window
+    # (server ~60s+retries, library ~120s+retries) rather than giving up early.
+    health_attempts = 90 if container_name == "lightspeed-stack" else 20
+    wait_for_container_health(container_name, max_attempts=health_attempts)
+
+    if container_name == "lightspeed-stack":
+        # Published host port can lag Docker's in-container healthy; confirm from Behave.
+        wait_for_lightspeed_stack_http_ready()
 
     if container_name == "llama-stack":
         from tests.e2e.features.steps.health import (
@@ -487,26 +490,26 @@ def wait_for_lightspeed_stack_http_ready(
     max_attempts: int = 40,
     delay_s: float = 1.5,
 ) -> None:
-    """Block until Lightspeed Stack accepts HTTP on the host-mapped port.
+    """Block until Lightspeed Stack is ready on the host-mapped port.
 
     Used from proxy e2e steps only: ``docker inspect`` health can report
     ``healthy`` before the published port accepts connections (Podman/Docker
-    timing). Polls ``/liveness`` using the same host/port as Behave
-    (``E2E_LSC_*``).
+    timing). Polls ``/readiness`` (providers + default model) using the same
+    host/port as Behave (``E2E_LSC_*``).
 
     Parameters:
     ----------
         max_attempts: Maximum GET attempts.
         delay_s: Sleep between attempts.
     Raises:
     ------
-        AssertionError: If ``/liveness`` does not return HTTP 200 in time.
+        AssertionError: If ``/readiness`` does not return HTTP 200 in time.
     """
     if is_prow_environment():
         return
     host = os.getenv("E2E_LSC_HOSTNAME", "localhost")
     port = os.getenv("E2E_LSC_PORT", "8080")
-    url = f"http://{host}:{port}/liveness"
+    url = f"http://{host}:{port}/readiness"
     for attempt in range(max_attempts):
         try:
             response = requests.get(url, timeout=5)
@@ -518,7 +521,7 @@ def wait_for_lightspeed_stack_http_ready(
             print(f"⏱ HTTP wait LSC {attempt + 1}/{max_attempts} ({url})...")
             time.sleep(delay_s)
     raise AssertionError(
-        f"Lightspeed Stack did not become reachable at {url!r} "
+        f"Lightspeed Stack did not become ready at {url!r} "
         f"after {max_attempts} attempts (~{max_attempts * delay_s:.0f}s)"
     )