[NVIDIA #9] Test cases for GPU-agnostic compatibility (CPU-only environment)

[AugustChaoTW]

Overview

Comprehensive test suite to verify all NVIDIA/Prefix-KV features degrade gracefully in a CPU-only environment (no NVIDIA GPU, no CUDA, no Ollama optionally).

Covers three test levels:

1. Unit tests — individual component fallbacks (pytest, no external deps)
2. Integration tests — full pipeline with mocked GPU services
3. CI smoke test — end-to-end on GitHub-hosted runner (no GPU)

---

1. ahe/gpu_detect.py Unit Tests

File: tests/test_gpu_detect.py

import importlib, subprocess
from unittest.mock import patch, MagicMock
import pytest

def reload_module():
    import ahe.gpu_detect as m
    importlib.reload(m)
    return m

class TestHasNvidiaGpu:
    def test_no_nvidia_smi_returns_false(self):
        with patch("shutil.which", return_value=None):
            m = reload_module()
            assert m.has_nvidia_gpu() is False

    def test_nvidia_smi_nonzero_exit_returns_false(self):
        with patch("shutil.which", return_value="/usr/bin/nvidia-smi"), \
             patch("subprocess.run") as mock_run:
            mock_run.return_value = MagicMock(returncode=1, stdout="")
            m = reload_module()
            assert m.has_nvidia_gpu() is False

    def test_nvidia_smi_success_returns_true(self):
        with patch("shutil.which", return_value="/usr/bin/nvidia-smi"), \
             patch("subprocess.run") as mock_run:
            mock_run.return_value = MagicMock(returncode=0, stdout="NVIDIA GB10\n")
            m = reload_module()
            assert m.has_nvidia_gpu() is True

    def test_subprocess_exception_returns_false(self):
        with patch("shutil.which", return_value="/usr/bin/nvidia-smi"), \
             patch("subprocess.run", side_effect=FileNotFoundError):
            m = reload_module()
            assert m.has_nvidia_gpu() is False

class TestVramGb:
    def test_no_gpu_returns_zero(self):
        with patch("shutil.which", return_value=None):
            m = reload_module()
            assert m.vram_gb() == 0.0

    def test_parses_multiple_gpus(self):
        with patch("shutil.which", return_value="/usr/bin/nvidia-smi"), \
             patch("subprocess.run") as mock_run:
            mock_run.return_value = MagicMock(returncode=0, stdout="40960\n40960\n")
            m = reload_module()
            assert m.vram_gb() == pytest.approx(80.0, abs=0.1)

---

2. vLLM Backend Fallback Tests

File: tests/test_vllm_fallback.py

from unittest.mock import patch
from ahe.run_eval import load_backend

class TestVllmFallback:
    def test_vllm_local_with_no_gpu_falls_back(self):
        with patch("ahe.gpu_detect.has_nvidia_gpu", return_value=False):
            backend = load_backend("vllm-local")
            assert backend["name"] != "vllm-local", \
                "Should not use vllm-local without GPU"
            assert backend["name"] in ("opencode", "claude-code"), \
                f"Unexpected fallback backend: {backend['name']}"

    def test_vllm_local_with_gpu_uses_vllm(self, mock_vllm_config):
        with patch("ahe.gpu_detect.has_nvidia_gpu", return_value=True):
            backend = load_backend("vllm-local")
            assert backend["name"] == "vllm-local"

    def test_opencode_backend_unaffected_by_gpu_status(self):
        for gpu in [True, False]:
            with patch("ahe.gpu_detect.has_nvidia_gpu", return_value=gpu):
                backend = load_backend("opencode")
                assert backend["name"] == "opencode"

    def test_trtllm_without_gpu_falls_back(self):
        with patch("ahe.gpu_detect.has_nvidia_gpu", return_value=False):
            backend = load_backend("trtllm-local")
            assert backend["name"] != "trtllm-local"

---

3. cuOpt Eviction Fallback Tests

File: tests/test_cuopt_eviction.py

import pytest
from unittest.mock import patch

MEMORIES = [
    {"id": f"mem_{i}", "score": float(i) / 10, "pinned": False}
    for i in range(20)
]
SLOTS = 10

class TestCuoptEviction:
    def test_cpu_fallback_returns_correct_count(self):
        """solve_eviction_cpu keeps exactly SLOTS memories."""
        from src.cuopt_eviction import solve_eviction_cpu
        evict_ids = solve_eviction_cpu(MEMORIES, SLOTS)
        kept = len(MEMORIES) - len(evict_ids)
        assert kept == SLOTS

    def test_cpu_fallback_keeps_highest_scoring(self):
        """CPU path should keep the top-SLOTS memories by score."""
        from src.cuopt_eviction import solve_eviction_cpu
        evict_ids = set(solve_eviction_cpu(MEMORIES, SLOTS))
        # The 10 highest-score memories should be kept
        top10 = {m["id"] for m in sorted(MEMORIES, key=lambda x: -x["score"])[:SLOTS]}
        kept = {m["id"] for m in MEMORIES} - evict_ids
        assert kept == top10

    def test_pinned_memories_never_evicted(self):
        """Pinned memories must always be kept."""
        from src.cuopt_eviction import solve_eviction_cpu
        mems_with_pinned = [
            {"id": "pinned_0", "score": 0.0, "pinned": True},  # low score but pinned
            *MEMORIES[:19]
        ]
        evict_ids = set(solve_eviction_cpu(mems_with_pinned, SLOTS))
        assert "pinned_0" not in evict_ids

    def test_scipy_unavailable_falls_back_to_sort(self):
        """If scipy not installed, sort-by-score fallback must still work."""
        import builtins
        real_import = builtins.__import__
        def mock_import(name, *args, **kwargs):
            if name == "scipy":
                raise ImportError("mocked scipy unavailable")
            return real_import(name, *args, **kwargs)

        with patch("builtins.__import__", side_effect=mock_import):
            from src.cuopt_eviction import solve_eviction_cpu
            evict_ids = solve_eviction_cpu(MEMORIES, SLOTS)
            assert len(MEMORIES) - len(evict_ids) == SLOTS

    def test_cuopt_unavailable_uses_cpu_path(self):
        """If cuOpt not installed (no GPU), solve_eviction routes to CPU path."""
        with patch("ahe.gpu_detect.has_nvidia_gpu", return_value=False):
            from src.cuopt_eviction import solve_eviction
            evict_ids = solve_eviction(MEMORIES, SLOTS)
            assert len(MEMORIES) - len(evict_ids) == SLOTS

    def test_cuopt_import_error_with_gpu_falls_to_cpu(self):
        """Even with GPU, if cuOpt not installed, CPU path is used."""
        with patch("ahe.gpu_detect.has_nvidia_gpu", return_value=True), \
             patch("src.cuopt_eviction.solve_eviction_optimal",
                   side_effect=ImportError("cuOpt not installed")):
            from src.cuopt_eviction import solve_eviction
            evict_ids = solve_eviction(MEMORIES, SLOTS)
            assert len(MEMORIES) - len(evict_ids) == SLOTS

---

4. EmbeddingService CPU Fallback Tests (TypeScript)

File: src/embedding.test.ts

import { describe, it, expect, beforeEach, mock } from "bun:test";

describe("EmbeddingService — CPU-only environment", () => {
  it("marks available=false when Ollama is unreachable", async () => {
    const fetchMock = mock(() => Promise.reject(new Error("ECONNREFUSED")));
    globalThis.fetch = fetchMock as any;

    const { EmbeddingService } = await import("./index");
    const svc = new EmbeddingService();
    const result = await svc.embed("test content");

    expect(result).toBeNull();
    expect(svc.isAvailable()).toBe(false);
  });

  it("returns cached embedding on second call without hitting Ollama", async () => {
    let callCount = 0;
    globalThis.fetch = mock(async () => {
      callCount++;
      return { ok: true, json: async () => ({ embeddings: [[0.1, 0.2, 0.3]] }) };
    }) as any;

    const { EmbeddingService } = await import("./index");
    const svc = new EmbeddingService();
    await svc.embed("same content");
    await svc.embed("same content");

    expect(callCount).toBe(1);  // second call served from LRU cache (#43)
  });

  it("queryMemories falls back to FTS5 when embedding unavailable", async () => {
    // Simulate no Ollama: vecAvailable=false
    // memory_query should still return results via FTS5
    // (Verified via existing tests/test_integration.py pattern)
  });
});

---

5. Skill Compatibility Check — No GPU

File: tests/test_skill_compatibility_no_gpu.py

from unittest.mock import patch
import pytest

class TestSkillCompatibilityNoGpu:
    def test_cuopt_skill_detected_incompatible_without_gpu(self):
        """cuopt-install skill compatibility check should warn without GPU."""
        from ahe.run_eval import inject_skills
        # cuopt skills have `compatibility: requires CUDA >= 7.0`
        # skill_check_compatibility should return compatible=False on CPU-only
        with patch("ahe.gpu_detect.has_nvidia_gpu", return_value=False):
            # The skill is still injected (knowledge is useful regardless)
            # but a ⚠ warning is prepended
            augmented, ids, _, _ = inject_skills(
                "install cuopt", {"description": "install cuopt", "task_id": "t001"}, no_skills=False
            )
            # Skills should still be injected (text knowledge is always useful)
            # but must NOT fail or raise
            assert isinstance(augmented, str)

    def test_aiq_research_skill_works_without_local_gpu(self):
        """aiq-research skill works via cloud API even without local GPU."""
        # AI-Q Blueprint can use NVIDIA_API_KEY for cloud inference
        # No GPU detection should block this skill
        pass  # documented: aiq-research does not have GPU compatibility requirement

    def test_skill_install_succeeds_without_gpu(self):
        """skill_install should always succeed — it's just downloading SKILL.md."""
        # NVIDIA skills are plain text; no GPU needed to install them
        pass

---

6. CI Workflow

File: .github/workflows/test-cpu-only.yml

name: CPU-only compatibility
on:
  push:
    branches: [main]
  pull_request:
    paths:
      - 'ahe/**'
      - 'src/**'
      - 'tests/**'

jobs:
  test-cpu-only:
    runs-on: ubuntu-latest   # GitHub-hosted — no NVIDIA GPU
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          pip install -r requirements.txt
          pip install scipy   # CPU LP fallback for cuOpt tests

      - name: Verify no GPU present
        run: |
          if nvidia-smi &>/dev/null; then
            echo "ERROR: GPU found on CI runner — test must run CPU-only"
            exit 1
          fi
          echo "Confirmed: no GPU present"

      - name: Run tests (CPU-only env)
        run: pytest tests/ -v --timeout=60 -m "not requires_gpu"
        env:
          MEMORY_EMBED_MODEL: "nomic-embed-text"
          MEMORY_EVICTION: "lru"
          # No NVIDIA_API_KEY — AI-Q tests skipped via marker

      - name: Run bun tests (TypeScript)
        run: bun test src/
        env:
          MEMORY_EMBED_ENDPOINT: "http://localhost:99999"  # unreachable — forces fallback

pytest markers to add in pytest.ini

[pytest]
markers =
    requires_gpu: mark test as requiring NVIDIA GPU (skipped in CPU-only CI)
    requires_ollama: mark test as requiring Ollama running locally
    requires_cuopt: mark test as requiring cuOpt installed

---

Acceptance Criteria

• tests/test_gpu_detect.py — all 6 test cases pass on CPU-only
• tests/test_vllm_fallback.py — vllm-local + trtllm-local both fall back correctly
• tests/test_cuopt_eviction.py — CPU path + scipy fallback + sort fallback all pass
• src/embedding.test.ts — EmbeddingService graceful degradation passes
• tests/test_skill_compatibility_no_gpu.py — skill install/inject unblocked without GPU
• .github/workflows/test-cpu-only.yml — CI passes on GPU-free runner
• pytest.ini updated with requires_gpu, requires_ollama, requires_cuopt markers
• Total test count on CPU-only CI ≥ 157 (existing) + new tests

Blocks

• All NVIDIA series PRs ([NVIDIA #2] Add vLLM local backend with RadixAttention prefix caching to AHE eval #50, [NVIDIA #3] Upgrade embedding backend: Ollama nomic-embed-text → nv-embed-v2 on GB10 #51, [NVIDIA #5] GPU-accelerated memory eviction scheduling via cuOpt (replace LRU) #53, [NVIDIA #6] TensorRT-LLM as ultra-low-latency AHE eval backend on GB10 #54) must pass this test suite before merge
• References: [NVIDIA #8] GPU-agnostic compatibility layer — all features must degrade gracefully without NVIDIA GPU #56 (compatibility contract)