fix(test): reliable nightly GPU runs — phased execution as default, script-owned timings

mellea/backends/adapters/adapter.py

@@ -1454,8 +1454,17 @@ def from_hub(
         local_root = cache_root / "mellea" / "embedded-adapter-configs" / cache_key
 
         try:
-            if not local_root.is_dir():
+            # Validate the cache by content, not just directory existence: a
+            # prior run that crashed or lost a `temporary_dir.replace()` race
+            # (see below) can leave `local_root` as a directory missing
+            # `adapter_index.json`, which `is_dir()` alone would trust forever.
+            if not (local_root / "adapter_index.json").is_file():
                 local_root.parent.mkdir(parents=True, exist_ok=True)
+                if local_root.is_dir():
+                    # Invalid leftover from a prior partial run -- clear it so
+                    # `temporary_dir.replace(local_root)` below doesn't fail
+                    # trying to rename onto a non-empty stale directory.
+                    shutil.rmtree(local_root)
                 temporary_dir = pathlib.Path(
                     tempfile.mkdtemp(dir=local_root.parent, prefix=f"{cache_key}-")
                 )

test/README.md

@@ -416,8 +416,10 @@ uv run pytest -rs
 # CI mode locally (mirrors what PR CI does)
 CICD=1 uv run pytest test
 
-# Nightly-style local run on a GPU host
-./test/scripts/run_tests_with_ollama_and_vllm.sh --group-by-backend -v -s
+# Nightly-style local run on a GPU host (phased execution is the default)
+./test/scripts/run_tests_with_ollama_and_vllm.sh
+# Legacy single-process run (both servers up for the whole run)
+SERIAL_PHASES=0 ./test/scripts/run_tests_with_ollama_and_vllm.sh --group-by-backend -v -s
 ```
 
 ### Scoping a test run
@@ -453,7 +455,7 @@ Tests skip automatically when requirements are not met:
 | **Pre-commit** | Every commit (local) | Local hook | ruff, mypy, uv-lock, codespell, markdownlint |
 | **PR CI** | Every push / merge group | GitHub Actions, Ubuntu | `pytest test/` on Python 3.11/3.12/3.13 with Ollama. `CICD=1` (qualitative skipped). `slow` excluded. |
 | **Notebooks** | Every PR (advisory) | GitHub Actions, Ubuntu | `pytest --nbmake docs/examples/notebooks` on Python 3.12 with Ollama. `slow` notebooks excluded. |
-| **Nightly** | Scheduled | IBM internal LSF cluster (GPU) | Full `pytest test/ --group-by-backend`, Ollama + vLLM, qualitative enabled. All notebooks including `slow` when `WITH_EXAMPLES=1`. Failures file an auto-issue. |
+| **Nightly** | Scheduled | IBM internal LSF cluster (GPU) | Full suite via `run_tests_with_ollama_and_vllm.sh` (phased execution by default: one pytest process per backend group, per-phase server lifecycles; `SERIAL_PHASES=0` for the legacy `--group-by-backend` run), Ollama + vLLM, qualitative enabled. All notebooks including `slow` when `WITH_EXAMPLES=1`. Failures file an auto-issue. |
 | **On-demand nightly** | Not yet available | IBM internal LSF cluster | Comment-triggered nightly against a PR branch. Tracked in [#734](https://github.com/generative-computing/mellea/issues/734); ask a maintainer if you need pre-merge GPU validation today. |
 
 **PR CI** (`ci.yml` → `quality.yml`): pre-commit checks, then Ollama installed
@@ -469,10 +471,21 @@ just a PR. So this is advisory: it reports on every PR but is not part of
 `code-checks`. Installs Ollama, pulls `granite4.2:3b`, then runs
 `pytest --nbmake docs/examples/notebooks`. See [Notebooks](#notebooks) below.
 
-**Nightly** (`test/scripts/run_tests_with_ollama_and_vllm.sh`): starts local
-Ollama and (when GPU present) a local vLLM server, then runs
-`pytest test/ --group-by-backend`. The `--group-by-backend` flag reorders tests
-to run each backend as a contiguous group, reducing GPU memory fragmentation.
+**Nightly** (`test/scripts/run_tests_with_ollama_and_vllm.sh`): starts a local
+Ollama and (when a GPU is present) a local vLLM server, then runs the suite.
+By default it runs **phased execution** (`SERIAL_PHASES=1`): each backend group
+runs as its own pytest process with per-phase server lifecycles — only one CUDA
+context is alive at a time, peak GPU memory drops from ~72 GiB to ~33 GiB, and
+the single-context phases can run on an exclusive LSF GPU
+(`-gpu "num=1"`; the Ollama phase needs `mode=shared` because Ollama runs one
+`llama-server` worker per model). `SERIAL_PHASES=0` selects the legacy
+single-process run, which starts both servers up front and runs
+`pytest test/ --group-by-backend` (the flag reorders tests into contiguous
+backend groups, reducing GPU memory fragmentation). The script records full
+per-test durations and JSON reports under its log directory
+(`pytest_full.log` + `pytest_report.json` legacy; `phase_<name>.log` +
+`pytest_report_<name>.json` per phase) — see the script header for the LSF
+submission patterns (single job vs the recommended two-job exclusive split).
 With `WITH_EXAMPLES=1` it follows up with a notebook pass using `-m e2e`, which
 replaces the configured `-m "not slow"` and so covers the notebooks PR CI skips.
 
@@ -589,7 +602,9 @@ Two mechanisms in `test/conftest.py` handle this:
   Always active, no flags required.
 - **Group warm-up/eviction** (`pytest_runtest_setup`) — warms up a fixed set of CI
   models (`keep_alive=-1`) when entering the Ollama backend group and evicts them
-  when leaving. Requires `--group-by-backend`.
+  when leaving. Triggered by `--group-by-backend` ordering in legacy mode, and in
+  phased mode (default) by the per-phase `--group-by-backend` the script passes —
+  each phase process contains a single group.
 
 **Trade-off:** if two consecutive test files use the same model, it will be unloaded
 and reloaded (~5–15 s overhead). Predictable memory behaviour is more important

test/backends/test_model_ids.py

@@ -122,10 +122,32 @@ def test_ollama_model_names_exist(const_name: str, ollama_name: str) -> None:
     except ConnectionError as e:
         pytest.skip(f"Ollama server unreachable: {e}")
 
-    # Check the Ollama registry manifest without downloading anything.
     # Strip any tag so we query the base model name.
     base_name = ollama_name.split(":")[0]
     tag = ollama_name.split(":")[1] if ":" in ollama_name else "latest"
+
+    # "hf.co/<repo>[:tag]" is Ollama's syntax for importing a GGUF file directly
+    # from the Hugging Face Hub -- these are never hosted on registry.ollama.ai,
+    # so validate against the Hub instead (mirrors test_hf_model_names_exist).
+    if base_name.startswith("hf.co/"):
+        pytest.importorskip("huggingface_hub", reason="huggingface_hub not installed")
+        from huggingface_hub import model_info
+        from huggingface_hub.errors import GatedRepoError, RepositoryNotFoundError
+
+        hf_repo = base_name.removeprefix("hf.co/")
+        try:
+            model_info(hf_repo, token=False)
+        except GatedRepoError:
+            pass
+        except RepositoryNotFoundError:
+            pytest.skip(
+                f"{const_name}.ollama_name={ollama_name!r} (hf.co repo {hf_repo!r}) "
+                "not found on HuggingFace Hub (may be gated -- re-run with "
+                "HF_TOKEN to confirm it exists)."
+            )
+        return
+
+    # Check the Ollama registry manifest without downloading anything.
     # Unnamespaced models (no "/" in base_name) live under the "library" namespace.
     registry_name = base_name if "/" in base_name else f"library/{base_name}"
     registry_url = f"https://registry.ollama.ai/v2/{registry_name}/manifests/{tag}"