From af7d372702878e13ba59b7688b6cb995254ed216 Mon Sep 17 00:00:00 2001 From: Jiacheng Xu <169599847+Stardust0831@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:01:24 +0800 Subject: [PATCH] test: expand SAI GPU validation matrix --- .ci/slurm/README.md | 2 +- .ci/slurm/config.ini | 534 ++++++++++++++++++++++++++++++++++++--- .ci/slurm/runner.py | 79 +++++- .ci/slurm/test_runner.py | 97 ++++++- 4 files changed, 655 insertions(+), 57 deletions(-) diff --git a/.ci/slurm/README.md b/.ci/slurm/README.md index efb0e8efd5..f8da74ad23 100644 --- a/.ci/slurm/README.md +++ b/.ci/slurm/README.md @@ -97,7 +97,7 @@ python3 .ci/slurm/runner.py run --help - `[resource.NAME]`: the same allocation fields plus `parallelism`, the maximum number of array tasks running at once. Each resource must have a case. There is one rank per GPU and no resource may exceed 16 GPUs. - `[case.NNN]`: contiguous, zero-padded sections with `suite`, `name`, `resource`, and `runner` (`autotest` or `cusolvermp`). -Resource component labels are generated, not configured separately. A single-node resource is shown as `N GPU` or `N GPUs`; a multi-node resource is shown as `N nodes / M GPUs`. Thus `gpu1`, `gpu2`, and `gpu4` display `1 GPU`, `2 GPUs`, and `4 GPUs`; `gpu8x2` displays `2 nodes / 16 GPUs`. `case.049` is `15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU`; it uses `gpu8x2` and the `cusolvermp` runner. +Resource labels are generated, not configured separately. A single-node resource is shown as `N GPU` or `N GPUs`; a multi-node resource is shown as `N nodes / M GPUs`. Thus `gpu1`, `gpu2`, and `gpu4` display `1 GPU`, `2 GPUs`, and `4 GPUs`; `gpu4x2` displays `2 nodes / 8 GPUs`. Test results are reported by their `tests/` folder even though Slurm arrays remain grouped by resource. `case.049` is `15_rtTDDFT_GPU/19_NO_Si48_CUSOLVERMP_TDDFT_GPU`; it uses `gpu4x2` and the `cusolvermp` runner. ## Results and retention diff --git a/.ci/slurm/config.ini b/.ci/slurm/config.ini index fbca522c4c..328cd3a9c3 100644 --- a/.ci/slurm/config.ini +++ b/.ci/slurm/config.ini @@ -47,14 +47,22 @@ gpus_per_node = 4 time_seconds = 900 parallelism = 16 -[resource.gpu8x2] +[resource.gpu4x2] qos = flood-gpu nodes = 2 -tasks_per_node = 8 -gpus_per_node = 8 +tasks_per_node = 4 +gpus_per_node = 4 time_seconds = 2400 parallelism = 1 +[resource.pw_gpu1] +qos = flood-1o2gpu +nodes = 1 +tasks_per_node = 1 +gpus_per_node = 1 +time_seconds = 900 +parallelism = 8 + [case.001] suite = 11_PW_GPU name = scf_bpcg @@ -68,17 +76,17 @@ runner = autotest [case.003] suite = 11_PW_GPU name = scf_cg_single -resource = gpu4 +resource = gpu2 runner = autotest [case.004] suite = 11_PW_GPU name = scf_dav -resource = gpu4 +resource = gpu2 runner = autotest [case.005] suite = 11_PW_GPU name = scf_dav_sub -resource = gpu4 +resource = gpu2 runner = autotest [case.006] suite = 11_PW_GPU @@ -88,87 +96,87 @@ runner = autotest [case.007] suite = 11_PW_GPU name = scf_out_wf_norm -resource = gpu4 +resource = gpu2 runner = autotest [case.008] suite = 12_NAO_Gamma_GPU name = 001_NO_BiSeCuO_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.009] suite = 12_NAO_Gamma_GPU name = 002_NO_H2O_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.010] suite = 12_NAO_Gamma_GPU name = 003_NO_H2_DZP_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.011] suite = 12_NAO_Gamma_GPU name = 004_NO_H2_DZP_S2_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.012] suite = 12_NAO_Gamma_GPU name = 005_NO_H2_SZ_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.013] suite = 12_NAO_Gamma_GPU name = 006_NO_H2_SZ_S2_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.014] suite = 12_NAO_Gamma_GPU name = 007_NO_H_DZP_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.015] suite = 12_NAO_Gamma_GPU name = 008_NO_H_DZP_S2_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.016] suite = 12_NAO_Gamma_GPU name = 009_NO_Si2_DZP_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.017] suite = 12_NAO_Gamma_GPU name = 010_NO_Si2_DZP_NEQ_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.018] suite = 12_NAO_Gamma_GPU name = 011_NO_Si2_DZP_NEQ_S2_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.019] suite = 12_NAO_Gamma_GPU name = 012_NO_Si2_DZP_S2_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.020] suite = 12_NAO_Gamma_GPU name = 013_NO_Si2_TZDP_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.021] suite = 12_NAO_Gamma_GPU name = 014_NO_Si2_TZDP_NEQ_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.022] suite = 12_NAO_Gamma_GPU name = 015_NO_Si2_TZDP_NEQ_S2_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.023] suite = 12_NAO_Gamma_GPU name = 016_NO_Si2_TZDP_S2_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.024] suite = 13_NAO_multik_GPU @@ -178,62 +186,62 @@ runner = autotest [case.025] suite = 13_NAO_multik_GPU name = 002_NO_KP_Si2_DZP_NEQ_S2_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.026] suite = 13_NAO_multik_GPU name = 003_NO_KP_Si2_TZDP_S2_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.027] suite = 15_rtTDDFT_GPU name = 01_NO_KP_ocp_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.028] suite = 15_rtTDDFT_GPU name = 02_NO_CH_OW_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.029] suite = 15_rtTDDFT_GPU name = 03_NO_CO_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.030] suite = 15_rtTDDFT_GPU name = 04_NO_CO_ocp_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.031] suite = 15_rtTDDFT_GPU name = 05_NO_cur_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.032] suite = 15_rtTDDFT_GPU name = 06_NO_dir_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.033] suite = 15_rtTDDFT_GPU name = 07_NO_EDM_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.034] suite = 15_rtTDDFT_GPU name = 09_NO_HEAV_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.035] suite = 15_rtTDDFT_GPU name = 10_NO_HHG_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.036] suite = 15_rtTDDFT_GPU name = 11_NO_O3_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.037] suite = 15_rtTDDFT_GPU @@ -243,27 +251,27 @@ runner = autotest [case.038] suite = 15_rtTDDFT_GPU name = 14_NO_TRAP_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.039] suite = 15_rtTDDFT_GPU name = 15_NO_TRI_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.040] suite = 15_rtTDDFT_GPU name = 16_NO_vel_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.041] suite = 15_rtTDDFT_GPU name = 17_NO_vel_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.042] suite = 15_rtTDDFT_GPU name = 18_NO_hyb_TDDFT_GPU -resource = gpu4 +resource = gpu2 runner = autotest [case.043] suite = 16_SDFT_GPU @@ -298,5 +306,449 @@ runner = autotest [case.049] suite = 15_rtTDDFT_GPU name = 19_NO_Si48_CUSOLVERMP_TDDFT_GPU -resource = gpu8x2 +resource = gpu4x2 runner = cusolvermp + +[case.050] +suite = 01_PW +name = nscf_out_pot +resource = pw_gpu1 +runner = autotest_gpu + +[case.051] +suite = 01_PW +name = scf_out_elf +resource = pw_gpu1 +runner = autotest_gpu + +[case.052] +suite = 01_PW +name = 001_PW_UPF100_Al +resource = pw_gpu1 +runner = autotest_gpu + +[case.053] +suite = 01_PW +name = 002_PW_UPF100_RAPPE_Fe +resource = pw_gpu1 +runner = autotest_gpu + +[case.054] +suite = 01_PW +name = 003_PW_UPF100_USPP_Fe +resource = pw_gpu1 +runner = autotest_gpu + +[case.055] +suite = 01_PW +name = 004_PW_UPF201_Si +resource = pw_gpu1 +runner = autotest_gpu + +[case.056] +suite = 01_PW +name = 005_PW_UPF201_UPF100 +resource = pw_gpu1 +runner = autotest_gpu + +[case.057] +suite = 01_PW +name = 006_PW_UPF201_Eu +resource = pw_gpu1 +runner = autotest_gpu + +[case.058] +suite = 01_PW +name = 008_PW_UPF201_USPP_NaCl +resource = pw_gpu1 +runner = autotest_gpu + +[case.059] +suite = 01_PW +name = 009_PW_UPF201_USPP +resource = pw_gpu1 +runner = autotest_gpu + +[case.060] +suite = 01_PW +name = 010_PW_0TYPE +resource = pw_gpu1 +runner = autotest_gpu + +[case.061] +suite = 01_PW +name = 011_PW_0ATOM +resource = pw_gpu1 +runner = autotest_gpu + +[case.062] +suite = 01_PW +name = 012_PW_DJ +resource = pw_gpu1 +runner = autotest_gpu + +[case.063] +suite = 01_PW +name = 013_PW_ONCV_LDA +resource = pw_gpu1 +runner = autotest_gpu + +[case.064] +suite = 01_PW +name = 014_PW_UPF201_BLPS +resource = pw_gpu1 +runner = autotest_gpu + +[case.065] +suite = 01_PW +name = 015_PW_GTH +resource = pw_gpu1 +runner = autotest_gpu + +[case.066] +suite = 01_PW +name = 020_PW_kspace +resource = pw_gpu1 +runner = autotest_gpu + +[case.067] +suite = 01_PW +name = 021_PW_kspace3 +resource = pw_gpu1 +runner = autotest_gpu + +[case.068] +suite = 01_PW +name = 022_PW_CG +resource = pw_gpu1 +runner = autotest_gpu + +[case.069] +suite = 01_PW +name = 023_PW_DA +resource = pw_gpu1 +runner = autotest_gpu + +[case.070] +suite = 01_PW +name = 024_PW_DS +resource = pw_gpu1 +runner = autotest_gpu + +[case.071] +suite = 01_PW +name = 025_PW_DS_sca +resource = pw_gpu1 +runner = autotest_gpu + +[case.072] +suite = 01_PW +name = 027_PW_PINT_RKS +resource = pw_gpu1 +runner = autotest_gpu + +[case.073] +suite = 01_PW +name = 028_PW_PINT_UKS +resource = pw_gpu1 +runner = autotest_gpu + +[case.074] +suite = 01_PW +name = 029_PW_15_CF_CS_S1_smallg +resource = pw_gpu1 +runner = autotest_gpu + +[case.075] +suite = 01_PW +name = 030_PW_15_CF_CS_S2_smallg +resource = pw_gpu1 +runner = autotest_gpu + +[case.076] +suite = 01_PW +name = 031_PW_15_CF_CS +resource = pw_gpu1 +runner = autotest_gpu + +[case.077] +suite = 01_PW +name = 032_PW_15_CF_CS_bspline +resource = pw_gpu1 +runner = autotest_gpu + +[case.078] +suite = 01_PW +name = 033_PW_CF_CS_S1_smallg +resource = pw_gpu1 +runner = autotest_gpu + +[case.079] +suite = 01_PW +name = 034_PW_CF_CS_S2_smallg +resource = pw_gpu1 +runner = autotest_gpu + +[case.080] +suite = 01_PW +name = 035_PW_15_SO +resource = pw_gpu1 +runner = autotest_gpu + +[case.081] +suite = 01_PW +name = 036_PW_AF +resource = pw_gpu1 +runner = autotest_gpu + +[case.082] +suite = 01_PW +name = 037_PW_FM +resource = pw_gpu1 +runner = autotest_gpu + +[case.083] +suite = 01_PW +name = 039_PW_FD_smear +resource = pw_gpu1 +runner = autotest_gpu + +[case.084] +suite = 01_PW +name = 040_PW_FX_smear +resource = pw_gpu1 +runner = autotest_gpu + +[case.085] +suite = 01_PW +name = 041_PW_GA_smear +resource = pw_gpu1 +runner = autotest_gpu + +[case.086] +suite = 01_PW +name = 042_PW_M2_smear +resource = pw_gpu1 +runner = autotest_gpu + +[case.087] +suite = 01_PW +name = 043_PW_MP_smear +resource = pw_gpu1 +runner = autotest_gpu + +[case.088] +suite = 01_PW +name = 044_PW_MV_smear +resource = pw_gpu1 +runner = autotest_gpu + +[case.089] +suite = 01_PW +name = 045_PW_BD_chgmix +resource = pw_gpu1 +runner = autotest_gpu + +[case.090] +suite = 01_PW +name = 046_PW_KK_chgmix +resource = pw_gpu1 +runner = autotest_gpu + +[case.091] +suite = 01_PW +name = 047_PW_PK_chgmix +resource = pw_gpu1 +runner = autotest_gpu + +[case.092] +suite = 01_PW +name = 048_PW_PL_chgmix +resource = pw_gpu1 +runner = autotest_gpu + +[case.093] +suite = 01_PW +name = 049_PW_PU_chgmix +resource = pw_gpu1 +runner = autotest_gpu + +[case.094] +suite = 01_PW +name = 050_PW_CHG_mismatch +resource = pw_gpu1 +runner = autotest_gpu + +[case.095] +suite = 01_PW +name = 051_PW_OBOD_MemSaver +resource = pw_gpu1 +runner = autotest_gpu + +[case.096] +suite = 01_PW +name = 053_PW_OD +resource = pw_gpu1 +runner = autotest_gpu + +[case.097] +suite = 01_PW +name = 056_PW_IW +resource = pw_gpu1 +runner = autotest_gpu + +[case.098] +suite = 01_PW +name = 058_PW_RE_MB +resource = pw_gpu1 +runner = autotest_gpu + +[case.099] +suite = 01_PW +name = 059_PW_RE_MB_traj +resource = pw_gpu1 +runner = autotest_gpu + +[case.100] +suite = 01_PW +name = 060_PW_RE_MG +resource = pw_gpu1 +runner = autotest_gpu + +[case.101] +suite = 01_PW +name = 066_PW_CR_fix_abc +resource = pw_gpu1 +runner = autotest_gpu + +[case.102] +suite = 01_PW +name = 073_PW_SY +resource = pw_gpu1 +runner = autotest_gpu + +[case.103] +suite = 01_PW +name = 074_PW_SY_LiRH +resource = pw_gpu1 +runner = autotest_gpu + +[case.104] +suite = 01_PW +name = 075_PW_CHG_BINARY +resource = pw_gpu1 +runner = autotest_gpu + +[case.105] +suite = 01_PW +name = 076_PW_elec_add +resource = pw_gpu1 +runner = autotest_gpu + +[case.106] +suite = 01_PW +name = 077_PW_elec_minus +resource = pw_gpu1 +runner = autotest_gpu + +[case.107] +suite = 01_PW +name = 078_PW_S2_elec_add +resource = pw_gpu1 +runner = autotest_gpu + +[case.108] +suite = 01_PW +name = 079_PW_S2_elec_minus +resource = pw_gpu1 +runner = autotest_gpu + +[case.109] +suite = 01_PW +name = 080_PW_dipole +resource = pw_gpu1 +runner = autotest_gpu + +[case.110] +suite = 01_PW +name = 081_PW_efield +resource = pw_gpu1 +runner = autotest_gpu + +[case.111] +suite = 01_PW +name = 082_PW_gatefield +resource = pw_gpu1 +runner = autotest_gpu + +[case.112] +suite = 01_PW +name = 083_PW_sol_H2 +resource = pw_gpu1 +runner = autotest_gpu + +[case.113] +suite = 01_PW +name = 084_PW_sol_H2O +resource = pw_gpu1 +runner = autotest_gpu + +[case.114] +suite = 01_PW +name = 085_PW_get_pchg +resource = pw_gpu1 +runner = autotest_gpu + +[case.115] +suite = 01_PW +name = 090_PW_VWR +resource = pw_gpu1 +runner = autotest_gpu + +[case.116] +suite = 01_PW +name = 091_PW_CR_VDW3 +resource = pw_gpu1 +runner = autotest_gpu + +[case.117] +suite = 01_PW +name = 094_PW_NPT +resource = pw_gpu1 +runner = autotest_gpu + +[case.118] +suite = 01_PW +name = 098_PW_15_SO_avg +resource = pw_gpu1 +runner = autotest_gpu + +[case.119] +suite = 01_PW +name = 101_PW_MD_1O +resource = pw_gpu1 +runner = autotest_gpu + +[case.120] +suite = 01_PW +name = 102_PW_MD_2O +resource = pw_gpu1 +runner = autotest_gpu + +[case.121] +suite = 01_PW +name = 209_PW_DFTHALF +resource = pw_gpu1 +runner = autotest_gpu + +[case.122] +suite = 01_PW +name = 210_PW_kspace_shift +resource = pw_gpu1 +runner = autotest_gpu + +[case.123] +suite = 07_OFDFT +name = 31_OF_KE_WT_GPU +resource = pw_gpu1 +runner = autotest diff --git a/.ci/slurm/runner.py b/.ci/slurm/runner.py index 1e033f3d73..8a3b29582a 100644 --- a/.ci/slurm/runner.py +++ b/.ci/slurm/runner.py @@ -200,7 +200,9 @@ def load_config(path: Path = ROOT / "config.ini") -> Config: case = Case(section["suite"], section["name"], section["resource"], section["runner"]) if not all(NAME.fullmatch(value) for value in (case.suite, case.name, case.resource)): raise ValueError("invalid case name") - if case.resource not in profiles or case.runner not in ("autotest", "cusolvermp"): + if case.resource not in profiles or case.runner not in ( + "autotest", "autotest_gpu", "cusolvermp", + ): raise ValueError("invalid case resource or runner") matrix.append(case) if len({case.case_id for case in matrix}) != len(matrix): @@ -272,6 +274,27 @@ def _component(name: str, label: str, state: str, job: str = "", slurm: str = "" } +def _folder_components(result: Mapping[str, Any]) -> List[Dict[str, str]]: + components = [dict(result["components"][0])] + suites: Dict[str, List[Mapping[str, Any]]] = {} + for row in result["cases"]: + suite = row["case_id"].split("/", 1)[0] + suites.setdefault(suite, []).append(row) + for suite in sorted(suites): + rows = suites[suite] + states = [row["state"] for row in rows] + state = "PASS" if all(item == "PASS" for item in states) else ( + "FAIL" if any(item in ("FAIL", "TIMEOUT") for item in states) else "INFRA" + ) + jobs = list(dict.fromkeys( + row["job_id"].split("_", 1)[0] for row in rows if row["job_id"] + )) + components.append(_component( + suite, "tests/" + suite, state, ", ".join(jobs), + )) + return components + + def _result_row(case: Case, state: str, **values: Any) -> Dict[str, Any]: row = { "case_id": case.case_id, "resource": case.resource, @@ -303,18 +326,19 @@ def _site_credit() -> str: def _result_markdown(result: Mapping[str, Any]) -> str: + components = _folder_components(result) lines = [ "# GPU validation result", "", "Passed: **{}**; failed: **{}**; infrastructure: **{}**".format( result["passed"], result["failed"], result["infrastructure"] - ), "", "| Component | State | Slurm job |", "| --- | --- | --- |", + ), "", "| Component | State | Slurm jobs |", "| --- | --- | --- |", ] - lines.extend("| {} | {} | {} |".format(item["label"], item["state"], item["job_id"]) for item in result["components"]) + lines.extend("| {} | {} | {} |".format(item["label"], item["state"], item["job_id"]) for item in components) lines.extend(("", "| Case | Resource | State | Duration | Slurm job |", "| --- | --- | --- | --- | --- |")) lines.extend("| {} | {} | {} | {} | {} |".format( row["case_id"], row["resource"], row["state"], _time(row["elapsed_seconds"]), row["job_id"], - ) for row in result["cases"]) + ) for row in sorted(result["cases"], key=lambda item: item["case_id"])) lines.extend(("", _site_credit())) return "\n".join(lines) + "\n" @@ -433,10 +457,44 @@ def _stream(command: Sequence[str], cwd: Path, log: Path) -> int: def _mpi_startup_failure(log: Path) -> bool: data = log.read_bytes() return ( - bool(PMIX.search(data)) and b"MPI_Init_thread" in data and b"PMIx_Init failed" in data + bool(PMIX.search(data)) and b"MPI_Init_thread" in data ) or bool(SRUN_DAEMON.search(data)) +def _force_gpu_inputs(case: Path, artifacts: Optional[Path] = None) -> None: + inputs = sorted(path for path in case.rglob("INPUT") if path.is_file()) + if not inputs: + raise ValueError("GPU autotest case has no INPUT") + for path in inputs: + text = path.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + active = [ + index for index, line in enumerate(lines) + if re.match(r"^\s*device(?:\s*=|\s+)", line) and not line.lstrip().startswith("#") + ] + if len(active) > 1: + raise ValueError("GPU autotest INPUT has duplicate device entries: {}".format(path)) + if active: + index = active[0] + ending = "\r\n" if lines[index].endswith("\r\n") else "\n" if lines[index].endswith("\n") else "" + body = lines[index][:-len(ending)] if ending else lines[index] + match = re.match( + r"^(\s*)device(?:\s*=|\s+)\s*\S+(\s*(?:#.*)?)$", body, + ) + if not match: + raise ValueError("invalid device entry in GPU autotest INPUT: {}".format(path)) + lines[index] = "{}device gpu{}{}".format(match.group(1), match.group(2), ending) + else: + if text and not text.endswith(("\n", "\r")): + lines.append("\n") + lines.append("device gpu\n") + path.write_text("".join(lines), encoding="utf-8") + if artifacts is not None: + destination = artifacts / "effective-inputs" / path.relative_to(case) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(str(path), str(destination)) + + def worker(source: Path, control: Path, install: Path, results: Path, manifest: Path) -> int: task_id = int(os.environ["SLURM_ARRAY_TASK_ID"]) fields = manifest.read_text(encoding="utf-8").splitlines()[task_id].split("\t") @@ -454,6 +512,8 @@ def worker(source: Path, control: Path, install: Path, results: Path, manifest: os.symlink(str(source / "tests" / "integrate"), str(tests / "integrate")) os.symlink(str(source / "tests" / "PP_ORB"), str(tests / "PP_ORB")) shutil.copytree(str(source / "tests" / suite / name), str(case)) + if runner == "autotest_gpu": + _force_gpu_inputs(case, artifacts) launcher = work / "launcher" launcher.mkdir() os.symlink(str(control / "mpirun_with_mapping.sh"), str(launcher / "mpirun")) @@ -467,7 +527,7 @@ def worker(source: Path, control: Path, install: Path, results: Path, manifest: returncode = 2 final_startup_failure = False try: - if runner == "autotest": + if runner in ("autotest", "autotest_gpu"): cases_file = case.parent / "CASES.task.txt" cases_file.write_text(name + "\n", encoding="utf-8") command = ( @@ -1006,7 +1066,7 @@ def _print_result( print("{} passed, {} failed, {} infrastructure\n".format( result["passed"], result["failed"], result["infrastructure"], )) - for component in result["components"]: + for component in _folder_components(result): print(" {:<24} {}".format(component["label"], component["state"])) print("\nSummary: {}".format(root / "results" / "summary.md")) print("Raw results: {}".format(root / "results")) @@ -1183,7 +1243,10 @@ def report(args: argparse.Namespace) -> int: values = {"available": "false", "passed": "", "failed": "", "infrastructure": "", "total": ""} else: result = _read_result(args.result) - components = [{key: item[key] for key in ("name", "label", "state")} for item in result["components"]] + components = [ + {key: item[key] for key in ("name", "label", "state")} + for item in _folder_components(result) + ] counts = {name: result[name] for name in ("passed", "failed", "infrastructure", "total")} values = {"available": "true", **{name: str(value) for name, value in counts.items()}} if args.summary: diff --git a/.ci/slurm/test_runner.py b/.ci/slurm/test_runner.py index 4518caed0a..05c92d7d60 100644 --- a/.ci/slurm/test_runner.py +++ b/.ci/slurm/test_runner.py @@ -42,11 +42,32 @@ def valid_result(): class ConfigTests(unittest.TestCase): def test_current_matrix_is_loaded_from_ini(self): config = runner.load_config() - self.assertEqual(len(config.cases), 49) - self.assertEqual(list(config.resources), ["gpu1", "gpu2", "gpu4", "gpu8x2"]) + self.assertEqual(len(config.cases), 123) + self.assertEqual(list(config.resources), ["gpu1", "gpu2", "gpu4", "gpu4x2", "pw_gpu1"]) self.assertEqual(config.resources["gpu4"].label, "4 GPUs") - self.assertEqual(config.resources["gpu8x2"].label, "2 nodes / 16 GPUs") - self.assertEqual(config.cases[-1].runner, "cusolvermp") + self.assertEqual(config.resources["gpu4x2"].label, "2 nodes / 8 GPUs") + resources = {(case.suite, case.name): case.resource for case in config.cases} + self.assertEqual(resources[("11_PW_GPU", "scf_cg")], "gpu4") + self.assertEqual(resources[("13_NAO_multik_GPU", "001_NO_KP_BiSeCuO_GPU")], "gpu4") + self.assertEqual(resources[("15_rtTDDFT_GPU", "12_NO_re_TDDFT_GPU")], "gpu4") + self.assertEqual(resources[("15_rtTDDFT_GPU", "19_NO_Si48_CUSOLVERMP_TDDFT_GPU")], "gpu4x2") + self.assertEqual( + {identity for identity, resource in resources.items() if resource == "gpu4"}, + { + ("11_PW_GPU", "scf_cg"), + ("13_NAO_multik_GPU", "001_NO_KP_BiSeCuO_GPU"), + ("15_rtTDDFT_GPU", "12_NO_re_TDDFT_GPU"), + }, + ) + pw_cases = [case for case in config.cases if case.suite == "01_PW"] + self.assertEqual(len(pw_cases), 73) + self.assertTrue(all(case.resource == "pw_gpu1" for case in pw_cases)) + self.assertTrue(all(case.runner == "autotest_gpu" for case in pw_cases)) + ofdft_cases = [case for case in config.cases if case.suite == "07_OFDFT"] + self.assertEqual( + ofdft_cases, + [runner.Case("07_OFDFT", "31_OF_KE_WT_GPU", "pw_gpu1", "autotest")], + ) self.assertEqual(config.site.name, "Open Source Supercomputing Center of SAI") self.assertEqual(config.site.url, "https://www.open-sai.com/") self.assertEqual(config.site.acknowledgement, "Computing resources were provided by") @@ -193,6 +214,55 @@ def test_modules_do_not_spell_dependency_paths(self): self.assertNotRegex(text, r"CUSOLVERMP_PATH|CUBLASMP_PATH|NCCL_PATH|/lib/lib") +class GpuInputTests(unittest.TestCase): + def test_replaces_existing_device_and_preserves_comment(self): + with tempfile.TemporaryDirectory() as directory: + case = Path(directory) / "case" + case.mkdir() + path = case / "INPUT" + path.write_text("INPUT_PARAMETERS\n device cpu # selected device\n", encoding="utf-8") + runner._force_gpu_inputs(case) + self.assertEqual( + path.read_text(encoding="utf-8"), + "INPUT_PARAMETERS\n device gpu # selected device\n", + ) + + def test_replaces_equals_form(self): + with tempfile.TemporaryDirectory() as directory: + case = Path(directory) / "case" + case.mkdir() + path = case / "INPUT" + path.write_text("device = cpu\n", encoding="utf-8") + runner._force_gpu_inputs(case) + self.assertEqual(path.read_text(encoding="utf-8"), "device gpu\n") + + def test_appends_device_and_archives_nested_input(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + case = root / "case" + nested = case / "nested" + nested.mkdir(parents=True) + path = nested / "INPUT" + path.write_text("INPUT_PARAMETERS", encoding="utf-8") + artifacts = root / "artifacts" + runner._force_gpu_inputs(case, artifacts) + self.assertEqual(path.read_text(encoding="utf-8"), "INPUT_PARAMETERS\ndevice gpu\n") + self.assertEqual( + (artifacts / "effective-inputs" / "nested" / "INPUT").read_text(encoding="utf-8"), + "INPUT_PARAMETERS\ndevice gpu\n", + ) + + def test_rejects_duplicate_active_device_entries(self): + with tempfile.TemporaryDirectory() as directory: + case = Path(directory) / "case" + case.mkdir() + (case / "INPUT").write_text( + "# device cpu\ndevice cpu\n device = gpu\n", encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "duplicate device"): + runner._force_gpu_inputs(case) + + class SlurmTests(unittest.TestCase): def test_wait_reports_array_progress(self): responses = [ @@ -745,9 +815,10 @@ def test_local_result_summary_is_concise_and_points_to_artifacts(self): runner._print_result(result, root, "/remote/archives/manual/1-1.tar.gz") text = output.getvalue() self.assertIn("GPU validation: PASS", text) - self.assertIn("49 passed, 0 failed, 0 infrastructure", text) + self.assertIn("123 passed, 0 failed, 0 infrastructure", text) self.assertIn("Compile PASS", text) - self.assertIn("2 nodes / 16 GPUs PASS", text) + self.assertIn("tests/01_PW PASS", text) + self.assertRegex(text, r"tests/15_rtTDDFT_GPU\s+PASS") self.assertIn("Summary: {}/results/summary.md".format(root.resolve()), text) self.assertIn("Raw results: {}/results".format(root.resolve()), text) self.assertIn("Remote archive: /remote/archives/manual/1-1.tar.gz", text) @@ -764,9 +835,11 @@ def test_report_publishes_dynamic_components(self): runner.report(args) output = args.output.read_text(encoding="utf-8") self.assertIn("available=true", output) - self.assertIn('"name":"gpu8x2"', output) + self.assertIn('"name":"01_PW","label":"tests/01_PW"', output) + self.assertIn('"name":"15_rtTDDFT_GPU","label":"tests/15_rtTDDFT_GPU"', output) summary = args.summary.read_text() self.assertTrue(summary.startswith("# GPU validation result\n")) + self.assertIn("| Component | State | Slurm jobs |", summary) self.assertIn("| Case | Resource | State | Duration | Slurm job |", summary) self.assertIn("| 11_PW_GPU/scf_out_wf | gpu1 | PASS | 00:00:10 | 102_0 |", summary) self.assertTrue(summary.rstrip().endswith( @@ -774,6 +847,14 @@ def test_report_publishes_dynamic_components(self): "[Open Source Supercomputing Center of SAI](https://www.open-sai.com/)." )) + def test_folder_components_aggregate_case_failures(self): + result = valid_result() + failed = next(row for row in result["cases"] if row["case_id"].startswith("12_NAO_Gamma_GPU/")) + failed["state"] = "FAIL" + components = {item["name"]: item for item in runner._folder_components(result)} + self.assertEqual(components["12_NAO_Gamma_GPU"]["state"], "FAIL") + self.assertEqual(components["13_NAO_multik_GPU"]["state"], "PASS") + def test_report_rejects_untrusted_counts(self): invalid = ( {"passed": "x[$(printf ARITH_EXEC >&2)0]", "failed": 0, "infrastructure": 0, "total": 1}, @@ -809,6 +890,8 @@ def test_mpi_startup_failure_requires_complete_signature(self): path = Path(directory) / "log" path.write_bytes(b"PMIX_ERR_FILE_OPEN_FAILURE MPI_Init_thread PMIx_Init failed") self.assertTrue(runner._mpi_startup_failure(path)) + path.write_bytes(b"PMIX_ERR_FILE_OPEN_FAILURE MPI_Init_thread Local abort before MPI_INIT") + self.assertTrue(runner._mpi_startup_failure(path)) path.write_bytes(b"PMIX_ERR_FILE_OPEN_FAILURE") self.assertFalse(runner._mpi_startup_failure(path)) path.write_bytes(b"srun returned non-zero exit status (512) from launching the per-node daemon")